diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index eea6a4717a1..d10da163f07 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -121,6 +121,7 @@ https://github.com/elastic/beats/compare/v6.4.0...master[Check the HEAD diff] - Add tag "multiline" to "log.flags" if event consists of multiple lines. {pull}7997[7997] - Add haproxy module. {pull}8014[8014] - Release `docker` input as GA. {pull}8328[8328] +- Keep original messages in case of Filebeat modules. {pull}8448[8448] *Heartbeat* diff --git a/filebeat/_meta/fields.common.yml b/filebeat/_meta/fields.common.yml index 930e0f67f90..aedfc08cf91 100644 --- a/filebeat/_meta/fields.common.yml +++ b/filebeat/_meta/fields.common.yml @@ -112,6 +112,12 @@ description: > This field contains the flags of the event. + - name: log.original + type: keyword + description: > + The unprocessed original log message. This can be used for reprocessing logs. + index: false + - name: event.created type: date description: > diff --git a/filebeat/channel/factory.go b/filebeat/channel/factory.go index 86db045c84f..0825bd5f5ab 100644 --- a/filebeat/channel/factory.go +++ b/filebeat/channel/factory.go @@ -43,6 +43,9 @@ type clientEventer struct { // inputOutletConfig defines common input settings // for the publisher pipeline. type inputOutletConfig struct { + // KeepOriginalMsg determines if the original message needs to be kept for a module. + KeepOriginalMsg bool `config:"keep_original_message"` + // event processing common.EventMetadata `config:",inline"` // Fields and tags to add to events. Processors processors.PluginConfig `config:"processors"` @@ -59,6 +62,10 @@ type inputOutletConfig struct { } +var defaultConfig = inputOutletConfig{ + KeepOriginalMsg: true, +} + // NewOutletFactory creates a new outlet factory for // connecting an input to the publisher pipeline. func NewOutletFactory( @@ -82,7 +89,7 @@ func NewOutletFactory( // This guarantees ordering between events as required by the registrar for // file.State updates func (f *OutletFactory) Create(p beat.Pipeline, cfg *common.Config, dynFields *common.MapStrPointer) (Outleter, error) { - config := inputOutletConfig{} + config := defaultConfig if err := cfg.Unpack(&config); err != nil { return nil, err } @@ -101,6 +108,7 @@ func (f *OutletFactory) Create(p beat.Pipeline, cfg *common.Config, dynFields *c meta := common.MapStr{} setMeta(meta, "pipeline", config.Pipeline) + keepOriginal := false fields := common.MapStr{} setMeta(fields, "module", config.Module) setMeta(fields, "name", config.Fileset) @@ -108,6 +116,8 @@ func (f *OutletFactory) Create(p beat.Pipeline, cfg *common.Config, dynFields *c fields = common.MapStr{ "fileset": fields, } + keepOriginal = config.KeepOriginalMsg + } if config.Type != "" { fields["prospector"] = common.MapStr{ @@ -119,13 +129,14 @@ func (f *OutletFactory) Create(p beat.Pipeline, cfg *common.Config, dynFields *c } client, err := p.ConnectWith(beat.ClientConfig{ - PublishMode: beat.GuaranteedSend, - EventMetadata: config.EventMetadata, - DynamicFields: dynFields, - Meta: meta, - Fields: fields, - Processor: processors, - Events: f.eventer, + PublishMode: beat.GuaranteedSend, + EventMetadata: config.EventMetadata, + DynamicFields: dynFields, + Meta: meta, + Fields: fields, + KeepOriginalMsg: keepOriginal, + Processor: processors, + Events: f.eventer, }) if err != nil { return nil, err diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 399928d810f..a876789827e 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -3042,6 +3042,18 @@ Logging level. This field contains the flags of the event. +-- + +*`log.original`*:: ++ +-- +type: keyword + +The unprocessed original log message. This can be used for reprocessing logs. + + +Field is not indexed. + -- *`event.created`*:: diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml index c91b2eb8a72..a6ae640e5ae 100644 --- a/filebeat/filebeat.reference.yml +++ b/filebeat/filebeat.reference.yml @@ -27,6 +27,8 @@ filebeat.modules: # Input configuration (advanced). Any input configuration option # can be added under this section. #input: + #Keeps the original message, so the data can be processed again on Ingest Node. + #keep_original_message: true # Authorization logs #auth: @@ -42,6 +44,8 @@ filebeat.modules: # Input configuration (advanced). Any input configuration option # can be added under this section. #input: + #Keeps the original message, so the data can be processed again on Ingest Node. + #keep_original_message: true #------------------------------- Apache2 Module ------------------------------ #- module: apache2 @@ -56,6 +60,8 @@ filebeat.modules: # Input configuration (advanced). Any input configuration option # can be added under this section. #input: + #Keeps the original message, so the data can be processed again on Ingest Node. + #keep_original_message: true # Error logs #error: @@ -68,6 +74,8 @@ filebeat.modules: # Input configuration (advanced). Any input configuration option # can be added under this section. #input: + #Keeps the original message, so the data can be processed again on Ingest Node. + #keep_original_message: true #------------------------------- Auditd Module ------------------------------- #- module: auditd @@ -81,6 +89,8 @@ filebeat.modules: # Input configuration (advanced). Any input configuration option # can be added under this section. #input: + #Keeps the original message, so the data can be processed again on Ingest Node. + #keep_original_message: true #---------------------------- elasticsearch Module --------------------------- - module: elasticsearch @@ -142,6 +152,8 @@ filebeat.modules: # Input configuration (advanced). Any input configuration option # can be added under this section. #input: + #Keeps the original message, so the data can be processed again on Ingest Node. + #keep_original_message: true # Debug logs #debug: @@ -154,6 +166,8 @@ filebeat.modules: # Input configuration (advanced). Any input configuration option # can be added under this section. #input: + #Keeps the original message, so the data can be processed again on Ingest Node. + #keep_original_message: true # Startup logs #startup: @@ -166,6 +180,8 @@ filebeat.modules: # Input configuration (advanced). Any input configuration option # can be added under this section. #input: + #Keeps the original message, so the data can be processed again on Ingest Node. + #keep_original_message: true #--------------------------------- IIS Module -------------------------------- #- module: iis @@ -180,6 +196,8 @@ filebeat.modules: # Input configuration (advanced). Any input configuration option # can be added under this section. #input: + #Keeps the original message, so the data can be processed again on Ingest Node. + #keep_original_message: true # Error logs #error: @@ -192,6 +210,8 @@ filebeat.modules: # Input configuration (advanced). Any input configuration option # can be added under this section. #input: + #Keeps the original message, so the data can be processed again on Ingest Node. + #keep_original_message: true #-------------------------------- Kafka Module ------------------------------- - module: kafka @@ -250,6 +270,8 @@ filebeat.modules: # Input configuration (advanced). Any input configuration option # can be added under this section. #input: + #Keeps the original message, so the data can be processed again on Ingest Node. + #keep_original_message: true #-------------------------------- MySQL Module ------------------------------- #- module: mysql @@ -264,6 +286,8 @@ filebeat.modules: # Input configuration (advanced). Any input configuration option # can be added under this section. #input: + #Keeps the original message, so the data can be processed again on Ingest Node. + #keep_original_message: true # Slow logs #slowlog: @@ -276,6 +300,8 @@ filebeat.modules: # Input configuration (advanced). Any input configuration option # can be added under this section. #input: + #Keeps the original message, so the data can be processed again on Ingest Node. + #keep_original_message: true #-------------------------------- Nginx Module ------------------------------- #- module: nginx @@ -290,6 +316,8 @@ filebeat.modules: # Input configuration (advanced). Any input configuration option # can be added under this section. #input: + #Keeps the original message, so the data can be processed again on Ingest Node. + #keep_original_message: true # Error logs #error: @@ -302,6 +330,8 @@ filebeat.modules: # Input configuration (advanced). Any input configuration option # can be added under this section. #input: + #Keeps the original message, so the data can be processed again on Ingest Node. + #keep_original_message: true #------------------------------- Osquery Module ------------------------------ - module: osquery @@ -330,6 +360,8 @@ filebeat.modules: # Input configuration (advanced). Any input configuration option # can be added under this section. #input: + #Keeps the original message, so the data can be processed again on Ingest Node. + #keep_original_message: true #-------------------------------- Redis Module ------------------------------- #- module: redis @@ -364,6 +396,8 @@ filebeat.modules: # Input configuration (advanced). Any input configuration option # can be added under this section. #input: + #Keeps the original message, so the data can be processed again on Ingest Node. + #keep_original_message: true #=========================== Filebeat inputs ============================= diff --git a/filebeat/include/fields.go b/filebeat/include/fields.go index c3c862b6682..bff2cdc5d10 100644 --- a/filebeat/include/fields.go +++ b/filebeat/include/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJzsfet33LiR7/f5K3D0JfY97R75Md7Ee+6edSR7rMQPxZKTm3XmtNAkuhsjEuAAoOSePfnf70EBIEESfHVT0mS3/WFGTYJVPzyrUKgqPEHXZPsKJXz9HUKKqoS8Qu/5Gq1oQlDEmSJMfYdQTGQkaKYoZ6/Qf3yHEEInnClMmdTfmuIJZUTOv0NoRUkSy1dQ7AliOCWvkOS5iAg8QkhtM/JKc77lIrbPBPklp4LEr5ASuSsY4Kv/XW6IYbkSPEW3GxptkNoYBOgWSyQIjufockOlAQNVAbS6GF5KnuSKoAyrDVIcHmp684LDWy4Q+YbTTDfI1fc3WHyf8PX3cisVSecJX1/Nv6vUj69WkqhK/RLO1o3KrXAih9bO0AR0gmRcKBKbKkqFhZIIqxqIlEiJ19VWVuSbg0XXjAuywEt+Q16h4x0b3o4KxFdlm+v2Np0Bj+yIqKGTShCcDhoCA1pJj1JDEd1uCAMIlK1dTxOhYcgZijBDS4J+J1XMc/U7xAX8TYT4XRVeJrjMSKS4mGtw3a2TCRJhpR+/nD/vbzPKslxBnetDltzottRjdk0YEZpmZeBSiWAMmEF6g5OcIA2TriiJCx4rLuD9lWZxhTiAQJTBQ8Nckgge2m57SxOyJFjp9lpR21/o0emb889vTl5fvjl9hSQh6Ao+hga5elxtr/LNjgPpX6RRqrXWw2yhaEqkwmnWXckzhiIsieW3JlKhjGYEZkyGhSRmOSqoVWeQnWdyhqhCUnFBZEFZl+GCrinDCbr6z4LCFXok9NiUhCk9GRx5M0Uc5coy+di0CC2JQxvXqq1bQhI1T3mcJwP6tmhJ8wFSG6zKzgR+ppdb+OhfI7jYzwazkVuZ8PV8hSOaULWdbtm2BBH5pgSONIaiTzNBuaBqG4bi3k4GxRF0Y9vw6WoNSW6I/mKR4CVJplqnNZZNnmKzQuNlQpBj1N0pdw7DMZo35EBEpJxngq/FdPJKA9AMXH9Y8m3MaTzdSKCxxxTIV5maMeF6ZTK+jqBjHhx6RNzQiPjzPdTSLVwuzNdAq0ZYj6SE3HQOoHbNYq0XT/g8QHaV4LXsq35Y84RPu9rDPIsE0etXBXqMVU+bV76t8tUfI86CItZ+MC/EFV8VJIsVQ5q1lMoOSYKW22JFNtR4mmFBJWcFwVJUaVrekNSrdVgOah5zdLZCS642CAuCaKzFW4STgixnydanLTc8T2Kt9+WS1GXZRqlsLojMOJNkLhVWuVxEPCZtI7+lvd9dXp4jRwd5dNw2othAvDh+0QWBJDiTxKgVIzG8MZ8aIb8k6paAKvxLrpUNzOISH2UopUlCtc7DWVxfA6qIrO6xSAhbq81ITCd2g2A+dqO92lpLHtfXXYsAoM9TojY8Hj93P9uqm+/n331nN7h6TJY73D+aX1272oinKWfIahd6P4vwDaYJSA7KEE4SO4c0usq2t1IrmAzDtBlfOmiESBIWOy1OzwS7vZMwG4pS8Jmnvmk9yCq5Ro3NBQYlVytJM/2cGT3J6M1UmjmiaVKlfzKufGLwCdpwqSwnW/6SI7c7LXDM9DujdOufV+UErSjfTVzzZqM5jgPkusMGC5HKBSOwGIGWnGldULei2btXV0EA7rWdyBmjbB1AoyfYr5wNQONK3iWaGyIkLZbVDjC2oBtWMJyHKslH5YJ61CaKgnu+FRcpVpVyxVL4Ol/nUqFnL9UGPTt++nKGnj579fyHVz88nz9//mxY65o1vhBEZhrqCSJIxEVc2zhWK6V6ZfdrsaRKYLGFsqa1rBFBj/eMCNNRenXVP5TATGLYR5b7s21WV0jM6lBpR778mURurpkfixFrXbFW5ZKIck6BagvM6rqFEFxUAKwFz3v2sG/0R24FtDqFHr84jqkuixNE2YrrmW2VB8NHOiHoGwNRq60KhexVHbBKaJbOvMHAk+goJL0GUffFuTeISqMHapFPg6ibYWJFVJTwPC5l1In+qbWjGxoTXU2FY6xwWGx9sG+N5hRVPpW6r8olCMfxAgosHEmngnHRKsV00Tl8NXdk6xObRD2z96Mn3qoI5+icS0n1wAWZJEHLI9GzGVpHZIa4QDFdU4UTHhHMGkbPAhtlUmEWkQXtmTpntiA6O3WQtBBBKY42Wt3s59AvmQoevlwfxsUWWHjjrGhn9WyekpjmaTf3D4aEMa+NYm7VHLMH90RegSCXTwiW6snTqGch9QghkIi0lHZUGjhUlmKuY8jB2lj0agHFvnnybfjQs59oLD9yvk6ImWnt3AVZ94raz1Cmr352osc8uob5Y2f6qfsdIG7eweZCL79JQkqjknmn56zccKEWRgKU23PMog0Xjt+TYpa3nNAUsFBQPrSt44W9fU7j/dbEL4z+khPPgE/j0KpesEtD4mMUR39cADmnnVoAWpFY5jRRiLMuKN5isCOSk4KnsWW08wKrmGxwq+gSqFuf6MFyBi1h+BSDVg/mcsi+M78CRM60MuANVGuDry495djUz3tHpuU9blzu3yfv7Lai2RsTjXSzQAQGORbRhioSqVxMUIcKOfSIzNdz9O33LxcvX8wQFukMZVk0QynN5OMmFC7nWYKVVun3Q/LpAjlCFkNEmOJyhvJlzlQ+Q7eUxfy2BUR1x7M7BksnyGOFU5ps92ZhyNhKChJvsJqhmCwpZjO0EoQsZdxT22siGEn2Q3IZ2G/+TiJDur0daNZgW3nUwfE9lXAsfHb+BMexIFIS2WSQ4mi/ijk2GyziWyxIyWyGcpnjJNmiD69PfAxuFbvOl7r6Co7S7Fr2Z/9ZgG35vlDCqxp1SRT5K1m3UC4/6l3+KqDRqEUw4/EEwslrgYzHZmUNssr3XRg9Tuc8Rl/OTpuM9H9lhqPpKlVSbDLT+79JW1BTbGnCoaJ9GCNDDaU4a3LCjHEF1rfJ2HkkwzynVJc8vlFFc+piO4HCGORr6NoVBmc42pBn5fJy9No8OQqvLvYt+uAO1qvLhrWqhZaFkhMaY9BxDJ2JyDxtW0BwpJemRqP5fHqarLBSWRuS0wgdjneXl+enlg+47My9z+uwUMUTI+WKLCrCqatbe3AC1oQSptDZObKyYx7knEsiFrVBvCdnLazBjAfGglyS2Ng3l1jSCOFcbcyRl7GiWxN8EFzl5GQIsmIz/eOby/Gg3VkTHO+4U5dgo4lk2uaqcP7y+X2Y7UapbNFUHifgD3wbahSqjFBz2rWomSJRmzlyDOfiKK1qovT5L3m8XUjC1Hy5VUQOReDM96GPBqBjebokQitoQKDwPyHihoj6CWC42VZEiMIUUcW7X3c50mHGeG28XZtca0bpASxP/OP3nD0Bf6/YzHHgg6QSlK3n6BNLtsj6bCFqGksXa5A0n71JsFQ0kkTv6lCW5GvK7Kmdd0LJBTxoXyZgDWuvcH2BH1tjW90vZXWNV9lUtS1rilkcqGZYdPgNEJMbGtVnJeoZZwOaAYV8fTZbSSOcWKZ1qP7e6GfebIqOuToCENCuHwiW47EDFGV3B0rT3gVUhlW0ubveA/K74AqoBUNgFUL4ZCN4kMIOw24IXl5f4Yeg3QFL3RLRhWhx79NgHLr7ng+j0O04AO+0SwW+bYUUEK0D8SD0Gd/6EtU4tCzJigsjgTS25db6Xj/RJZ+YkkaQhCXjmvCWPcU+QvFHws/O4ahcK1e6c9dYbYggsdbxSYw4s4Eedlfj3LDrFEMC1BAfJCsb9HaRnXrvSxlh6h5HW8GzfZhFPGdKbBdU8pDKPRGwE8MFnV18CujeqOIsajZsrTjWhC8yThs62Igm0ssNVXlsFKEEK/jRMRXhpPKO+80wqR1l1ZFEVG3vGIdm0YPCtsfdDhl7PNwcMSHXHLSLleWtNa44x1ljXTF0R1lVfJ/pIS0wYOdV+IEDbTefG347PooIDDHTwiitOqUXCph7Gv5wtt3a7Ct1vyW0xzbRTOH1msTdDZLRsElnNwOCPXFAZ6dhbmpSbmoD3uFtzCohRVV+O/e1jTrKBI/zyHOhrbSzs9jmMVWxb7CFBy32WmOnBSum0zAMgWKWDTfgOsZojP22PtNr3FGHMddE+VabeI815j1l+TfDH+Ii0EeuwC/a+UsLgmIe5Slhel5pZQctSYRzWe1ttSFbU3jLcEojkGQ3WGy17mbIl57Ww63DERfxouapN3D4dDH1lO0kXuC8MVV66L81CzJl9cAKUKmT2DI/OzVWYGcuBzUXYrqQ4g2iQAOohqEycjs1VEZuC6hzr9XOTp23LOAPgRU4ImiVgzuCo8zLWupHVrOlwgZ7qC2KNljr8ehRQq+bcnpJIp7q2Sg4V4/bO0yONWn29pckEjZr0/fYtFh1h5VY5+hM1ToKKUoQDm0QdA1qHbbc+sSCVZDkl5ywho1tH1HiT0xH3hqcW0y6UbSDRDZ7ygj2E8hGFkgeUdAPbqna+PFbIbZNcT1EQTlthOkFad8lcapIupfNHwhAgAzraiBdbDwb/ZULAWcxjbAi0npcwiueF2kJFFc4qeNqbgMgUM+WohL9SgR/Avvxf0fY2hP4Ch2jlGAmbXyMyRYhpAKiLePueHztDE0s1iAx3ZJoA0UinCStp0zjeQki80R5scGOB3okc3MWywVaYZrkgrQspw9rKLkyis9cax5ar79qkOw4cTgYTO5rC15BBMHWbWDuxTLhwzEMD+Ykv312NCfds/nE7tyIP3+9DVzlecs+rlKm9L4J7dPqbNDw7Vqrc3LAraxC4Mh3Tdelj7ySxWHR0c3fPv5J/tfzo8a2rt7eZdKWmHzr5nymi0DxMM+VDfF+oohUTyCPylj+tNXdynKncZg3/vTj+vR2+eXz6uSvP/zb64vol+XJ+nY4e7nBIu5kX6RKgKJhFMfDGYKQ2n3T3Wmpw9vGsXm1MjChdalqfh0X7ukyyEAaI0GkmplYxowL/Q7RbLGiiSLiqMalbAn9Vf1t+4SvJDno3ZoDfBe+ZPfiG6wQj6JcQMgpZpxtU57LhXEfW8SEURLPav5SC63GwONaKfNzLTBT+nfEGTP5gILP3GcKp5lWRxbWAWmGRM4W2CNkf5sP2huvyn98M5ru62/Hv4HlRXkuU/WOR4+ab8yYwejzm4tL9Pr8zH382B8lxXcmB0RE6E2poZXF9NadkeTxDGRYsgAf2EfGJhdpNV3/plLm1vzqWLW3XUln53azxuDeIejZjWtpqpqN1g746R+ezZ++/P386fzFszDkmi5dZoShLKIZrhvlm0CLkuiR3sDqzx+bKWMmQG1atGNdFBNrfOPWAqHbsPp6mPnEINXjiHwjUd7ZmFGSS0XEq5Qzqrj4PsW0UZ1+qLmgvThh9BMWg1qFvnw+awX1/eJbhqPr7yWJckHV9vuF19zDzdulYgVja/AC6cbiiFY8SQgWF5HgSWLTZoxvQ8t2seTxtherLlQq33bxpCtEmN5sdSDVH4axVU5cStcuk54vFC+0s+gtdr3N6JURNvQfT4pcZVUH7BBLn222wTI8inbYbFtLvk2ZFyHFNTBgMXZne3fbNV8D/vHExRTqlSII1Ot+m1hkIUnUCm2VcLzjPumkhqRgCCZDYVK2GOPNn/ANRjdUqBwnfvhjGLiMRL5cyG265MlC6TkBKYHuqh7oHEO2FppCZLbNC4SihGBI8ZBnyGBBgCVgPasBB4fWewA+ADdA6cV9S/D1QpCVXFijKOC/Q+SXGrPMwAWp4AgwjGsyYRGRXqW6/B8FThKSLASREWb3hdpr7xSLa0iTRm+IDRoCY2xCEM6yxAtWkIpnWdNo5h/3YykXOUu4Ta55DzUx3GC8MDgAARADWz/Kcj9bVxNjaFEeiPHcHs6fnH8xY9yOFyJWXKQmxa1bgAIQ25dsVHf/Djcy6m3ogRXR/2qV4LmSNDabEROIGqqAt7Bs5QOgpKwOEnWiFAQn9wHzEs40bLa4OmjFId9eQpTLYlBIKdi2QM5oOMejjMpN2KT/8026EDlrmYLtFRniBaKhApI//fWDRZNn3mybISwRNuT1KDcqd9fhnnEskQs461noVaZt8dgZ+Y9YLPG60pqWqz1h0lxtN4QWjWIg6yUQpIvDPHUTawiK82vdxQaUxdmJy0uHVYWwk+vNjyfgZGNE77qF5YbgyU6N3hGcIZw4yzgYrW2/0F9H67L6m8X1snVRp0yRdSBUZZjoAVi68sBHD/xrmnCIkWoXNFoy3RmkLxLccnDWAcb3nViTcAjdDh33KYmdyx04ukdRnmEWbX/7PQidx1fg+uHV4DfQna1t2t+7W56z9ZT9+3dN8F+8h7f1OvwG+rijXcPoSmcccVNhWjXPXJjoTHdzQ/OAoz4Gmv1UHpumGWd1990qu/eQ396Wq1p2SqsPn5N5NE/nH4jCp1jhE0hUDAdENvFz9cs2wRW03NQRGdEVItgc/V12Ghg0XXPlyHThjyft5q6wqSs0C8OzpVizWXODUsVS59SFosNzq9AmbpuObpMzLLtzwW+I2BAcd/Rr2+AK9XSFUTFxEn5bdZytzRzz3vnFgYb7pn4A3eT/9dnx098/OX755NkfLp8evzp++erpi9kfnj//6evZx7ef0E9fzUmpITG3IOa/5ERsf0JfbxZ//dPm57/+hL6mRAkawXnsy/nz+fETTXd+/HL+7OVPX49/ApXw64v5D6n8aQY/FpAFWn59Ab+14ryhSn59+ocXz3/Qj7YZkV9/mpmUc/AHQIBjpq9/+fLm898Xl+/efFy8fXN58q6gAael8utTXR5u+vn63/84ArT/OHr13/84SrGKNgucJObnknOp/nH06un8+J///OdPs33WG3DrFt2LzdpmVmgbDcHGXhFV7b3+JUY3cAcSUNKpKvR0a6OH/Ro0Vhu+58fHqQxBqUUcFDh0L3YB0e/HTI32KsM46WB1obCiMBvG8GuplzcWu1gapw5dqo1nfSCPrDMM8QV0WReOhN929+uISTKileDykUXlxq0QvDe6mK2L73A3QT95C03fdIC54JLT271qC4IXz0ZORre6dWEw2zKqJmVqlsNetrrvKYmNr0kbgGfjAAieK1qT0FXen02Jtm6Wx0/f/dezv/zx+g8/375YqzV+q9i46UE7BPJZPMmq07MCXHZM/ZhHXbxcvkucCf5t63mV2Sct/mT2bcOTzFgOC9tHQRXt70RmTxDqHpMVGrXMt5DMqUmo3SHqvIgUa0ho0JYatExM3aI9ZWBRwLlSt8blWcduRhUtAvMuT849nxwtQ22TzluhZFy0ZjPzijg4moPxgOoFUwKZD2ydlYC7MeLOPqsUKqI3vI50BdAjLlBCpdLbwccWYuGFA/npy8tVangb0JY4uu5D5pcJAbPvg7husUSS2LSwiqMUMy/hrtehZbKgAErzohOkVySEUavmLhmR4p43j4fCYC0uRECV0MiULOywEeSXNhC1Yg6IsXu4k0tf5Fkr/i2mYPxecYEwWuVJ4hIXGfeOIvjODstHjCvjqAy5Dni8fYzwShHhRSkst4pUHLSGjlaoxC85yVvbuiwxuob2KpAbLCjPJQIichQyNxptx3VirJXdqT+aQ5VIhZcJld4logwndnTNEGVRksMRo9C7tJHVs+PYJczqrF6t7M7VK+eF1PO4OgIN7RmqDLkYKzyqWs5ZorM+hUcFLa9IrLjcCZJiyvQqFyl6A9WzgmDmlureJnAHae7apNqMqToqVT0cwYbpFhNXumj9cvkY1i629zpnml+mWrcyxslCdSFRt0T412vZvCjg/2uT8XtdDoSHAnazqRNxpdCkkC3l30m0TvjS6NEjwNM+KUc7RJwRa/YGFFBaamK3V9BCboFFM41ABUOlkMNB/AtPXDj7covevT4H3bN+B0uzRSrbtQayekDXaO+20UFcpTIYyHYzZeBWPWiry4TbGay1Z2KCAQFaA8OP9gDSHXLUE27UHWo0KN1Ff4hRf0zYnv3QmqGmLzRuT74tGWmGhVPtwbsRQtUUzkSklFnHSVV1Wa4K6HpBtzxpqUqtiwQposO1MIR7UUj1OWFxcTkXqs4+u3bKNgTFMuOUR5gqsH30vvZUg5aw56GW5qp3+01ntIVjGhZ3EWcQvMJUBSmvQGy2lW5Ds9zXjx1Dgg4FNpxdkIvt5E6gi68nRG0FfBdot9Uch3mDWZyUmfsdkQmhN85ZG8itwjUOuFQ0Sdzg4hXNbULwdpvShd4WKVQjH7jbGbky5FtGBCUsci0O97ZbkIBabK2ztNuX1fb/rfDNg+r/PDObUq0ay3lCtI6E49h/PnRVQCGLa3OHVuNp/e0FSbC1fqh6GueARXLcwXPzHtvw6CuLuT4EHOY5CIfy1koL1Ghpvb2BUHgybxWRC1E/pq3P56JU28ww2aQhqCylStXhVS8mphIRU2pX2BHOVC5IvIg4v6YjkwjVPgbdlNlLUHGCjjSL/wvpJY4QAV3M5rMwuaqw8iu2wfaSTkfMZYq0l/UOrMeG4JiIkbkiiq+LXOaWDFQJdkNwFYt96IrXMaI4J667jEYfFWrwkf2oLGyoHcGYJKl1O/dlTbhPw5ub7pCq4dO0+e0us/Q3MKqs2Nj419NiN7YguQ1V0lX4vgaXiSIbObaMYWbY0DJlJx5Z3tjCt0UtEtrwu6mK0FrZUn+27tsVi68u4ZsSTe5EteHxrHLRt38dgcst318ZrzrlH/YIjEaUrbF3AnYGD1oOwMzL7kwKBUW0/+lXTJb5Xuns2i4msRUB+qNSZ65wBHddTrdruzABepDkCKvSYw4u2TIwrRWoN6OmS8I5HbjQna1H0GpHM3TEuKIR0X/5ngUzdHSLBaNsfYQCObSPIkHhYv+jh869WXDEdI8Y0t5Bpskfxtj/8jEGsTD5NIbX8DCzHA4j7X/ZSHOCnEpfip9dDM9te3Z2UTiFw9AJinXafjNhC2o/l2yDB7r3C8k0hB2uILOHclNeQXZZauh915AdbvqqsAVHShuEfjf8gYPVrSFxBWYtV1jVfI7QnmlPHQDwS+qKiPxN30x3Bxf2XZZWkr7Z8mC3ij30LXASUjNglQ++AG4oc5kvPbNhmPstZc+fTc//b+b2ZNTL3220wa2kkdp2ikkZ8i1p6QmqyB3MTk3WzE7MYkSZVLgvLXLYjW4CLP75thVjYOx3TnZOzlv71C2W5c0LLeHCD3iBY9BIt+9yBYam0r3U2N2kycdXuJF0i5eNvZl+WmRwHb2xkAGfbgz/opdLAuzCEfK3BN2sJu3ID7dT7hOQHLqdMj/cTqkOt1MebqfshXW4ndJDdLid8nA75aCETYfbKQ+XLQz3550I2P+I6wTutt8Ot1NW/9397ZRtJvfx11M+tA0RuE9s3bXMe427D3vaYLlPXHfLvLfuD2kFOpyzVNg+tD1bECw5W2Qb0ZYRe19rvqaPDP3Wo6b8Liy5cAzp5c7NOE86okUOuuBBFzzoggddcEIsbVdtXePVte8K+mf9u8WNBN6V1zqHPEYcObS/H+ielxobsAlfg6PtYD1U0ZRIhdORi6xLhAyflskcHPuWQMzAlexlOpu/vf78sZ45b5irkCH80F5wqLIshlJH7hnr6rzMvNAQe1Wwbv8WIAlu3Ay0a+XhDgggOAoC3JI8lXBH6BIuXaasY7wNkKaBZkHTLDy1VjJ3RHe1E+odrWgSK94HG26f4TInEKBrh7PKk/p8nQYL3BKbJ4lrnnpvusWaLjHzV2vzoGW5Ni+7HfcLiuhfdsGeNNH5n02b9Sc7r0dS78n3xIaGmrhrvrJAWvet9du8DWtzWUftlXm4CCZDS/haKiz9Ozbdo5ZB5V53DyuPLpp8YFmg7z2g1WYYMeh8d1U95RzRUbaraWVqyxG+nhghRl3KxJ671kKVcMuj5T9zUYvC7Oohou49X7/42RRvc2t1I2ZCiIYm4sKKmNviSsTaTZhdF01M1HFn3s4aL3luNBORM2bCuyAGtgSoW7cHXsLXC6jH8Nneg/GamDzt5swK3ODXJmlXgT0QGVgseo2UyaMnXJPEYWYdZta9z6z2WTUe3Wd8i+I8zYoDasM6CTAp3EjAMjaxobGSEhQYdPFWzdtl9xkx9rbKkvcrdMayXMkZegt3DcsZ+pQr/USPqRMek6jt6hrOrxeUhdIM726IfgMZuSGHDdxXZOOonIlyiJevw8Uwa7iv3BksYNaFynZnhgVu8YIeP6IvzC17RkhUehVFnK3oupnqrwXQIiik9pNfT/6jiqwCyQQy2DQwdX+LQX9Y1TjlbM3jpacZ2yfDY6w+6A9O/9gfZ1XyQmNirarqq8etN9hqTyEeOPhtQxBC0RPu1zc47TelAA0J78KOdlZ53LbEdRuqehC9zRnkx8EJirAiay7or/YSlx5wJ58+fHj98XQkRNaY0QMUH/JN9cKhjCrMYpMKcRSoENkhSobLg9hlvvJWMTc3t/KXxJuZH7YXf3k/fF5qVvBJdWbKDRdqYVaTV0iJvG1369ijXQMjWwCgjhk7vatGFch4j437tJQbFW9BwwrleLH7Grz0Tc1/mP/b/JlVvF2GIqNR0niO3nJhy1lXAokyQTmk0vW+bHCAloO5Wjqn2yxwtOXYv+c4wAYkd1S0e6vx0OcBE24ie8ay5jBqKAciAQZU1DADR1DIwRPBNVYmyB0iSttjfMYzgxgeqGe5z+lg7Xqhzdu04V4wxImhzOc/HRAT3KsXhPnU95KWCaJLNFqHn+11N2nCo+s7wYtTntvwsSrmW0x1k7q9gQagV58lKd0q5ppCg6rRkqncq76C30oIB5to6a1GTGnqZbYqq7Z3TB5AoxdFyshUwiCASEaYDQPUJgX3AZMz+s2TkQpfE1aucVcXby7Lt1dd4Jr3GA3z3SuuN2pZPKZseS+95NlpMcgtd6vvsTVl3zx976P+PU7fg0921Pcce7SPvhcAgO49HUYJZIekGIVf2EJvEIJDAAuBRw6418x8ZZLGaw6eoCFyjs6UlzZuSSKcS7hrzZwhp+bCBpNGjczQkkgaE+mlWWxwLMnPKqxMX7msdAm9Jujq/z15y8UtFjGJ9V9Xc3RBCMKJNHnproo2uQo5y92hc/NJw7HZHCLDNQdZvkxo1BDYVcTQi1em8efobIUYLz9s8CtbCQuXj09ZrTmg61ocgt5g1dQcQkCaHAFYq772m82GcfAqrrB9SAfvh/Zo/hcNpX+wjCqHSPipI+G/HCLhD5Hwh0j4QyT8IRL+EAkfhnSIhD9EPzULHKKfDtFP/4Ojn+oo7iQSvrS2jT9dndjp8I0BAB4Tj8h8PTeQZsilMn7c4l00ma33vDj9JEzRFSUCPTo/O23hqya0MduzXMe2LULJmaGnO2U+KU3bfeynP4at3CnpDOlcuiMBZ0r/ZJ60GNOtEZt8y7hQ5XnIlaVz1R0MWHJD+wcBCCLzRO03RcFavArXydBHKVFCi3A1dKJOb4b0ha49tdxgVabTNEZXcC5tMaNEAaG3B6i3XCDKIgEXq+hNNFZ4hlIsrsEtWGtRxjG4SP2J47hxPIdMGsyU35AYrPoRZmhJ4P5XvkJH8M3RDB3ZMkcz/cGRZDiTG65acq1vuFSLcnZN2xPeWuXWcziHr2Q+taPcqsBUOr/kpsj7qFXPJNkWhJqSsbAOMfoNTpknWoq+VI8U7eiCMeQfhyNJWWS9vDMebeboi7RHzxFPs1y547Sr//ROICOe5GlbplWcEBZjEaxMvnPvWA9VQawiXrjbGU01Sdxd3jQlcOZt1H47322XFeeLGZdqLUjVqezcPBztWVZ+t+NxYwUN2t0htArkrn1C6+edbc3g/v1mXMtoSn7l3Rc7tbP61a5eBdv78V/z1anw+tG06JauZDhOKRvlSOZCCxpkC2MuVnjZTNtS8ky3xnN6NMsg5WEuc29fX75+P7XDXOCae9Tp+lPieX48Px4F59Q5tfMVwmMdPUq+F2/evzm5RP8Hvf386QP0ofz3UTj+Yu9HsHerPZQnoV2tBYkr95581r9b1mh41x2r6sihB4+ANmCL1XLgYjndFu3Sc1I9O3XS1KAKXdxaOmVNHXymKVb5u+z3c3RSURuvUiwVEVczdCUTfEP0H9GGJvEVeqQl8+fTt9+//vQW3ep9LlsjePd4FtJNr7QiQRlJrob7504VB9ioFoRm6srcELHkEuplLiu6Ar34yl5Q1IL1TiZjg+qELr0XzmcX/EvMRcM3WvXUUtwMgRuKEUaMqFsurr0N+1CtIkrHeGUMcl1LU8xiRCCIq+2g1wmM+WT3ZLyDpmJrRBU4tCLFHQZ332RqboxAKY1Ed/zYpKtHuWp0CKtrMuH1XprrNdlWt2SuAfRWtLtzsJgyewS48Yp1roWkNFeuhkFFOEk0JCvRzPGNJ9Iu4MHwfYchsON+o+CO9vFvDEFAXQ6OudpMud94T1n+DaiW4Vf3Hs4CV+/iuESl8XSnRmq58mNgSADYinbgmgm+FjjdXT/YmfGk6815ueA4YGArky4vVD+g6SXloKC2/UJPwJxTRl2UBkHjYCWR4oGoV5+vlHXnjZ2PWO1MlOYWyEhLo4uLd7relBlUctj5Zldw/oAtsW6YGuO6WnX0OopIpoyd8S2mSWFmPGM3OKHx0dwrE+CREswkwkjm4D+9yhPDbl5SsGWKa7mhm6x/mAtVLo6bAyzsWX6Br06vrCJWiqSZghu/V1C43s6dPqkjmrTm/2rdTOuNm2EptdA8ghY1vsTXZHvUhqpxyu8GYeDFIKhltudagFK1vbQETnHzkLbQ2ATPMhI3/bUnxqdbtlRjbRdr9ZdnhJk7v9KUxBQrkmwdqjbQgfzNnR4xYwBDFue9mlTSNcMqF80BPwhH8Xlh4rXAjL/6Ndm2MQ45k3StdQMAjXYpubJTWs+ieUuogPk3tW9J2Luk3b9khIdJ/7n8QB+iEX4mw3wX7g4ZVY1xhga7dtwZLMO2s7X6/XImQ9fvnTPIP2eIh86I9hrqpTPGL2WyJmv1TvHxyDzmd6ixGT2tiN91B/2a65Xbuo7U4mouNeZfYZUGtejjp0s4fcxjTkTTEXaQbKg4OmhqEZZGRGmyxba7W0FSjSvHB3K/vPy7JxQrHGmb8cET2rc7KmWRzRcZU0EixcV2DxBB7/+inwTnO+riCos1UXabwj1LSB2gvKUq2gSOzL2sLGlIvA1rqpqVDuyIGkLPDknjxnF4t3qnc84y3nHaBaXPoIYqw9+WhLK1ceJoHTSNffxgbbOL/dlpqyI3OUPoxA6Om1AcwAC6+ju04knsuY0wYhylW/XjDQmkFh7ALCYrnCfKEOhgFxzi0AIPMsYd53sf5L7ipFsJgNzBmGsFUFqsAuw9k+xdpUgxpD1z7QNbSC2ee7eRDuF7R1bSQawbQ28Kc+gQzvdoELXHH0pgsqLX3vnHpXkyzvHKftSfbq/kh/Y58QjyQw+S08FB2SerQ7DDJ8pN0KpgHaL4D1H8hyj+ELpDFD86RPEfovjZIYr/EMU/GNYhiv8QxX+I4h+H5xDFf4jir8E6RPEfovg9HL/5KP4qEtjPLmAUT7hb9DLEGg4yyH4lOFOExe2Gjd1saP4cdjxg0QlvWXF0rUG0WQt6MITtKqK4rciSt2eOzoJAwR5lkmV+9/8DAAD//3LvWus=" + return "eJzsfe1z3LiR9/f9K1D6Evup8az8sr7EV8/VOZK9VuIXxZKTJ+dsjTAkZgYrEuACoOTZq/zvT6EBkCAJvs1Q0uZu/GFXQ4LoH4AGutHobjxB12T7CiV8/R1CiqqEvELv+RqtaEJQxJkiTH2HUExkJGimKGev0H98hxBCJ5wpTJnU35riCWVEzr9DaEVJEstXUOwJYjglr5DkuYgIPEJIbTPySlO+5SK2zwT5JaeCxK+QErkrGKCr/11uiCG5EjxFtxsabZDaGAToFkskCI7n6HJDpQEDTQG0uhheSp7kiqAMqw1SHB7q+uYFhbdcIPINp5nukKvvb7D4PuHr7+VWKpLOE76+mn9XaR9frSRRlfYlnK0bjVvhRA5tnakT0AmScaFIbJooFRZKIqxqIFIiJV5Xe1mRbw4WXTMuyAIv+Q15hY537HjLFYivyj7X/W0GAx5Zjqihk0oQnA5igQG9pLnU1IhuN4QBBMrWbqSJ0DDkDEWYoSVBv5Mq5rn6HeIC/iZC/K4KLxNcZiRSXMw1uO7eyQSJsNKPX86f9/cZZVmuoM11liU3ui81z64JI0LXWWFcKhHwgGHSG5zkBGmYdEVJXNBYcQHvrzSJK8QBBKIMHhrikkTw0A7bW5qQJcFK99eK2vFCj07fnH9+c/L68s3pKyQJQVfwMXTI1eNqf5VvdmSkf5FOqbZas9lC0ZRIhdOsu5FnDEVYEktvTaRCGc0IzJgMC0nMclTUVp1Bdp7JGaIKScUFkUXNugwXdE0ZTtDVfxY1XKFHQvOmJEzpyeCqN1PE1VxZJh+bHqFl5dDHtWbrnpBEzVMe58mAsS160nyA1AarcjCBnhnlFjr61wgq9rPBZORWJnw9X+GIJlRtp1u2bYWIfFMCRxpDMaaZoFxQtQ1DcW8ng+IqdLxt6HT1hiQ3RH+xSPCSJFOt0xrLJk+xWaHxMiHIEeoelDuH4QjNG3IgIlLOM8HXYjp5pQFoAm48bPVtxGk8HSfQ2CMK1VeJGp5wozIZXVehIx5kPSJuaET8+R7q6RYqF+ZrqKtWseakhNx0MlC7ZrHWiyd8Hqh2leC17Gt+WPOET7v6Q9fvFvbxyHWv58wOMolLEaFnvlUOrSC1alGui2lBqcUGfAYt52vpdGHKYvLNjXmAbSJB9FpbARtj1cMflW+rfaQ/RpwF1QH7wbwQrXxVVFmsbtKs+1R2SD203BbSw9TG0wwLKjkrKizFqq7Lmz5asoRltqYxR2crtORqg7AgiMZaFEfFaCLEWbL165YbniexG4waP2yUyuaCyIwzSeZSYZXLRcRj0jZLW/r73eXlOXL1IK8eN8zFZufF8YsuCCTBmSRGBRqJ4Y351CgkS6JuCajtv+RaMcIsLvFRhlKaJFTrZ5zF9fWqisjqSYuEsLXajMR0Yjcz5mM3M6u9teRxXUZYBAB9nhK14fH42frZNt18P//uO7sZ1zxZ7sb/aH517cAjnqacIasJ6b03wjeYJiDlKEM4Sewc0ugqW/RKq2AyDNO8fEmmESJJWOw0Tm+1kTAbilLwmadqap3NKuRG5c4FBoVcK3Qz/ZwZnc7o+FSaOaLrpEr/ZFz5lcEnaMOlspRs+UuO3E66wDHT78wGQf+8KidoZaPQxDVvdpqjOEAHcdhgIVK5YAQWI9DoM6236l40dobqKgjAvb4TOWOUrQNo9AT7lbMBaFzJu0RzQ4SkxbLaAcYWdGwF7DxUoT8qF9SjNlEU3J+uuEixqpQrlsLX+TqXCj17qTbo2fHTlzP09Nmr5z+8+uH5/PnzZ8N616zxhSAy01BPEEEiLuLaJrfaKNWrZ7wWS6oEFlsoa3rLSnbN7xkRZqD06qp/KIGZxLDnLfeS26yuPJnVodKPfPkzidxcMz8WI9a6Yq3KJRHlnAI1HIjVVVIhuKgAWAue9+y33+iP3ApodQrNvziOqS6LE0TZiuuZbZUHQ6fQdXzDJWq1q6GQba0DVgnNqWANAp5ERyHpNah2X5x7TFQaaFCLfBpUu2ETK6KihOdxKaNO9E+tHd3QmOhmKhxjhcNi64N9azSnqPKp1GNVLkE4jhdQYOGqdCoYF61STBedw1dzV219YpOoZ/Z+9MRbFeEcnXMpqWZckEkStDwSPZuhdURmiAsU0zVVOOERwaxhoC2wUSYVZhFZ0J6pc2YLorNTB0kLEZTiaKPVzX4K/ZKpoOHL9WFUbIGFx2dFP6tn85TENE+7qX8wVRhT4CjiVs0x9gJP5BUIcvmEYKmePI16FlKvIgQSkZbSjkoDh8pSzHWwHKyNxagWUOybJ9+Gs579RGP5kfN1QsxMa6cuyLpX1H6GMn3tsxM95tE1zB8700/d70Dl5h1sLvTymySkNICZd3rOyg0XamEkQGlKwCzacOHoPSlmectpUgELBeVD2zpenA3MabzfmviF0V9y4h020Di0qhfk0pD4GEXR5wuozmmnFoBWJJY5TRTirAuKtxjsiOSkoGnsLu20wIInG9QqugTq1id6sJxBTxg6BdNqZi5Z9p35FajkTCsDHqPa84Lq0lPypn7ey5mW9ji+3H9M3tltRXM0JuJ0s0AEmByLaEMViVQuJmhDpTr0iMzXc/Tt9y8XL1/MEBbpDGVZNEMpzeTjJhQu51mClVbp90Py6QK5iiyGiDDF5Qzly5ypfIZuKYv5bQuI6o5ndwy2niCNFU5pst2bhKnGNlKQeIPVDMVkSTGboZUgZCnjntZeE8FIsh+Sy8B+83cSmarb+4FmDbKVRx0U31MJR9hn509wHAsiJZFNAimO9muYI7PBIr7FgpTEZiiXOU6SLfrw+sTH4Fax63ypm6/g2M+uZX/2nwXIlu8LJbyqUZeVIn8l6xbK5Ue9y18FNBq1CGY8nkA4eT2Q8disrEFS+b4Lo0fpnMfoy9lpk5D+r8xwNF2jyhqbxPT+b9Ie1DW2dOFQ0T6MkKkNpThrUsKMcQXWt8nIeVWGaU6pLnl0o4rm1EV2AoUxSNfUa1cYnOFoQ56Vy8vRa/PkKLy62Lfog3MCqC4b1qoWWhZKSmiMQccRdCYi87RtAcGRXpoanebT6emywkplbUhOI3Q43l1enp9aOpVDshAsVPEaSbkii4pw6hrWHpyANaGEKXR2jqzsmAcp55KIRY2J96QMp4zSMrE5QQT75hJLGiGcq4058jJWdGuCD4KrnJwMQVZspn98czketDtrguMdd+oS7DSRTNtdFcpfPr8Pk90olS2ayuME9IFuQ41CFQ41p12LmikStZkjx1AujtKqJkqf/pLH24UkTM2XW0XkUATOfB/6aAA6lqdLIrSCBhUUvjJE3BBRPwEMd9uKCFGYIqp49xsuV3WYMF4bz9wm1ZpRegDJE//4PWdPwDctNnMc6CCpBGXrOfrEki2y/mWIms7SxRpVms/eJFgqGkmid3UoS/I1ZfbUzjuh5AIetC8TsIa1N7i+wI9tsW3ul7K5xgNuqtaWLcUsDjQzLDr8DojJDY3qsxL18NmAbkAhv6TNVtIIJ5ZoHaq/N/qZN7uiY66OAAR11w8ES37sAEXZ3YHSde8CKsMq2tzd6EH1u+AKqAVDYBVC+GQjeLCGHdhuCF5eX+GHoN0BS90S0YVoce/TYBy6+54Po9DtyIB3OqQC37ZCCojWgXgQ+oxvfYlqHFqWZMWFkUAa23Jr/cSf6JJPTEkjSMKScU14y55iH6H4I+Fn53BUrpUrPbhrrDZEkFjr+CRGnNmgFLurcS7j9RpDAtRUPkhWNurbRXbqvS9lhKl75LaCZjubRTxnSmwXVPKQyj0RsBNDBZ1dfAro3qjieGo2bK041oQvMk4bOtiILtLLDVV5bBShBCv40TEV4aTyjsfNEKkdZdWRRFRt7xiHJtGDwvbH3bKMPR5uckzINQftYmV5a40rznHWWFdMvaOsKr5/95AeGLDzKnzWoW43nxt+Oz6KCAwx08IorTqlFwqYexr+cLbf2uwrdb8ltMc20Uzh9ZrE3R2S0bBJZzcDgj1xQGenYWpqUmpqA97hbcQq4U9VejuPtY2QygSP88hzoa30s7PY5jFVsW+whQct9lpjpwUrptMwTAXFLBtuwHWE0Rj7bX2m16ijDmOuiUiudvEea8x7yvJvhj7EWKCPXIFftPOXFgTFPMpTwvS80soOWpII57I62mpDtqbwluGURiDJbrDYat3NVF96Wg+3DkdcxIuap95A9uki6inbSbzAeWOq9NT/1izIlNUDK0ClTmJL/Oy0jCMp9nIQf4YUb1QKdUCtYaiM3E4NlZHbAurc67Wz00ocTAiswBFBqxzcEVzNvGylfmQ1WypssIfaomiDtR6PHiX0uimnlyTiqZ6NgnP1uH3A5FiTZu94SSJhszb9iE2LVQ9YiXWOzlRtoJCiBOHQBkG3oDZgy61fWbAJkvySE9awse0jSvyJ6aq3BucWk24U7SCRzZ4ygv0EspEFkkcU9INbqjZ+rFmIbFNcD1FQThshhcG677Jyqki6l80fKoAAGdbVQbrYeDL6KxeuzmIaYUWk9biEVzwvUigornBSx9XcBkDcni1FJfqVCP4E9uP/jrC1J/AVOkYpwUza+BiT2UJIBZW28N3x+NaZOrFYg8R0S6INFIlwkrSeMo2nJYjME+XFMTsa6JHMzVksF2iFaZIL0rKcPqyh5MooPnOteWi9/qpRZceJw8Fgcl9b8AoiCAxvA3MvlgkfjiF4MCf5/bOjOemezSd250b8+ett4CrPW/ZxlTKl901on1Yng4Zv11qdkwNuZZUKjnzXdF36yCtZHBYd3fzt45/kfz0/amzr6v1dJpiJybduyme6CBQP01zZEO8nikj1BHK+jKVPW92tLHUah2njTz+uT2+XXz6vTv76w7+9voh+WZ6sb4eTlxss4k7yRVoHKBpGcTycIAip3TfdnZY6vG0cm1cbAxNal6rmAnLhni6VAaRcEkSqmYllzLjQ7xDNFiuaKCKOalTKntBf1d+2T/hKkoPerTnAd+FLdi++wQrxKMoFhJxixtk25blcGPexRUwYJfGs5i+10GoMPK6VMj/XAjOlf0ecMZO7KPjMfaZwmml1ZGEdkGZI5GyBvYrsb/NBe+dV6Y/vRjN8/f34N7C8KM9lqj7w6FHzjeEZjD6/ubhEr8/P3MePfS4pvjM5ICJCb0oNrSymt+6MJI9nIMOSBfjAPjI2uUir6fo3lTK35ldHqr3vynp27jdrDO5lQc9uXEup1ey0dsBP//Bs/vTl7+dP5y+ehSHXdOkyew1lEc1w3SjfBFqURI/0BlZ//thMGTMBatOiHeuimFjjO7cWCN2G1dfDzCcGqeYj8o1EeWdnRkkuFRGvUs6o4uL7FNNGc/qh5oL24gTuJywGtQp9+XzWCur7xbcMR9ffSxLlgqrt9wuvu4ebt0vFCnhr8ALpeHFEL54kBIuLSPAksWkzxvehJbtY8njbi1UXKpVvu3jSFSJMb7Y6kOoPw9gqJy6la5dJJRiKF9pZ9Ba73mb0yggb+o8nRV61qgN2iKRPNttgGeaiHTbb1pJv0/tFSHENDEiM3dne3XbN14B/PHExhXqlCAL1ht8mFllIErVCWyUc77hPOqkhKQiCyVCYlC3GePMnfIPRDRUqx4kf/hgGLiORLxdymy55slB6TkBKoLtqBzrHkK2FphCZbfMCoSghGFI85BkyWBBgCVjPasDBofUegA/ADVB6cd8SfL0QZCUXZaKuO0V+qTHLDFyQytRgGoZxTSYsItJrVJf/o8BJQpKFIDLC7L5Qe/2dYnENic3oDbFBQ2CMTQjCWZZ4wQpS8SxrGs38434s5SJnCbeJQO+hJYYa8AuDAxAAMbD3oyz3s3U1MYYW5YEYz+3h/Mn5F8Pjll+IWHGRmnS8bgEKQGxfslHd/Tvcyai3owc2RP+rNYLnStLYbEZMIGqoAd7CspUPgJKyOkjUiVIQnNwHzEs407DZ4uqgFYd8ewlRLotBIaVg2wL5reEcjzIqN2GT/s836ULkrGUKtjdkiBeIhgpI/vTXDxZNnnmzbYawRNhUr7ncqNxdh3vGsUQu4KxnoVeZtsVjZ+Q/YrHE60pvWqr2hElTtcMQWjQKRtZLIEgXh3nqLtYQFOfXeogNKIuzE5eXDqsKYSfXmx9PwMnGiN51C8kNwZOdGr0jOEM4cZZxMFrbcaG/jtZl9TeL62Xrok6ZIutAqMow0QOwdOOBjmb8a5pwiJFqFzRaMt0ZpC8S3HJw1gHG951Yk3AI3Q4D9ymJncsdOLpHUZ5hFm1/+yMIg8dX4PrhteA3MJytfdo/ulues/WU4/t3XeG/+Ahv6234DYxxR7+G0ZXOOOKmQrRqnrkw0ZnulonmAUedB5rjVB6bphlndffdKrn3kIvflqtadkqrD5+TeTRP5x+IwqdY4RNIVAwHRDZJdfXLNsEVtNzUERnRFaqwyf1ddhpgmq65cmSG8MeTdnNX2NQVmoXh2VKs2ay5QaliqVPqQtHhuVVoE7dNR7fJCZbDueA3RGwIjjvGtY25QiNdIVRMnITfVh1nazPHvHd+caDhvqkfQDfpf312/PT3T45fPnn2h8unx6+OX756+mL2h+fPf/p69vHtJ/TTV3NSaqqYWxDzX3Iitj+hrzeLv/5p8/Nff0JfU6IEjeA89uX8+fz4ia53fvxy/uzlT1+PfwKV8OuL+Q+p/GkGPxaQBVp+fQG/teK8oUp+ffqHF89/0I+2GZFff5qZlHPwB0CAY6avf/ny5vPfF5fv3nxcvH1zefKuqANOS+XXp7o83Er09b//cQRo/3H06r//cZRiFW0WOEnMzyXnUv3j6NXT+fE///nPn2b7rDfg1i26F5u1zazQxg3Bzl4RVR29/iVGd3AHElDSqSr0dGujh/0adFYbvufHx6kMQalFHBQ49Ch2AdHvx0yN9iYDn3SQulBYUZgNY+i1tMvjxS6SxqlDl2qjWWfkkW0GFl/AkHXhSPht97iOmCQjegkuSllUbgcLwXuji9m2+A53E4yTt9D0TQeYCy45vd2rtiB48WzkZHSrWxcGsy2jalKiZjnsJavHnpLY+Jq0AXg2DoDguaI1CV2l/dmUaBtmefz03X89+8sfr//w8+2LtVrjt4qNmx60QyCfxZOsOj0rwGXH1I951EXL5bvEmeDftp5XmX3S4k9m3zY8yYzlsLB9FLWi/Z3I7AlC3WOyUkct8y0kc2pW1O4QdV5EijUkNGhLjbpMTN2iPWVgUcC5UrfG5VnHbkYVLQLzLk/OPZ8cLUNtl85boWRctGYz84o4OJqC8YDqBVMCmQ/snZWAuzHizjGrFCqiN7yBdAXQIy5QQqXS28HHFmLhhQP56cvLVWp4G9CWOLruQ+aXCQGz74O4brFEkti0sIqjFDMv4a43oGWyoABK86ITpFckhFGr5i4ZkeKeN4+HwmAtLkRAldDIlCws2wjySxuIWjEHxNg93MmlL/KsFf8WUzB+r7hAGK3yJHGJi4x7RxF8Z9nyEePKOCpDrgMebx8jvFJEeFEKy60iFQetodwKjfglJ3lrX5clRrfQXgVygwXluURQiRyFzHGjHbhOjLWyO41Hk1WJVHiZUOldeMpwYrlrhiiLkhyOGIXepY1snuVjlzCrs3m1sjs3r5wXUs/jKgeaumeownIxVnhUs5yzRGd7Co8KWl7nWHG5EyTFlOlVLlL0BppnBcHMLdW9XeAO0ty1SbUZU3VUqno4gg3TLSaudNH75fIxrF/s6HXONL9MtW1ljJOF6kKibonwr9eyeVHA/9cm4/eGHCoeCtjNpk7ElUKTQrY1/06idcKXRo8eAZ72STnaIeKMWLM3oIDSUhO7vYIWcgssmmkEKhgqhRwO4l944sLZl1v07vU56J71O1iaPVLZrjWQ1QO6Rnu3jQ7iKpXBQLabKQO36kFbXSbczmCtPRMTDAjQGhh+tAeQ7pCjnnCj7lCjQeku+kOM+mPC9hyH1gw1faFxe9JtyUgzLJxqD9qNEKqmcCYipcw6Tqqqy3JVQNcLuuVJS1VqXSRIER2uhSHci0KqzwmLi8u5UHX22bVTtiEolhmnPMJUge2j97WnGrSEPQ+1NFe92286oy0c0bC4iziD4BWmKkh5BWKzr3QfmuW+fuwYEnQosOHsglxsJ3cCXXw9IWor4LtAu63mOMwbzOKkzNzvKpkQeuOctYHcKlzjgEtFk8QxF69obhOCt9uULvS2SKEa+cDdzsiVId8yIihhketxuGPeggTUYmudpd2+rLb/b4VvHlT/55nZlGrVWM4TonUkHMf+86GrAgpZXJs7tBpN628vSIKt9UPV0zgHLJLjDp6b99iGua8s5sYQcJjnIBzKWystUKOl9Y4GQuHJvFVELkT9mLY+n4tSbTPDZJOGoLKUKlWHV72YmEpETKldYUc4U7kg8SLi/JqOTCJU+xh0U2YvQcUJOtIk/i+klzhCBHQxm8/C5KrCym/YBttLOl1lLlOkvax3YDs2BMdEjMwVUXxd5DK31UCTYDcEV7HYh654HSOKc+KGy2j0UaEGH9mPysKmtiPgSZJat3Nf1oTHNLy56Q6pGj5Nm9/uMkt/A1xlxcbGv54WO96C5DZUSdfg+2IuE0U2kreMYWYYa5myE3OWx1v4tmhFQht+N1URWitb6s/Wfbti8dUlfFOiyZ2oNjyeVS769q8jcLnl+xvjNaf8wx6B0YiyNfZOwM7gQcsBmHnZnUmhqBHtf/oVk2W+Vzq7totJbEOg/lGpM1c4grsup9u1XZgAPUhyhFXpMQeXbBmY1grUm1HTJeGcDlzoztYj6LWjGTpiXNGI6L98z4IZOrrFglG2PkKBHNpHkaBwsf/RQ+feLChiukcMaS+T6eoPPPa/nMcgFiafxvAaZjNL4cBp/8s4zQlyKn0pfnYxPLft2dlF4RQOrBMU67T9ZsIW1H4u2QYNdO8XkmkIO1xBZg/lpryC7LLU0PuuITvc9FUhC46UNgj9bugDBatbQ+IKzFqusKr5HKE90546AOCX1BUR+Zu+me4OLuy7LK0kfbPlwW4Ve+hb4CSkZsAqH3wB3FDiMl96ZsMw9VvKnj+bnv7fzO3JqJe+22iDW0kjte0UkzLkW9IyElSRO5idulozOzGLEWVS4b60yGE3ugmw+OfbVoyBsd852Tk5b+1Tt1iWNy+0hAs/4AWOQSPdvssVGJpK91Jjd5MmH1/hRtItXjb2ZvppkcF19MZCBnS6MfyLXi4JsAtHyN8SdLOatCM/3E65T0By6HbK/HA7pTrcTnm4nbIX1uF2Sg/R4XbKw+2UgxI2HW6nPFy2MNyfdyJg/yOuE7jbcTvcTln9d/e3U7aZ3MdfT/nQNkSgPrF11xLvNe4+7GmDpT5x2y3x3rY/pBXocM5SIfvQ9mxBsORskW1EW0bsfa35un5k6m89asrvwpILx5Be7tyM86QjWuSgCx50wYMueNAFJ8TSdtXWNV5d+66gf9a/W9xI4F15rXPIY8RVh/b3A93zUmMDNuFrcLQdrIcqmhKpcDpykXWJkOHTMpmDI98SiBm4kr1MZ/O3158/1jPnDXMVMhU/tBccqiyLodSRe8a6Oi8zLzTEXhWs+78FSIIbNwPt2ni4AwIqHAUBbkmeSrgjdAmXLlPWwW8DpGmgW9A0C0+tl8wd0V39hHq5FU1ixftgw+0zXOYEAnTtcFZ5Up+v02CBW2LzJHHdUx9Nt1jTJWb+am0etCzX5mW3435RI/qXXbAnTXT+Z9Nn/cnO65HUe9I9saGhJu6aryyQ1n1r/TZvQ9pc1lF7ZR4ugsnQEr6WCkv/jk33qIWp3OtutvLqRZMzlgX63gNa7YYRTOe7q+op5yodZbuaVqa2HOHriREi1KVM7LlrLVQJtzxa+jMXtSjMrh4i6t7z9YufTfE2t1bHMRNCNHUiLqyIuS2uRKzdhNl10cREA3fm7azxkudGMxE5Yya8C2JgS4C6d3vgJXy9gHYMn+09GK+JydNuzqzADX5tknYV2AORgcWi10iZPHrCNas4zKzDzLr3mdU+q8aj+4xvUZynWXFAbUgnASKFGwlYxiY2NFZSggKBLtqqebvsPhxjb6ssab9CZyzLlZyht3DXsJyhT7nSTzRPnfCYRG1X13B+vaAslGZ4d0P0G8jIDTls4L4iG0flTJRDvHwdLoZZw33lzmABsS5UdjgzLHCLF/R4jr4wt+wZIVEZVRRxtqLrZqq/FkCLoJDaT349+Y8qsgokE8hg08DU/S0G/WFV45SzNY+XnmZsnwyPsfqgPzj9Y3+cVUkLjYm1qqqvHrXeYKs9hXjg4LcNQQhFT7hfH3Pab0oBGhLehR3trPK4bYnrNlT1IHqbM8iPgxMUYUXWXNBf7SUuPeBOPn348Prj6UiIrDGjByg+5JvqhUMZVZjFJhXiKFChaocoGS4PYpf5ylvF3Nzcyl8Sb2Z+2F785f3wealJwSfVmSk3XKiFWU1eISXytt2tI492DYxsAYA6Zuz0rhpVIOM9Nu7TUm5UvAUNK5Tjxe5r8NI3Lf9h/m/zZ1bxdhmKjEZJ4zl6y4UtZ10JJMoE5ZBK1/uyQQF6DuZq6Zxus8DRlmP/nuMAG5Dc0dDurcZDnwdMuIns4WVNYRQrByIBBjTUEANHUMjBE8E1VibIHSJK22N8xhODGB5oZ7nP6SDtRqHN27ThXjDEiaHM5z8dEBPcqxeE+dT3kpYJoks0Woef7XU3acKj6zvBi1Oe2/CxKuZbTHWXur2BBqBXnyUp3SrmuoZGrUZLpnKv9gp+KyEcbKKltxoxpWsvs1VZtb1j8gAavShSRqYSBgFEMsJsGKA2KbgPmJzRb56MVPiasHKNu7p4c1m+veoC17zHaJjvXnG9UcviMWXPe+klz04LJrfUrb7H1pR98/S9j/r3OH0PPtlR33Pk0T76XgAAuvd0GCWQHZJiFH5hC71BCLIAFgKPZLjXzHxlksZrCp6gIXKOzpSXNm5JIpxLuGvNnCGn5sIGk0aNzNCSSBoT6aVZbFAsq59VSJmxclnpEnpN0NX/e/KWi1ssYhLrv67m6IIQhBNp8tJdFX1yFXKWu0Pn5pOGY7M5RIZrDrJ8mdCoIbCriGEUr0znz9HZCjFeftigV/YSFi4fn7Jac0DXtTgEvcGqqTmEgDQpArBWfe03mw3j4FVcIfuQDt4P7dH8LxpK/2AZVQ6R8FNHwn85RMIfIuEPkfCHSPhDJPwhEj4M6RAJf4h+ahY4RD8dop/+B0c/1VHcSSR8aW0bf7o6sdPhGwMAPCYekfl6biDNkEtl/LjFu2gyW+95cfpJmKIrSgR6dH522kJXTWhjtme5jmxbhJIzQ093ynxSmrb7yE9/DFu5U9IZ0rl0RwLOlP7JPGkxplsjNvmWcaHK85ArW89VdzBgSQ3tHwQgiMwTtd8UBWvxKtwmUz9KiRJahKuhE3V6M6QvdO2p5QarMp2mMbqCc2mLGSUKCL09QL3lAlEWCbhYRW+iscIzlGJxDW7BWosyjsFF6k8cx43jOWTSYKb8hsRg1Y8wQ0sC97/yFTqCb45m6MiWOZrpD44kw5nccNWSa33DpVqUs2vakfDWKreewzl8JfOp5XKrAlPp/JKbIu+jVj2TZFtU1JSMhXWI0W9wyjzRUvSleqRouQt4yD8OR5KyyHp5ZzzazNEXaY+eI55muXLHaVf/6Z1ARjzJ07ZMqzghLMYi2Jh859GxHqqCWEW8cLczmmqSuLu8aUrgzNuo/Xa+2yErzhczLtVakKpT2bl5ONqzrPxux+PGChq0u0NoFchd+4TWzzvbusH9+824ltGU/Mq7L3ZqJ/WrXb0Ksvfjv+arU+H1o2nRLV3JcJxSNsqRzIUWNKotjLlY4WUzbUtJM90az+nRJIM1D3OZe/v68vX7qR3mAtfco07XnxLP8+P58Sg4p86pna8QHuvoUdK9ePP+zckl+j/o7edPH2AM5b+PwvEXez+CvVvtoTwJ7WotSFy59+Sz/t2yRsO77lhVVx168AhoA7ZYLQcultNt0S49J9WzUydNDarQxa2lU9bUwWe6xip9l/1+jk4qauNViqUi4mqGrmSCb4j+I9rQJL5Cj7Rk/nz69vvXn96iW73PZWsE7x7PQrrplVYkKCPJ1XD/3KniABvNgtBM3ZgbIpZcQrvMZUVXoBdf2QuKWrDeyWRs1DqhS++F89kF/xJz0fCNVj21FDcscEMxwogRdcvFtbdhH6pVROkYr4xBrmtpilmMCARxtR30OoExn+yejHfQVWyNqAKHVqS4w+Dum0zNjREopZHojh+bdPUoV40OYXVNJrzeS1O9Jtvqlsx1gN6Kdg8OFlNmjwA3XrHOtZCU5srVMKgIJ4mGZCWaOb7xRNoFPBi+7zAV7LjfKKijffwbQxBQl4NjrjZT7jfeU5Z/g1rL8Kt7D2eBq3dxXKLSeLpTI7Vc+TEwJABsRTtQzQRfC5zurh/sTHjS9ea8XHAcMLCVSZcXqh/Q9JJyUFDbfqEnYM4poy5Kg6BxsJJI8UDUq09Xyrrzxs5HrHYmSnMLZKSl0cXFO91uygwqOex8sys4f8CWWHdMjXBdrTp6HUUkU8bO+BbTpDAznrEbnND4aO6VCdBICWYSYSRz8J9e5YkhNy9rsGWKa7lhmKx/mAtVLo6bAyTsWX6Br15f2USsFEkzBTd+r6BwvZ87fVJHdGnN/9W6mdY7N8NSaqF5BD1qfImvyfaoDVXjlN8xYeDFIKhltudagFK1v7QETnHzkLbQ2ATPMhI3/bUnxqd7tlRj7RBr9ZdnhJk7v9KUxBQrkmwdqjbQgfzNnR4xYwBDFue9ulTSNcMqF02GH4Sj+Lww8Vpgxl/9mmzbCIecSbrWugGARruUXNkprWfRvCVUwPyb2rck7F3S7l8ywsOk/1x+oA/RCD+TYb4Ld4eMqgafocGuHXcGy5Dt7K1+v5zJ0PV75wzyzxnioTOiv4Z66YzxS5msy1q9U3w8Mo/5HWpsRk8r4nfdQb+meuW2riO1uJpLjflXWKVBLfr46RJOH/OYE9F0hB0kGyqODrq2CEsjonS1xba7W0FSjSvHB1K/vPy7JxQrFGmb8cET2rc7KmWRzRcZU0EixcV2DxBB7/9inATnO+riCos1UXabwj1LSB2gvKUq2gSOzL2sLGlIvA3rqpqVDuyIGkLPDknjxnF4t3qnc84S3nHaBaXPoI4qw9+WhLK1ceJoZZrGPn6wttlF/uy0VZGbnCAMYgfFTSgOYEC9+ju04knsuY0wYhylW/XjDQmkFh5ALCYrnCfKVNBBLsji0AMPwuOO8r0zua846V4CIHfAc60ASotVgLxnkr2rFCmmas9c+8AWUovn3m2kQ+jekZV0EOkG601hDh1C+R4Novb4QwlMVvTaO/+4NE/GOV7Zj/rT7ZX00D4nHkF66EFyOjgo+2R1CA74RLkJWhWsQxT/IYr/EMUfQneI4keHKP5DFD87RPEfovgHwzpE8R+i+A9R/OPwHKL4D1H8NViHKP5DFL+H4zcfxV9FAvvZBXDxhLtFL0OsoSCD5FeCM0VY3G7Y2M2G5s9hRwMWnfCWFUfXGkSbtaAHQ9iuIorbimz19szRWRAo2KNMsszv/n8AAAD//yDtkSQ=" } diff --git a/filebeat/module/apache2/_meta/config.reference.yml b/filebeat/module/apache2/_meta/config.reference.yml index ad61cd6f5f1..6667cb4428c 100644 --- a/filebeat/module/apache2/_meta/config.reference.yml +++ b/filebeat/module/apache2/_meta/config.reference.yml @@ -10,6 +10,8 @@ # Input configuration (advanced). Any input configuration option # can be added under this section. #input: + #Keeps the original message, so the data can be processed again on Ingest Node. + #keep_original_message: true # Error logs #error: @@ -22,3 +24,5 @@ # Input configuration (advanced). Any input configuration option # can be added under this section. #input: + #Keeps the original message, so the data can be processed again on Ingest Node. + #keep_original_message: true diff --git a/filebeat/module/apache2/access/test/test.log-expected.json b/filebeat/module/apache2/access/test/test.log-expected.json index a8fbd26c0a2..e5062e0af46 100644 --- a/filebeat/module/apache2/access/test/test.log-expected.json +++ b/filebeat/module/apache2/access/test/test.log-expected.json @@ -11,6 +11,7 @@ "fileset.module": "apache2", "fileset.name": "access", "input.type": "log", + "log.original": "::1 - - [26/Dec/2016:16:16:29 +0200] \"GET /favicon.ico HTTP/1.1\" 404 209", "offset": 0, "prospector.type": "log" }, @@ -36,6 +37,7 @@ "fileset.module": "apache2", "fileset.name": "access", "input.type": "log", + "log.original": "192.168.33.1 - - [26/Dec/2016:16:22:13 +0000] \"GET /hello HTTP/1.1\" 404 499 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:50.0) Gecko/20100101 Firefox/50.0\"", "offset": 73, "prospector.type": "log" }, @@ -47,6 +49,7 @@ "fileset.module": "apache2", "fileset.name": "access", "input.type": "log", + "log.original": "::1 - - [26/Dec/2016:16:16:48 +0200] \"-\" 408 -", "offset": 238, "prospector.type": "log" }, @@ -71,6 +74,7 @@ "fileset.module": "apache2", "fileset.name": "access", "input.type": "log", + "log.original": "172.17.0.1 - - [29/May/2017:19:02:48 +0000] \"GET /stringpatch HTTP/1.1\" 404 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "offset": 285, "prospector.type": "log" } diff --git a/filebeat/module/apache2/error/test/test.log-expected.json b/filebeat/module/apache2/error/test/test.log-expected.json index 96d61106465..2ff1401fb3c 100644 --- a/filebeat/module/apache2/error/test/test.log-expected.json +++ b/filebeat/module/apache2/error/test/test.log-expected.json @@ -7,6 +7,7 @@ "fileset.module": "apache2", "fileset.name": "error", "input.type": "log", + "log.original": "[Mon Dec 26 16:22:08 2016] [error] [client 192.168.33.1] File does not exist: /var/www/favicon.ico", "offset": 0, "prospector.type": "log" }, @@ -19,6 +20,7 @@ "fileset.module": "apache2", "fileset.name": "error", "input.type": "log", + "log.original": "[Mon Dec 26 16:15:55.103786 2016] [core:notice] [pid 11379] AH00094: Command line: '/usr/local/Cellar/httpd24/2.4.23_2/bin/httpd'", "offset": 99, "prospector.type": "log" }, @@ -33,6 +35,7 @@ "fileset.module": "apache2", "fileset.name": "error", "input.type": "log", + "log.original": "[Fri Sep 09 10:42:29.902022 2011] [core:error] [pid 35708:tid 4328636416] [client 72.15.99.187] File does not exist: /usr/local/apache2/htdocs/favicon.ico", "offset": 229, "prospector.type": "log" } diff --git a/filebeat/module/auditd/_meta/config.reference.yml b/filebeat/module/auditd/_meta/config.reference.yml index 57776242584..af33a43204a 100644 --- a/filebeat/module/auditd/_meta/config.reference.yml +++ b/filebeat/module/auditd/_meta/config.reference.yml @@ -9,3 +9,5 @@ # Input configuration (advanced). Any input configuration option # can be added under this section. #input: + #Keeps the original message, so the data can be processed again on Ingest Node. + #keep_original_message: true diff --git a/filebeat/module/auditd/log/test/test.log-expected.json b/filebeat/module/auditd/log/test/test.log-expected.json index 4b63b828497..16ff3626a86 100644 --- a/filebeat/module/auditd/log/test/test.log-expected.json +++ b/filebeat/module/auditd/log/test/test.log-expected.json @@ -14,6 +14,7 @@ "fileset.module": "auditd", "fileset.name": "log", "input.type": "log", + "log.original": "type=MAC_IPSEC_EVENT msg=audit(1485893834.891:18877201): op=SPD-delete auid=4294967295 ses=4294967295 res=1 src=192.168.2.0 src_prefixlen=24 dst=192.168.0.0 dst_prefixlen=16", "offset": 0, "prospector.type": "log" }, @@ -48,6 +49,7 @@ "fileset.module": "auditd", "fileset.name": "log", "input.type": "log", + "log.original": "type=SYSCALL msg=audit(1485893834.891:18877199): arch=c000003e syscall=44 success=yes exit=184 a0=9 a1=7f564b2672a0 a2=b8 a3=0 items=0 ppid=1240 pid=1281 auid=4294967295 uid=0 gid=0 euid=0 suid=0 fsuid=0 egid=0 sgid=0 fsgid=0 tty=(none) ses=4294967295 comm=\"charon\" exe=2F7573722F6C6962657865632F7374726F6E677377616E2F636861726F6E202864656C6574656429 key=(null)", "offset": 174, "prospector.type": "log" } diff --git a/filebeat/module/elasticsearch/audit/test/test.log-expected.json b/filebeat/module/elasticsearch/audit/test/test.log-expected.json index 77948ecc89f..865c14ca5e6 100644 --- a/filebeat/module/elasticsearch/audit/test/test.log-expected.json +++ b/filebeat/module/elasticsearch/audit/test/test.log-expected.json @@ -9,6 +9,7 @@ "fileset.module": "elasticsearch", "fileset.name": "audit", "input.type": "log", + "log.original": "[2018-06-19T05:16:15,549] [rest] [authentication_failed] origin_address=[147.107.128.77], principal=[i030648], uri=[/_xpack/security/_authenticate]", "message": "[2018-06-19T05:16:15,549] [rest] [authentication_failed] origin_address=[147.107.128.77], principal=[i030648], uri=[/_xpack/security/_authenticate]", "offset": 0, "prospector.type": "log", @@ -25,6 +26,7 @@ "fileset.module": "elasticsearch", "fileset.name": "audit", "input.type": "log", + "log.original": "[2018-06-19T05:07:52,304] [v_VJhjV] [rest] [authentication_failed]\torigin_address=[172.22.0.3], principal=[rado], uri=[/_xpack/security/_authenticate]", "message": "[2018-06-19T05:07:52,304] [v_VJhjV] [rest] [authentication_failed]\torigin_address=[172.22.0.3], principal=[rado], uri=[/_xpack/security/_authenticate]", "offset": 155, "prospector.type": "log", @@ -42,6 +44,7 @@ "fileset.module": "elasticsearch", "fileset.name": "audit", "input.type": "log", + "log.original": "[2018-06-19T05:00:15,778] [transport] [access_granted] origin_type=[local_node], origin_address=[192.168.1.165], principal=[_xpack_security], action=[indices:data/read/scroll/clear], request=[ClearScrollRequest]", "message": "[2018-06-19T05:00:15,778] [transport] [access_granted] origin_type=[local_node], origin_address=[192.168.1.165], principal=[_xpack_security], action=[indices:data/read/scroll/clear], request=[ClearScrollRequest]", "offset": 306, "prospector.type": "log", @@ -57,6 +60,7 @@ "fileset.module": "elasticsearch", "fileset.name": "audit", "input.type": "log", + "log.original": "[2018-06-19T05:07:45,544] [v_VJhjV] [rest] [anonymous_access_denied]\torigin_address=[172.22.0.3], uri=[/_xpack/security/_authenticate]", "message": "[2018-06-19T05:07:45,544] [v_VJhjV] [rest] [anonymous_access_denied]\torigin_address=[172.22.0.3], uri=[/_xpack/security/_authenticate]", "offset": 519, "prospector.type": "log", @@ -72,6 +76,7 @@ "fileset.module": "elasticsearch", "fileset.name": "audit", "input.type": "log", + "log.original": "[2018-06-19T05:26:27,268] [rest] [authentication_failed]\torigin_address=[147.107.128.77], principal=[N078801], uri=[/_xpack/security/_authenticate]", "message": "[2018-06-19T05:26:27,268] [rest] [authentication_failed]\torigin_address=[147.107.128.77], principal=[N078801], uri=[/_xpack/security/_authenticate]", "offset": 654, "prospector.type": "log", @@ -89,6 +94,7 @@ "fileset.module": "elasticsearch", "fileset.name": "audit", "input.type": "log", + "log.original": "[2018-06-19T05:55:26,898] [transport] [access_denied]\torigin_type=[rest], origin_address=[147.107.128.77], principal=[_anonymous], action=[cluster:monitor/main], request=[MainRequest]", "message": "[2018-06-19T05:55:26,898] [transport] [access_denied]\torigin_type=[rest], origin_address=[147.107.128.77], principal=[_anonymous], action=[cluster:monitor/main], request=[MainRequest]", "offset": 802, "prospector.type": "log", @@ -106,6 +112,7 @@ "fileset.module": "elasticsearch", "fileset.name": "audit", "input.type": "log", + "log.original": "[2018-06-19T05:24:15,190] [v_VJhjV] [rest] [authentication_failed]\torigin_address=[172.18.0.3], principal=[elastic], uri=[/_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip], request_body=[body]", "message": "[2018-06-19T05:24:15,190] [v_VJhjV] [rest] [authentication_failed]\torigin_address=[172.18.0.3], principal=[elastic], uri=[/_nodes?filter_path=nodes.*.version%2Cnodes.*.http.publish_address%2Cnodes.*.ip], request_body=[body]", "offset": 986, "prospector.type": "log", diff --git a/filebeat/module/elasticsearch/gc/test/test.log-expected.json b/filebeat/module/elasticsearch/gc/test/test.log-expected.json index c9d0621afc9..aa32d790c77 100644 --- a/filebeat/module/elasticsearch/gc/test/test.log-expected.json +++ b/filebeat/module/elasticsearch/gc/test/test.log-expected.json @@ -14,6 +14,7 @@ "fileset.module": "elasticsearch", "fileset.name": "gc", "input.type": "log", + "log.original": "2018-03-03T19:37:06.157+0500: 14597.826: [GC (CMS Initial Mark) [1 CMS-initial-mark: 131804K(174784K)] 142444K(253440K), 0.0021716 secs] [Times: user=0.01 sys=0.00, real=0.00 secs]", "message": "2018-03-03T19:37:06.157+0500: 14597.826: [GC (CMS Initial Mark) [1 CMS-initial-mark: 131804K(174784K)] 142444K(253440K), 0.0021716 secs] [Times: user=0.01 sys=0.00, real=0.00 secs]", "offset": 0, "prospector.type": "log", @@ -27,6 +28,7 @@ "fileset.module": "elasticsearch", "fileset.name": "gc", "input.type": "log", + "log.original": "2018-06-11T01:53:11.382+0000: 1396138.752: Total time for which application threads were stopped: 0.0083760 seconds, Stopping threads took: 0.0000702 seconds", "message": "2018-06-11T01:53:11.382+0000: 1396138.752: Total time for which application threads were stopped: 0.0083760 seconds, Stopping threads took: 0.0000702 seconds", "offset": 181, "prospector.type": "log", @@ -54,6 +56,7 @@ "fileset.module": "elasticsearch", "fileset.name": "gc", "input.type": "log", + "log.original": "2018-06-30T16:35:26.632+0500: 224.671: [GC (CMS Final Remark) [YG occupancy: 113198 K (157248 K)]224.671: [Rescan (parallel) , 0.0148273 secs]224.686: [weak refs processing, 0.0003647 secs]224.687: [class unloading, 0.0188407 secs]224.705: [scrub symbol table, 0.0100207 secs]224.715: [scrub string table, 0.0005253 secs][1 CMS-remark: 277821K(349568K)] 391020K(506816K), 0.0457689 secs] [Times: user=0.12 sys=0.00, real=0.04 secs]", "message": "2018-06-30T16:35:26.632+0500: 224.671: [GC (CMS Final Remark) [YG occupancy: 113198 K (157248 K)]224.671: [Rescan (parallel) , 0.0148273 secs]224.686: [weak refs processing, 0.0003647 secs]224.687: [class unloading, 0.0188407 secs]224.705: [scrub symbol table, 0.0100207 secs]224.715: [scrub string table, 0.0005253 secs][1 CMS-remark: 277821K(349568K)] 391020K(506816K), 0.0457689 secs] [Times: user=0.12 sys=0.00, real=0.04 secs]", "offset": 339, "prospector.type": "log", diff --git a/filebeat/module/elasticsearch/server/test/test.log-expected.json b/filebeat/module/elasticsearch/server/test/test.log-expected.json index f53a28cf9fb..7aedf186e11 100644 --- a/filebeat/module/elasticsearch/server/test/test.log-expected.json +++ b/filebeat/module/elasticsearch/server/test/test.log-expected.json @@ -8,6 +8,7 @@ "fileset.name": "server", "input.type": "log", "log.level": "INFO", + "log.original": "[2018-05-17T08:29:12,177][INFO ][o.e.c.m.MetaDataCreateIndexService] [vWNJsZ3] [test-filebeat-modules] creating index, cause [auto(bulk api)], templates [test-filebeat-modules], shards [5]/[1], mappings [doc]", "message": "creating index, cause [auto(bulk api)], templates [test-filebeat-modules], shards [5]/[1], mappings [doc]", "offset": 0, "prospector.type": "log", @@ -21,6 +22,7 @@ "fileset.name": "server", "input.type": "log", "log.level": "INFO", + "log.original": "[2018-05-17T08:19:35,939][INFO ][o.e.n.Node ] [] initializing ...", "message": "initializing ...", "offset": 209, "prospector.type": "log", @@ -34,6 +36,7 @@ "fileset.name": "server", "input.type": "log", "log.level": "INFO", + "log.original": "[2018-05-17T08:19:36,089][INFO ][o.e.e.NodeEnvironment ] [vWNJsZ3] using [1] data paths, mounts [[/ (/dev/disk1s1)]], net usable_space [32.4gb], net total_space [233.5gb], types [apfs]", "message": "using [1] data paths, mounts [[/ (/dev/disk1s1)]], net usable_space [32.4gb], net total_space [233.5gb], types [apfs]", "offset": 289, "prospector.type": "log", @@ -47,6 +50,7 @@ "fileset.name": "server", "input.type": "log", "log.level": "INFO", + "log.original": "[2018-05-17T08:19:36,090][INFO ][o.e.e.NodeEnvironment ] [vWNJsZ3] heap size [990.7mb], compressed ordinary object pointers [true]", "message": "heap size [990.7mb], compressed ordinary object pointers [true]", "offset": 477, "prospector.type": "log", @@ -59,6 +63,7 @@ "fileset.name": "server", "input.type": "log", "log.level": "INFO", + "log.original": "[2018-05-17T08:19:36,116][INFO ][o.e.n.Node ] node name [vWNJsZ3] derived from node ID [vWNJsZ3nTIKh5a1ai-ftYQ]; set [node.name] to override", "message": "node name [vWNJsZ3] derived from node ID [vWNJsZ3nTIKh5a1ai-ftYQ]; set [node.name] to override", "offset": 611, "prospector.type": "log", @@ -72,6 +77,7 @@ "fileset.name": "server", "input.type": "log", "log.level": "INFO", + "log.original": "[2018-05-17T08:23:48,941][INFO ][o.e.c.r.a.DiskThresholdMonitor] [vWNJsZ3] low disk watermark [85%] exceeded on [vWNJsZ3nTIKh5a1ai-ftYQ][vWNJsZ3][/Users/ruflin/Downloads/elasticsearch-6.2.4/data/nodes/0] free: 33.4gb[14.3%], replicas will not be assigned to this node", "message": "low disk watermark [85%] exceeded on [vWNJsZ3nTIKh5a1ai-ftYQ][vWNJsZ3][/Users/ruflin/Downloads/elasticsearch-6.2.4/data/nodes/0] free: 33.4gb[14.3%], replicas will not be assigned to this node", "offset": 766, "prospector.type": "log", @@ -86,6 +92,7 @@ "fileset.name": "server", "input.type": "log", "log.level": "INFO", + "log.original": "[2018-05-17T08:29:09,245][INFO ][o.e.c.m.MetaDataCreateIndexService] [vWNJsZ3] [filebeat-test-input] creating index, cause [auto(bulk api)], templates [filebeat-test-input], shards [5]/[1], mappings [doc]", "message": "creating index, cause [auto(bulk api)], templates [filebeat-test-input], shards [5]/[1], mappings [doc]", "offset": 1034, "prospector.type": "log", @@ -101,6 +108,7 @@ "fileset.name": "server", "input.type": "log", "log.level": "INFO", + "log.original": "[2018-05-17T08:29:09,576][INFO ][o.e.c.m.MetaDataMappingService] [vWNJsZ3] [filebeat-test-input/aOGgDwbURfCV57AScqbCgw] update_mapping [doc]", "message": "update_mapping [doc]", "offset": 1239, "prospector.type": "log", @@ -116,6 +124,7 @@ "fileset.name": "server", "input.type": "log", "log.level": "INFO", + "log.original": "[2018-07-09T12:47:33,959][INFO ][o.e.c.m.MetaDataMappingService] [QGY1F5P] [.kibana/3tWftqb4RLKdyCAga9syGA] update_mapping [doc]", "message": "update_mapping [doc]", "offset": 1380, "prospector.type": "log", @@ -129,6 +138,7 @@ "fileset.name": "server", "input.type": "log", "log.level": "INFO", + "log.original": "[2018-05-17T08:29:25,598][INFO ][o.e.n.Node ] [vWNJsZ3] closing ...", "message": "closing ...", "offset": 1509, "prospector.type": "log", @@ -142,6 +152,7 @@ "fileset.name": "server", "input.type": "log", "log.level": "INFO", + "log.original": "[2018-05-17T08:29:25,612][INFO ][o.e.n.Node ] [vWNJsZ3] closed", "message": "closed", "offset": 1591, "prospector.type": "log", @@ -155,6 +166,7 @@ "fileset.name": "server", "input.type": "log", "log.level": "INFO", + "log.original": "[2018-07-03T11:45:48,548][INFO ][o.e.d.z.ZenDiscovery ] [srvmulpvlsk252_md] master_left [{srvmulpvlsk250_md}{igrwSoPGSJ6u_5b8k26tgQ}{PuRqciBFRbiQvL2_lS7LrQ}{srvmulpvlsk250.loganalytics.santanderuk.corp}{180.39.9.91:9300}{ml.max_open_jobs=10, ml.enabled=true}], reason [failed to ping, tried [3] times, each with maximum [30s] timeout]", "message": "master_left [{srvmulpvlsk250_md}{igrwSoPGSJ6u_5b8k26tgQ}{PuRqciBFRbiQvL2_lS7LrQ}{srvmulpvlsk250.loganalytics.santanderuk.corp}{180.39.9.91:9300}{ml.max_open_jobs=10, ml.enabled=true}], reason [failed to ping, tried [3] times, each with maximum [30s] timeout]", "offset": 1668, "prospector.type": "log", @@ -171,6 +183,7 @@ "multiline" ], "log.level": "WARN", + "log.original": "[2018-07-03T11:45:48,548][WARN ][o.e.d.z.ZenDiscovery ] [srvmulpvlsk252_md] master left (reason = failed to ping, tried [3] times, each with maximum [30s] timeout), current nodes: nodes:\n {srvmulpvlsk252_md}{uc5xdiQgRhaBIY-sszgjvQ}{X9pC0t1UQQix_NNOM0J6JQ}{srvmulpvlsk252.loganalytics.santanderuk.corp}{180.39.9.93:9300}{ml.max_open_jobs=10, ml.enabled=true}, local\n {srvmulpvlsk258_md}{HgW6EDn5QCmWVmICy4saHw}{o8zku7OJR4CTp0IjY8Ag4Q}{srvmulpvlsk258.loganalytics.santanderuk.corp}{180.39.9.99:9300}{ml.max_open_jobs=10, ml.enabled=true}\n {srvmulpvlsk250_md}{igrwSoPGSJ6u_5b8k26tgQ}{PuRqciBFRbiQvL2_lS7LrQ}{srvmulpvlsk250.loganalytics.santanderuk.corp}{180.39.9.91:9300}{ml.max_open_jobs=10, ml.enabled=true}, master\n {srvmulpvlsk254_id}{wZYeAh2URc2NwBIHZolLWQ}{3nduupo-TzSPaXjQaNu4Sg}{srvmulpvlsk254.loganalytics.santanderuk.corp}{180.39.9.95:9300}{ml.max_open_jobs=10, ml.enabled=true}", "message": "master left (reason = failed to ping, tried [3] times, each with maximum [30s] timeout), current nodes: nodes:\n {srvmulpvlsk252_md}{uc5xdiQgRhaBIY-sszgjvQ}{X9pC0t1UQQix_NNOM0J6JQ}{srvmulpvlsk252.loganalytics.santanderuk.corp}{180.39.9.93:9300}{ml.max_open_jobs=10, ml.enabled=true}, local\n {srvmulpvlsk258_md}{HgW6EDn5QCmWVmICy4saHw}{o8zku7OJR4CTp0IjY8Ag4Q}{srvmulpvlsk258.loganalytics.santanderuk.corp}{180.39.9.99:9300}{ml.max_open_jobs=10, ml.enabled=true}\n {srvmulpvlsk250_md}{igrwSoPGSJ6u_5b8k26tgQ}{PuRqciBFRbiQvL2_lS7LrQ}{srvmulpvlsk250.loganalytics.santanderuk.corp}{180.39.9.91:9300}{ml.max_open_jobs=10, ml.enabled=true}, master\n {srvmulpvlsk254_id}{wZYeAh2URc2NwBIHZolLWQ}{3nduupo-TzSPaXjQaNu4Sg}{srvmulpvlsk254.loganalytics.santanderuk.corp}{180.39.9.95:9300}{ml.max_open_jobs=10, ml.enabled=true}", "offset": 2008, "prospector.type": "log", @@ -186,6 +199,7 @@ "multiline" ], "log.level": "WARN", + "log.original": "[2018-07-03T11:45:52,666][WARN ][r.suppressed ] path: /_xpack/monitoring/_bulk, params: {system_id=logstash, system_api_version=2, interval=1s}\norg.elasticsearch.cluster.block.ClusterBlockException: blocked by: [SERVICE_UNAVAILABLE/2/no master];\n at org.elasticsearch.cluster.block.ClusterBlocks.globalBlockedException(ClusterBlocks.java:165) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.cluster.block.ClusterBlocks.globalBlockedRaiseException(ClusterBlocks.java:151) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.xpack.monitoring.action.TransportMonitoringBulkAction.doExecute(TransportMonitoringBulkAction.java:57) ~[?:?]\n at org.elasticsearch.xpack.monitoring.action.TransportMonitoringBulkAction.doExecute(TransportMonitoringBulkAction.java:40) ~[?:?]\n at org.elasticsearch.action.support.TransportAction.doExecute(TransportAction.java:146) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:170) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.lambda$apply$1(SecurityActionFilter.java:133) ~[?:?]\n at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:59) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.lambda$authorizeRequest$4(SecurityActionFilter.java:208) ~[?:?]\n at org.elasticsearch.xpack.security.authz.AuthorizationUtils$AsyncAuthorizer.maybeRun(AuthorizationUtils.java:127) ~[?:?]\n at org.elasticsearch.xpack.security.authz.AuthorizationUtils$AsyncAuthorizer.setRunAsRoles(AuthorizationUtils.java:121) ~[?:?]\n at org.elasticsearch.xpack.security.authz.AuthorizationUtils$AsyncAuthorizer.authorize(AuthorizationUtils.java:109) ~[?:?]\n at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.authorizeRequest(SecurityActionFilter.java:210) ~[?:?]\n at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.lambda$applyInternal$3(SecurityActionFilter.java:186) ~[?:?]\n at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:59) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:212) ~[?:?]\n at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$4(AuthenticationService.java:246) ~[?:?]\n at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:257) ~[?:?]\n at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:210) ~[?:?]\n at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:159) ~[?:?]\n at org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:122) ~[?:?]\n at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.applyInternal(SecurityActionFilter.java:185) ~[?:?]\n at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.apply(SecurityActionFilter.java:145) ~[?:?]\n at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:168) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:142) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:84) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.client.node.NodeClient.executeLocally(NodeClient.java:83) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.client.node.NodeClient.doExecute(NodeClient.java:72) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:408) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.action.ActionRequestBuilder.execute(ActionRequestBuilder.java:80) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.xpack.monitoring.rest.action.RestMonitoringBulkAction.lambda$doPrepareRequest$0(RestMonitoringBulkAction.java:77) ~[?:?]\n at org.elasticsearch.rest.BaseRestHandler.handleReques", "message": "path: /_xpack/monitoring/_bulk, params: {system_id=logstash, system_api_version=2, interval=1s}\norg.elasticsearch.cluster.block.ClusterBlockException: blocked by: [SERVICE_UNAVAILABLE/2/no master];\n at org.elasticsearch.cluster.block.ClusterBlocks.globalBlockedException(ClusterBlocks.java:165) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.cluster.block.ClusterBlocks.globalBlockedRaiseException(ClusterBlocks.java:151) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.xpack.monitoring.action.TransportMonitoringBulkAction.doExecute(TransportMonitoringBulkAction.java:57) ~[?:?]\n at org.elasticsearch.xpack.monitoring.action.TransportMonitoringBulkAction.doExecute(TransportMonitoringBulkAction.java:40) ~[?:?]\n at org.elasticsearch.action.support.TransportAction.doExecute(TransportAction.java:146) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:170) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.lambda$apply$1(SecurityActionFilter.java:133) ~[?:?]\n at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:59) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.lambda$authorizeRequest$4(SecurityActionFilter.java:208) ~[?:?]\n at org.elasticsearch.xpack.security.authz.AuthorizationUtils$AsyncAuthorizer.maybeRun(AuthorizationUtils.java:127) ~[?:?]\n at org.elasticsearch.xpack.security.authz.AuthorizationUtils$AsyncAuthorizer.setRunAsRoles(AuthorizationUtils.java:121) ~[?:?]\n at org.elasticsearch.xpack.security.authz.AuthorizationUtils$AsyncAuthorizer.authorize(AuthorizationUtils.java:109) ~[?:?]\n at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.authorizeRequest(SecurityActionFilter.java:210) ~[?:?]\n at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.lambda$applyInternal$3(SecurityActionFilter.java:186) ~[?:?]\n at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:59) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:212) ~[?:?]\n at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$4(AuthenticationService.java:246) ~[?:?]\n at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:257) ~[?:?]\n at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:210) ~[?:?]\n at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:159) ~[?:?]\n at org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:122) ~[?:?]\n at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.applyInternal(SecurityActionFilter.java:185) ~[?:?]\n at org.elasticsearch.xpack.security.action.filter.SecurityActionFilter.apply(SecurityActionFilter.java:145) ~[?:?]\n at org.elasticsearch.action.support.TransportAction$RequestFilterChain.proceed(TransportAction.java:168) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:142) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.action.support.TransportAction.execute(TransportAction.java:84) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.client.node.NodeClient.executeLocally(NodeClient.java:83) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.client.node.NodeClient.doExecute(NodeClient.java:72) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.client.support.AbstractClient.execute(AbstractClient.java:408) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.action.ActionRequestBuilder.execute(ActionRequestBuilder.java:80) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.xpack.monitoring.rest.action.RestMonitoringBulkAction.lambda$doPrepareRequest$0(RestMonitoringBulkAction.java:77) ~[?:?]\n at org.elasticsearch.rest.BaseRestHandler.handleReques", "offset": 2907, "prospector.type": "log", @@ -201,6 +215,7 @@ "multiline" ], "log.level": "WARN", + "log.original": "[2018-07-03T11:48:02,552][WARN ][r.suppressed ] path: /_xpack/license, params: {}\norg.elasticsearch.discovery.MasterNotDiscoveredException: NodeDisconnectedException[[srvmulpvlsk250_md][180.39.9.91:9300][cluster:monitor/xpack/license/get] disconnected]\n at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction$4.onTimeout(TransportMasterNodeAction.java:209) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.cluster.ClusterStateObserver$ContextPreservingListener.onTimeout(ClusterStateObserver.java:311) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:139) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:111) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction.retry(TransportMasterNodeAction.java:194) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction.access$500(TransportMasterNodeAction.java:107) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction$3.handleException(TransportMasterNodeAction.java:183) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1067) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1067) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.transport.TransportService$Adapter.lambda$onConnectionClosed$6(TransportService.java:893) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:569) [elasticsearch-5.6.3.jar:5.6.3]\n at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]\n at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]\n at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]\nCaused by: org.elasticsearch.transport.NodeDisconnectedException: [srvmulpvlsk250_md][180.39.9.91:9300][cluster:monitor/xpack/license/get] disconnected", "message": "path: /_xpack/license, params: {}\norg.elasticsearch.discovery.MasterNotDiscoveredException: NodeDisconnectedException[[srvmulpvlsk250_md][180.39.9.91:9300][cluster:monitor/xpack/license/get] disconnected]\n at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction$4.onTimeout(TransportMasterNodeAction.java:209) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.cluster.ClusterStateObserver$ContextPreservingListener.onTimeout(ClusterStateObserver.java:311) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:139) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.cluster.ClusterStateObserver.waitForNextChange(ClusterStateObserver.java:111) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction.retry(TransportMasterNodeAction.java:194) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction.access$500(TransportMasterNodeAction.java:107) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.action.support.master.TransportMasterNodeAction$AsyncSingleAction$3.handleException(TransportMasterNodeAction.java:183) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1067) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1067) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.transport.TransportService$Adapter.lambda$onConnectionClosed$6(TransportService.java:893) ~[elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.common.util.concurrent.ThreadContext$ContextPreservingRunnable.run(ThreadContext.java:569) [elasticsearch-5.6.3.jar:5.6.3]\n at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_161]\n at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_161]\n at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]\nCaused by: org.elasticsearch.transport.NodeDisconnectedException: [srvmulpvlsk250_md][180.39.9.91:9300][cluster:monitor/xpack/license/get] disconnected", "offset": 7412, "prospector.type": "log", @@ -219,6 +234,7 @@ "multiline" ], "log.level": "WARN", + "log.original": "[2018-07-03T11:45:27,896][WARN ][o.e.m.j.JvmGcMonitorService] [srvmulpvlsk252_md] [gc][young][3449979][986594] duration [3.8s], collections [1]/[4.3s], total [3.8s]/[8.8h], memory [16.5gb]->[15.7gb]/[30.8gb], all_po\nols {[young] [1.2gb]->[24mb]/[1.4gb]}{[survivor] [191.3mb]->[191.3mb]/[191.3mb]}{[old] [15.1gb]->[15.5gb]/[29.1gb]}", "message": "duration [3.8s], collections [1]/[4.3s], total [3.8s]/[8.8h], memory [16.5gb]->[15.7gb]/[30.8gb], all_po\nols {[young] [1.2gb]->[24mb]/[1.4gb]}{[survivor] [191.3mb]->[191.3mb]/[191.3mb]}{[old] [15.1gb]->[15.5gb]/[29.1gb]}", "offset": 9873, "prospector.type": "log", @@ -233,6 +249,7 @@ "fileset.name": "server", "input.type": "log", "log.level": "WARN", + "log.original": "[2018-07-03T11:45:45,604][WARN ][o.e.m.j.JvmGcMonitorService] [srvmulpvlsk252_md] [gc][3449992] overhead, spent [1.6s] collecting in the last [1.8s]", "message": "overhead, spent [1.6s] collecting in the last [1.8s]", "offset": 10205, "prospector.type": "log", @@ -246,6 +263,7 @@ "fileset.name": "server", "input.type": "log", "log.level": "WARN", + "log.original": "[2018-07-03T11:48:02,541][WARN ][o.e.a.b.TransportShardBulkAction] [srvmulpvlsk252_md] [[pro_neocrmbigdata_paas-2018-27][0]] failed to perform indices:data/write/bulk[s] on replica [pro_neocrmbigdata_paas-2018-27][0], node[igrwSoPGSJ6u_5b8k26tgQ], [R], s[STARTED], a[id=DKK34YLHRMmJMkWg8jQH6w]", "message": "[[pro_neocrmbigdata_paas-2018-27][0]] failed to perform indices:data/write/bulk[s] on replica [pro_neocrmbigdata_paas-2018-27][0], node[igrwSoPGSJ6u_5b8k26tgQ], [R], s[STARTED], a[id=DKK34YLHRMmJMkWg8jQH6w]", "offset": 10354, "prospector.type": "log", @@ -262,6 +280,7 @@ "multiline" ], "log.level": "WARN", + "log.original": "[2018-07-03T20:10:07,376][WARN ][o.e.x.m.MonitoringService] [srvmulpvlsk252_md] monitoring execution failed\norg.elasticsearch.xpack.monitoring.exporter.ExportException: Exception when closing export bulk\n at org.elasticsearch.xpack.monitoring.exporter.ExportBulk$1$1.(ExportBulk.java:106) ~[?:?]\n at org.elasticsearch.xpack.monitoring.exporter.ExportBulk$1.onFailure(ExportBulk.java:104) ~[?:?]\n at org.elasticsearch.xpack.monitoring.exporter.ExportBulk$Compound$1.onResponse(ExportBulk.java:217) ~[?:?]\n at org.elasticsearch.xpack.monitoring.exporter.ExportBulk$Compound$1.onResponse(ExportBulk.java:211) ~[?:?]\n at org.elasticsearch.xpack.common.IteratingActionListener.onResponse(IteratingActionListener.java:108) ~[?:?]\n at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:59) [elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.xpack.monitoring.exporter.http.HttpExportBulk$1.onSuccess(HttpExportBulk.java:115) [x-pack-5.6.3.jar:5.6.3]\n at org.elasticsearch.client.RestClient$FailureTrackingResponseListener.onSuccess(RestClient.java:597) [elasticsearch-rest-client-5.6.3.jar:5.6.3]\n at org.elasticsearch.client.RestClient$1.completed(RestClient.java:352) [elasticsearch-rest-client-5.6.3.jar:5.6.3]\n at org.elasticsearch.client.RestClient$1.completed(RestClient.java:343) [elasticsearch-rest-client-5.6.3.jar:5.6.3]\n at org.apache.http.concurrent.BasicFuture.completed(BasicFuture.java:119) [httpcore-4.4.5.jar:4.4.5]\n at org.apache.http.impl.nio.client.DefaultClientExchangeHandlerImpl.responseCompleted(DefaultClientExchangeHandlerImpl.java:177) [httpasyncclient-4.1.2.jar:4.1.2]\n at org.apache.http.nio.protocol.HttpAsyncRequestExecutor.processResponse(HttpAsyncRequestExecutor.java:436) [httpcore-nio-4.4.5.jar:4.4.5]\n at org.apache.http.nio.protocol.HttpAsyncRequestExecutor.inputReady(HttpAsyncRequestExecutor.java:326) [httpcore-nio-4.4.5.jar:4.4.5]\n at org.apache.http.impl.nio.DefaultNHttpClientConnection.consumeInput(DefaultNHttpClientConnection.java:265) [httpcore-nio-4.4.5.jar:4.4.5]\n at org.apache.http.impl.nio.client.InternalIODispatch.onInputReady(InternalIODispatch.java:81) [httpasyncclient-4.1.2.jar:4.1.2]\n at org.apache.http.impl.nio.client.InternalIODispatch.onInputReady(InternalIODispatch.java:39) [httpasyncclient-4.1.2.jar:4.1.2]\n at org.apache.http.impl.nio.reactor.AbstractIODispatch.inputReady(AbstractIODispatch.java:114) [httpcore-nio-4.4.5.jar:4.4.5]\n at org.apache.http.impl.nio.reactor.BaseIOReactor.readable(BaseIOReactor.java:162) [httpcore-nio-4.4.5.jar:4.4.5]\n at org.apache.http.impl.nio.reactor.AbstractIOReactor.processEvent(AbstractIOReactor.java:337) [httpcore-nio-4.4.5.jar:4.4.5]\n at org.apache.http.impl.nio.reactor.AbstractIOReactor.processEvents(AbstractIOReactor.java:315) [httpcore-nio-4.4.5.jar:4.4.5]\n at org.apache.http.impl.nio.reactor.AbstractIOReactor.execute(AbstractIOReactor.java:276) [httpcore-nio-4.4.5.jar:4.4.5]\n at org.apache.http.impl.nio.reactor.BaseIOReactor.execute(BaseIOReactor.java:104) [httpcore-nio-4.4.5.jar:4.4.5]\n at org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor$Worker.run(AbstractMultiworkerIOReactor.java:588) [httpcore-nio-4.4.5.jar:4.4.5]\n at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]\n", "message": "monitoring execution failed\norg.elasticsearch.xpack.monitoring.exporter.ExportException: Exception when closing export bulk\n at org.elasticsearch.xpack.monitoring.exporter.ExportBulk$1$1.(ExportBulk.java:106) ~[?:?]\n at org.elasticsearch.xpack.monitoring.exporter.ExportBulk$1.onFailure(ExportBulk.java:104) ~[?:?]\n at org.elasticsearch.xpack.monitoring.exporter.ExportBulk$Compound$1.onResponse(ExportBulk.java:217) ~[?:?]\n at org.elasticsearch.xpack.monitoring.exporter.ExportBulk$Compound$1.onResponse(ExportBulk.java:211) ~[?:?]\n at org.elasticsearch.xpack.common.IteratingActionListener.onResponse(IteratingActionListener.java:108) ~[?:?]\n at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:59) [elasticsearch-5.6.3.jar:5.6.3]\n at org.elasticsearch.xpack.monitoring.exporter.http.HttpExportBulk$1.onSuccess(HttpExportBulk.java:115) [x-pack-5.6.3.jar:5.6.3]\n at org.elasticsearch.client.RestClient$FailureTrackingResponseListener.onSuccess(RestClient.java:597) [elasticsearch-rest-client-5.6.3.jar:5.6.3]\n at org.elasticsearch.client.RestClient$1.completed(RestClient.java:352) [elasticsearch-rest-client-5.6.3.jar:5.6.3]\n at org.elasticsearch.client.RestClient$1.completed(RestClient.java:343) [elasticsearch-rest-client-5.6.3.jar:5.6.3]\n at org.apache.http.concurrent.BasicFuture.completed(BasicFuture.java:119) [httpcore-4.4.5.jar:4.4.5]\n at org.apache.http.impl.nio.client.DefaultClientExchangeHandlerImpl.responseCompleted(DefaultClientExchangeHandlerImpl.java:177) [httpasyncclient-4.1.2.jar:4.1.2]\n at org.apache.http.nio.protocol.HttpAsyncRequestExecutor.processResponse(HttpAsyncRequestExecutor.java:436) [httpcore-nio-4.4.5.jar:4.4.5]\n at org.apache.http.nio.protocol.HttpAsyncRequestExecutor.inputReady(HttpAsyncRequestExecutor.java:326) [httpcore-nio-4.4.5.jar:4.4.5]\n at org.apache.http.impl.nio.DefaultNHttpClientConnection.consumeInput(DefaultNHttpClientConnection.java:265) [httpcore-nio-4.4.5.jar:4.4.5]\n at org.apache.http.impl.nio.client.InternalIODispatch.onInputReady(InternalIODispatch.java:81) [httpasyncclient-4.1.2.jar:4.1.2]\n at org.apache.http.impl.nio.client.InternalIODispatch.onInputReady(InternalIODispatch.java:39) [httpasyncclient-4.1.2.jar:4.1.2]\n at org.apache.http.impl.nio.reactor.AbstractIODispatch.inputReady(AbstractIODispatch.java:114) [httpcore-nio-4.4.5.jar:4.4.5]\n at org.apache.http.impl.nio.reactor.BaseIOReactor.readable(BaseIOReactor.java:162) [httpcore-nio-4.4.5.jar:4.4.5]\n at org.apache.http.impl.nio.reactor.AbstractIOReactor.processEvent(AbstractIOReactor.java:337) [httpcore-nio-4.4.5.jar:4.4.5]\n at org.apache.http.impl.nio.reactor.AbstractIOReactor.processEvents(AbstractIOReactor.java:315) [httpcore-nio-4.4.5.jar:4.4.5]\n at org.apache.http.impl.nio.reactor.AbstractIOReactor.execute(AbstractIOReactor.java:276) [httpcore-nio-4.4.5.jar:4.4.5]\n at org.apache.http.impl.nio.reactor.BaseIOReactor.execute(BaseIOReactor.java:104) [httpcore-nio-4.4.5.jar:4.4.5]\n at org.apache.http.impl.nio.reactor.AbstractMultiworkerIOReactor$Worker.run(AbstractMultiworkerIOReactor.java:588) [httpcore-nio-4.4.5.jar:4.4.5]\n at java.lang.Thread.run(Thread.java:748) [?:1.8.0_161]\n", "offset": 10648, "prospector.type": "log", diff --git a/filebeat/module/elasticsearch/slowlog/test/test.log-expected.json b/filebeat/module/elasticsearch/slowlog/test/test.log-expected.json index c2a6ba28634..fe5ff9ecf08 100644 --- a/filebeat/module/elasticsearch/slowlog/test/test.log-expected.json +++ b/filebeat/module/elasticsearch/slowlog/test/test.log-expected.json @@ -17,6 +17,7 @@ "fileset.name": "slowlog", "input.type": "log", "log.level": "INFO", + "log.original": "[2018-06-29T10:06:14,933][INFO ][index.search.slowlog.query] [v_VJhjV] [metricbeat-6.3.0-2018.06.26][0] took[4.5ms], took_millis[4], total_hits[19435], types[], stats[], search_type[QUERY_THEN_FETCH], total_shards[1], source[{\"query\":{\"match_all\":{\"boost\":1.0}}}],", "message": "[2018-06-29T10:06:14,933][INFO ][index.search.slowlog.query] [v_VJhjV] [metricbeat-6.3.0-2018.06.26][0] took[4.5ms], took_millis[4], total_hits[19435], types[], stats[], search_type[QUERY_THEN_FETCH], total_shards[1], source[{\"query\":{\"match_all\":{\"boost\":1.0}}}],", "offset": 0, "prospector.type": "log", @@ -40,6 +41,7 @@ "fileset.name": "slowlog", "input.type": "log", "log.level": "INFO", + "log.original": "[2018-06-29T10:06:14,943][INFO ][index.search.slowlog.fetch] [v_VJhjV] [metricbeat-6.3.0-2018.06.26][0] took[10.8ms], took_millis[10], total_hits[19435], types[], stats[], search_type[QUERY_THEN_FETCH], total_shards[1], source[{\"query\":{\"match_all\":{\"boost\":1.0}}}],", "message": "[2018-06-29T10:06:14,943][INFO ][index.search.slowlog.fetch] [v_VJhjV] [metricbeat-6.3.0-2018.06.26][0] took[10.8ms], took_millis[10], total_hits[19435], types[], stats[], search_type[QUERY_THEN_FETCH], total_shards[1], source[{\"query\":{\"match_all\":{\"boost\":1.0}}}],", "offset": 265, "prospector.type": "log", @@ -63,6 +65,7 @@ "fileset.name": "slowlog", "input.type": "log", "log.level": "INFO", + "log.original": "[2018-06-29T09:01:01,821][INFO ][index.search.slowlog.query] [v_VJhjV] [metricbeat-6.3.0-2018.06.26][0] took[124.3ms], took_millis[124], total_hits[0], types[], stats[], search_type[QUERY_THEN_FETCH], total_shards[1], source[{\"size\":500,\"query\":{\"match_none\":{\"boost\":1.0}},\"version\":true,\"_source\":{\"includes\":[],\"excludes\":[]},\"stored_fields\":\"*\",\"docvalue_fields\":[\"@timestamp\",\"ceph.monitor_health.last_updated\",\"docker.container.created\",\"docker.healthcheck.event.end_date\",\"docker.healthcheck.event.start_date\",\"docker.image.created\",\"kubernetes.container.start_time\",\"kubernetes.event.metadata.timestamp.created\",\"kubernetes.node.start_time\",\"kubernetes.pod.start_time\",\"kubernetes.system.start_time\",\"mongodb.status.background_flushing.last_finished\",\"mongodb.status.local_time\",\"php_fpm.pool.start_time\",\"postgresql.activity.backend_start\",\"postgresql.activity.query_start\",\"postgresql.activity.state_change\",\"postgresql.activity.transaction_start\",\"postgresql.bgwriter.stats_reset\",\"postgresql.database.stats_reset\",\"system.process.cpu.start_time\"],\"script_fields\":{},\"sort\":[{\"@timestamp\":{\"order\":\"desc\",\"unmapped_type\":\"boolean\"}}],\"aggregations\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"time_zone\":\"Europe/Berlin\",\"interval\":\"30s\",\"offset\":0,\"order\":{\"_key\":\"asc\"},\"keyed\":false,\"min_doc_count\":1}}},\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fragment_size\":2147483647,\"fields\":{\"*\":{}}}}],", "message": "[2018-06-29T09:01:01,821][INFO ][index.search.slowlog.query] [v_VJhjV] [metricbeat-6.3.0-2018.06.26][0] took[124.3ms], took_millis[124], total_hits[0], types[], stats[], search_type[QUERY_THEN_FETCH], total_shards[1], source[{\"size\":500,\"query\":{\"match_none\":{\"boost\":1.0}},\"version\":true,\"_source\":{\"includes\":[],\"excludes\":[]},\"stored_fields\":\"*\",\"docvalue_fields\":[\"@timestamp\",\"ceph.monitor_health.last_updated\",\"docker.container.created\",\"docker.healthcheck.event.end_date\",\"docker.healthcheck.event.start_date\",\"docker.image.created\",\"kubernetes.container.start_time\",\"kubernetes.event.metadata.timestamp.created\",\"kubernetes.node.start_time\",\"kubernetes.pod.start_time\",\"kubernetes.system.start_time\",\"mongodb.status.background_flushing.last_finished\",\"mongodb.status.local_time\",\"php_fpm.pool.start_time\",\"postgresql.activity.backend_start\",\"postgresql.activity.query_start\",\"postgresql.activity.state_change\",\"postgresql.activity.transaction_start\",\"postgresql.bgwriter.stats_reset\",\"postgresql.database.stats_reset\",\"system.process.cpu.start_time\"],\"script_fields\":{},\"sort\":[{\"@timestamp\":{\"order\":\"desc\",\"unmapped_type\":\"boolean\"}}],\"aggregations\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"time_zone\":\"Europe/Berlin\",\"interval\":\"30s\",\"offset\":0,\"order\":{\"_key\":\"asc\"},\"keyed\":false,\"min_doc_count\":1}}},\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fragment_size\":2147483647,\"fields\":{\"*\":{}}}}],", "offset": 532, "prospector.type": "log", @@ -86,6 +89,7 @@ "fileset.name": "slowlog", "input.type": "log", "log.level": "INFO", + "log.original": "[2018-06-29T09:01:01,827][INFO ][index.search.slowlog.fetch] [v_VJhjV] [metricbeat-6.3.0-2018.06.26][0] took[7.2ms], took_millis[7], total_hits[0], types[], stats[], search_type[QUERY_THEN_FETCH], total_shards[1], source[{\"size\":500,\"query\":{\"match_none\":{\"boost\":1.0}},\"version\":true,\"_source\":{\"includes\":[],\"excludes\":[]},\"stored_fields\":\"*\",\"docvalue_fields\":[\"@timestamp\",\"ceph.monitor_health.last_updated\",\"docker.container.created\",\"docker.healthcheck.event.end_date\",\"docker.healthcheck.event.start_date\",\"docker.image.created\",\"kubernetes.container.start_time\",\"kubernetes.event.metadata.timestamp.created\",\"kubernetes.node.start_time\",\"kubernetes.pod.start_time\",\"kubernetes.system.start_time\",\"mongodb.status.background_flushing.last_finished\",\"mongodb.status.local_time\",\"php_fpm.pool.start_time\",\"postgresql.activity.backend_start\",\"postgresql.activity.query_start\",\"postgresql.activity.state_change\",\"postgresql.activity.transaction_start\",\"postgresql.bgwriter.stats_reset\",\"postgresql.database.stats_reset\",\"system.process.cpu.start_time\"],\"script_fields\":{},\"sort\":[{\"@timestamp\":{\"order\":\"desc\",\"unmapped_type\":\"boolean\"}}],\"aggregations\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"time_zone\":\"Europe/Berlin\",\"interval\":\"30s\",\"offset\":0,\"order\":{\"_key\":\"asc\"},\"keyed\":false,\"min_doc_count\":1}}},\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fragment_size\":2147483647,\"fields\":{\"*\":{}}}}],", "message": "[2018-06-29T09:01:01,827][INFO ][index.search.slowlog.fetch] [v_VJhjV] [metricbeat-6.3.0-2018.06.26][0] took[7.2ms], took_millis[7], total_hits[0], types[], stats[], search_type[QUERY_THEN_FETCH], total_shards[1], source[{\"size\":500,\"query\":{\"match_none\":{\"boost\":1.0}},\"version\":true,\"_source\":{\"includes\":[],\"excludes\":[]},\"stored_fields\":\"*\",\"docvalue_fields\":[\"@timestamp\",\"ceph.monitor_health.last_updated\",\"docker.container.created\",\"docker.healthcheck.event.end_date\",\"docker.healthcheck.event.start_date\",\"docker.image.created\",\"kubernetes.container.start_time\",\"kubernetes.event.metadata.timestamp.created\",\"kubernetes.node.start_time\",\"kubernetes.pod.start_time\",\"kubernetes.system.start_time\",\"mongodb.status.background_flushing.last_finished\",\"mongodb.status.local_time\",\"php_fpm.pool.start_time\",\"postgresql.activity.backend_start\",\"postgresql.activity.query_start\",\"postgresql.activity.state_change\",\"postgresql.activity.transaction_start\",\"postgresql.bgwriter.stats_reset\",\"postgresql.database.stats_reset\",\"system.process.cpu.start_time\"],\"script_fields\":{},\"sort\":[{\"@timestamp\":{\"order\":\"desc\",\"unmapped_type\":\"boolean\"}}],\"aggregations\":{\"2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"time_zone\":\"Europe/Berlin\",\"interval\":\"30s\",\"offset\":0,\"order\":{\"_key\":\"asc\"},\"keyed\":false,\"min_doc_count\":1}}},\"highlight\":{\"pre_tags\":[\"@kibana-highlighted-field@\"],\"post_tags\":[\"@/kibana-highlighted-field@\"],\"fragment_size\":2147483647,\"fields\":{\"*\":{}}}}],", "offset": 1999, "prospector.type": "log", @@ -107,6 +111,7 @@ "fileset.name": "slowlog", "input.type": "log", "log.level": "INFO", + "log.original": "[2018-07-04T13:48:07,452][INFO ][index.indexing.slowlog.index] [v_VJhjV] [metricbeat-6.3.0-2018.07.04/VLKxBLvUSYuIMKzpacGjRg] took[1.4ms], took_millis[1], type[doc], id[KUyMZWQBk9jw4gtg2y5-], routing[], source[{\"@timestamp\":\"2018-07-04T13:47:50.747Z\",\"system\":{\"process\":{\"ppid\":34526,\"state\":\"running\",\"cpu\":{\"total\":{\"value\":734879,\"pct\":0.0173,\"norm\":{\"pct\":0.0043}},\"start_time\":\"2018-07-04T06:56:34.863Z\"},\"pgid\":34526,\"cmdline\":\"/Applications/Firefox.app/Contents/MacOS/plugin-container.app/Contents/MacOS/plugin-container -childID 1 -isForBrowser -prefsLen 22119 -schedulerPrefs 0001,2 -greomni /Applications/Firefox.app/Contents/Resources/omni.ja -appomni /Applications/Firefox.app/Contents/Resources/browser/omni.ja -appdir /Applications/Firefox.app/Contents/Resources/browser -profile /Users/rado/Library/Application Support/Firefox/Profiles/pt6eoq1j.default-1484133908360 34526 gecko-crash-server-pipe.34526 org.mozilla.machname.231926932 tab\",\"name\":\"plugin-containe\",\"memory\":{\"size\":7489249280,\"rss\":{\"bytes\":567619584,\"pct\":0.033},\"share\":0},\"pid\":34528,\"username\":\"rado\"}},\"metricset\":{\"name\":\"process\",\"module\":\"system\",\"rtt\":43856},\"beat\":{\"hostname\":\"Rados-MacBook-Pro.local\",\"version\":\"6.3.0\",\"name\":\"Rados-MacBook-Pro.local\"},\"host\":{\"name\":\"Rados-MacBook-Pro.local\"}}]", "message": "[2018-07-04T13:48:07,452][INFO ][index.indexing.slowlog.index] [v_VJhjV] [metricbeat-6.3.0-2018.07.04/VLKxBLvUSYuIMKzpacGjRg] took[1.4ms], took_millis[1], type[doc], id[KUyMZWQBk9jw4gtg2y5-], routing[], source[{\"@timestamp\":\"2018-07-04T13:47:50.747Z\",\"system\":{\"process\":{\"ppid\":34526,\"state\":\"running\",\"cpu\":{\"total\":{\"value\":734879,\"pct\":0.0173,\"norm\":{\"pct\":0.0043}},\"start_time\":\"2018-07-04T06:56:34.863Z\"},\"pgid\":34526,\"cmdline\":\"/Applications/Firefox.app/Contents/MacOS/plugin-container.app/Contents/MacOS/plugin-container -childID 1 -isForBrowser -prefsLen 22119 -schedulerPrefs 0001,2 -greomni /Applications/Firefox.app/Contents/Resources/omni.ja -appomni /Applications/Firefox.app/Contents/Resources/browser/omni.ja -appdir /Applications/Firefox.app/Contents/Resources/browser -profile /Users/rado/Library/Application Support/Firefox/Profiles/pt6eoq1j.default-1484133908360 34526 gecko-crash-server-pipe.34526 org.mozilla.machname.231926932 tab\",\"name\":\"plugin-containe\",\"memory\":{\"size\":7489249280,\"rss\":{\"bytes\":567619584,\"pct\":0.033},\"share\":0},\"pid\":34528,\"username\":\"rado\"}},\"metricset\":{\"name\":\"process\",\"module\":\"system\",\"rtt\":43856},\"beat\":{\"hostname\":\"Rados-MacBook-Pro.local\",\"version\":\"6.3.0\",\"name\":\"Rados-MacBook-Pro.local\"},\"host\":{\"name\":\"Rados-MacBook-Pro.local\"}}]", "offset": 3462, "prospector.type": "log", @@ -131,6 +136,7 @@ "multiline" ], "log.level": "INFO", + "log.original": "[2018-07-04T21:51:30,411][INFO ][index.indexing.slowlog.index] [v_VJhjV] [metricbeat-6.3.0-2018.07.04/VLKxBLvUSYuIMKzpacGjRg] took[1.7ms], took_millis[1], type[doc], id[s01HZ2QBk9jw4gtgaFtn], routing[], source[\n{\n \"@timestamp\":\"2018-07-04T21:27:30.730Z\",\n \"metricset\":{\n \"name\":\"network\",\n \"module\":\"system\",\n \"rtt\":7264},\n \"system\":{\n \"network\":{\n \"name\":\"lo0\",\n \"in\":{\n \"errors\":0,\n \"dropped\":0,\n \"bytes\":77666873,\n \"packets\":244595},\n \"out\":{\n \"packets\":244595,\n \"bytes\":77666873,\n \"errors\":0,\n \"dropped\":0\n }\n }\n },\n \"beat\":{\n \"name\":\"Rados-MacBook-Pro.local\",\n \"hostname\":\"Rados-MacBook-Pro.local\",\n \"version\":\"6.3.0\"\n },\n \"host\":{\n \"name\":\"Rados-MacBook-Pro.local\"\n }\n }]", "message": "[2018-07-04T21:51:30,411][INFO ][index.indexing.slowlog.index] [v_VJhjV] [metricbeat-6.3.0-2018.07.04/VLKxBLvUSYuIMKzpacGjRg] took[1.7ms], took_millis[1], type[doc], id[s01HZ2QBk9jw4gtgaFtn], routing[], source[\n{\n \"@timestamp\":\"2018-07-04T21:27:30.730Z\",\n \"metricset\":{\n \"name\":\"network\",\n \"module\":\"system\",\n \"rtt\":7264},\n \"system\":{\n \"network\":{\n \"name\":\"lo0\",\n \"in\":{\n \"errors\":0,\n \"dropped\":0,\n \"bytes\":77666873,\n \"packets\":244595},\n \"out\":{\n \"packets\":244595,\n \"bytes\":77666873,\n \"errors\":0,\n \"dropped\":0\n }\n }\n },\n \"beat\":{\n \"name\":\"Rados-MacBook-Pro.local\",\n \"hostname\":\"Rados-MacBook-Pro.local\",\n \"version\":\"6.3.0\"\n },\n \"host\":{\n \"name\":\"Rados-MacBook-Pro.local\"\n }\n }]", "offset": 4753, "prospector.type": "log", diff --git a/filebeat/module/haproxy/log/test/haproxy.log-expected.json b/filebeat/module/haproxy/log/test/haproxy.log-expected.json index 990ec9ae854..4a0bc0add19 100644 --- a/filebeat/module/haproxy/log/test/haproxy.log-expected.json +++ b/filebeat/module/haproxy/log/test/haproxy.log-expected.json @@ -37,6 +37,7 @@ "haproxy.time_queue": 0, "haproxy.time_server_response": 0, "input.type": "log", + "log.original": "Jul 30 09:03:52 localhost haproxy[32450]: 1.2.3.4:38862 [30/Jul/2018:09:03:52.726] incoming~ docs_microservice/docs 0/0/1/0/2 304 168 - - ---- 6/6/0/0/0 0/0 {docs.example.internal||} {|||} \"GET /component---src-pages-index-js-4b15624544f97cf0bb8f.js HTTP/1.1\"", "message": "Jul 30 09:03:52 localhost haproxy[32450]: 1.2.3.4:38862 [30/Jul/2018:09:03:52.726] incoming~ docs_microservice/docs 0/0/1/0/2 304 168 - - ---- 6/6/0/0/0 0/0 {docs.example.internal||} {|||} \"GET /component---src-pages-index-js-4b15624544f97cf0bb8f.js HTTP/1.1\"", "offset": 0, "prospector.type": "log" diff --git a/filebeat/module/icinga/_meta/config.reference.yml b/filebeat/module/icinga/_meta/config.reference.yml index bbddd5bdbc6..7135d35978e 100644 --- a/filebeat/module/icinga/_meta/config.reference.yml +++ b/filebeat/module/icinga/_meta/config.reference.yml @@ -10,6 +10,8 @@ # Input configuration (advanced). Any input configuration option # can be added under this section. #input: + #Keeps the original message, so the data can be processed again on Ingest Node. + #keep_original_message: true # Debug logs #debug: @@ -22,6 +24,8 @@ # Input configuration (advanced). Any input configuration option # can be added under this section. #input: + #Keeps the original message, so the data can be processed again on Ingest Node. + #keep_original_message: true # Startup logs #startup: @@ -34,3 +38,5 @@ # Input configuration (advanced). Any input configuration option # can be added under this section. #input: + #Keeps the original message, so the data can be processed again on Ingest Node. + #keep_original_message: true diff --git a/filebeat/module/icinga/debug/test/test.log-expected.json b/filebeat/module/icinga/debug/test/test.log-expected.json index 2a8ec5dbb7d..68f48973263 100644 --- a/filebeat/module/icinga/debug/test/test.log-expected.json +++ b/filebeat/module/icinga/debug/test/test.log-expected.json @@ -7,6 +7,7 @@ "icinga.debug.message": "Add to metric list:'icinga2.demo.services.procs.procs.perfdata.procs.warn 250 1491306189'.", "icinga.debug.severity": "debug", "input.type": "log", + "log.original": "[2017-04-04 13:43:09 +0200] debug/GraphiteWriter: Add to metric list:'icinga2.demo.services.procs.procs.perfdata.procs.warn 250 1491306189'.", "offset": 0, "prospector.type": "log" }, @@ -18,6 +19,7 @@ "icinga.debug.message": "Query: UPDATE icinga_servicestatus SET acknowledgement_type = '0', active_checks_enabled = '1', check_command = 'mysql_health', check_source = 'demo', check_type = '0', current_check_attempt = '1', current_notification_number = '180', current_state = '2', endpoint_object_id = 242, event_handler = '', event_handler_enabled = '1', execution_time = '0.355594', flap_detection_enabled = '0', has_been_checked = '1', instance_id = 1, is_flapping = '0', is_reachable = '1', last_check = FROM_UNIXTIME(1491306189), last_hard_state = '2', last_hard_state_change = FROM_UNIXTIME(1491290599), last_notification = FROM_UNIXTIME(1491304989), last_state_change = FROM_UNIXTIME(1491290599), last_time_critical = FROM_UNIXTIME(1491306189), last_time_unknown = FROM_UNIXTIME(1491290589), latency = '0.001466', long_output = '', max_check_attempts = '5', next_check = FROM_UNIXTIME(1491306198), next_notification = FROM_UNIXTIME(1491306789), normal_check_interval = '0.166667', notifications_enabled = '1', original_attributes = 'null', output = 'CRITICAL - cannot connect to information_schema. Access denied for user \\'test1\\'@\\'blerims-mbp.int.netways.de\\' (using password: YES)', passive_checks_enabled = '1', percent_state_change = '0', perfdata = '', problem_has_been_acknowledged = '0', process_performance_data = '1', retry_check_interval = '0.166667', scheduled_downtime_depth = '0', service_object_id = 333, should_be_scheduled = '1', state_type = '1', status_update_time = FROM_UNIXTIME(1491306189) WHERE service_object_id = 333", "icinga.debug.severity": "debug", "input.type": "log", + "log.original": "[2017-04-04 13:43:09 +0200] debug/IdoMysqlConnection: Query: UPDATE icinga_servicestatus SET acknowledgement_type = '0', active_checks_enabled = '1', check_command = 'mysql_health', check_source = 'demo', check_type = '0', current_check_attempt = '1', current_notification_number = '180', current_state = '2', endpoint_object_id = 242, event_handler = '', event_handler_enabled = '1', execution_time = '0.355594', flap_detection_enabled = '0', has_been_checked = '1', instance_id = 1, is_flapping = '0', is_reachable = '1', last_check = FROM_UNIXTIME(1491306189), last_hard_state = '2', last_hard_state_change = FROM_UNIXTIME(1491290599), last_notification = FROM_UNIXTIME(1491304989), last_state_change = FROM_UNIXTIME(1491290599), last_time_critical = FROM_UNIXTIME(1491306189), last_time_unknown = FROM_UNIXTIME(1491290589), latency = '0.001466', long_output = '', max_check_attempts = '5', next_check = FROM_UNIXTIME(1491306198), next_notification = FROM_UNIXTIME(1491306789), normal_check_interval = '0.166667', notifications_enabled = '1', original_attributes = 'null', output = 'CRITICAL - cannot connect to information_schema. Access denied for user \\'test1\\'@\\'blerims-mbp.int.netways.de\\' (using password: YES)', passive_checks_enabled = '1', percent_state_change = '0', perfdata = '', problem_has_been_acknowledged = '0', process_performance_data = '1', retry_check_interval = '0.166667', scheduled_downtime_depth = '0', service_object_id = 333, should_be_scheduled = '1', state_type = '1', status_update_time = FROM_UNIXTIME(1491306189) WHERE service_object_id = 333", "offset": 141, "prospector.type": "log" }, @@ -29,6 +31,7 @@ "icinga.debug.message": "Running command '/usr/lib/nagios/plugins/check_ping' '-H' 'mysql.icinga.com' '-c' '5000,100%' '-w' '3000,80%': PID 8288", "icinga.debug.severity": "notice", "input.type": "log", + "log.original": "[2017-04-04 13:43:11 +0200] notice/Process: Running command '/usr/lib/nagios/plugins/check_ping' '-H' 'mysql.icinga.com' '-c' '5000,100%' '-w' '3000,80%': PID 8288", "offset": 1763, "prospector.type": "log" } diff --git a/filebeat/module/icinga/main/test/test.log-expected.json b/filebeat/module/icinga/main/test/test.log-expected.json index 59d4822ce5d..ecc24a85631 100644 --- a/filebeat/module/icinga/main/test/test.log-expected.json +++ b/filebeat/module/icinga/main/test/test.log-expected.json @@ -7,6 +7,7 @@ "icinga.main.message": "Sending 'Recovery' notification 'demo!load!mail-icingaadmin for user 'on-call'", "icinga.main.severity": "information", "input.type": "log", + "log.original": "[2017-04-04 11:16:34 +0200] information/Notification: Sending 'Recovery' notification 'demo!load!mail-icingaadmin for user 'on-call'", "offset": 0, "prospector.type": "log" }, @@ -21,6 +22,7 @@ "log.flags": [ "multiline" ], + "log.original": "[2017-04-04 11:16:34 +0200] warning/PluginNotificationTask: Notification command for object 'demo!load' (PID: 19401, arguments: '/etc/icinga2/scripts/mail-service-notification.sh') terminated with exit code 127, output: /etc/icinga2/scripts/mail-service-notification.sh: 20: /etc/icinga2/scripts/mail-service-notification.sh: mail: not found\n/usr/bin/printf: write error: Broken pipe\n", "offset": 133, "prospector.type": "log" }, @@ -32,6 +34,7 @@ "icinga.main.message": "Query queue items: 0, query rate: 5.38333/s (323/min 1610/5min 4778/15min);", "icinga.main.severity": "information", "input.type": "log", + "log.original": "[2017-04-04 11:16:48 +0200] information/IdoMysqlConnection: Query queue items: 0, query rate: 5.38333/s (323/min 1610/5min 4778/15min);", "offset": 518, "prospector.type": "log" } diff --git a/filebeat/module/icinga/startup/test/test.log-expected.json b/filebeat/module/icinga/startup/test/test.log-expected.json index 2f8cd6198c4..b385a6738cf 100644 --- a/filebeat/module/icinga/startup/test/test.log-expected.json +++ b/filebeat/module/icinga/startup/test/test.log-expected.json @@ -7,6 +7,7 @@ "icinga.startup.message": "Icinga application loader (version: r2.6.3-1)", "icinga.startup.severity": "information", "input.type": "log", + "log.original": "information/cli: Icinga application loader (version: r2.6.3-1)", "offset": 0, "prospector.type": "log" }, @@ -18,6 +19,7 @@ "icinga.startup.message": "Loading configuration file(s).", "icinga.startup.severity": "information", "input.type": "log", + "log.original": "information/cli: Loading configuration file(s).", "offset": 63, "prospector.type": "log" } diff --git a/filebeat/module/iis/_meta/config.reference.yml b/filebeat/module/iis/_meta/config.reference.yml index aebe3e38b09..042926ea067 100644 --- a/filebeat/module/iis/_meta/config.reference.yml +++ b/filebeat/module/iis/_meta/config.reference.yml @@ -10,6 +10,8 @@ # Input configuration (advanced). Any input configuration option # can be added under this section. #input: + #Keeps the original message, so the data can be processed again on Ingest Node. + #keep_original_message: true # Error logs #error: @@ -22,3 +24,5 @@ # Input configuration (advanced). Any input configuration option # can be added under this section. #input: + #Keeps the original message, so the data can be processed again on Ingest Node. + #keep_original_message: true diff --git a/filebeat/module/iis/access/test/test.log-expected.json b/filebeat/module/iis/access/test/test.log-expected.json index 2ef4c983d07..42b08568371 100644 --- a/filebeat/module/iis/access/test/test.log-expected.json +++ b/filebeat/module/iis/access/test/test.log-expected.json @@ -30,6 +30,7 @@ "iis.access.user_name": "-", "iis.access.win32_status": "0", "input.type": "log", + "log.original": "2018-01-01 08:09:10 127.0.0.1 GET / q=100 80 - 85.181.35.98 Mozilla/5.0+(Windows+NT+6.1;+Win64;+x64;+rv:57.0)+Gecko/20100101+Firefox/57.0 - 200 0 0 123", "offset": 257, "prospector.type": "log" }, @@ -61,6 +62,7 @@ "iis.access.user_name": "-", "iis.access.win32_status": "0", "input.type": "log", + "log.original": "2018-01-01 09:10:11 W3SVC1 GET / - 80 - 127.0.0.1 Mozilla/5.0+(Windows+NT+6.1;+Win64;+x64;+rv:57.0)+Gecko/20100101+Firefox/57.0 - - example.com 200 0 0 123 456 789", "offset": 709, "prospector.type": "log" }, @@ -102,6 +104,7 @@ "iis.access.user_name": "-", "iis.access.win32_status": "0", "input.type": "log", + "log.original": "2018-01-01 10:11:12 W3SVC1 MACHINE-NAME 127.0.0.1 GET / - 80 - 85.181.35.98 HTTP/1.1 Mozilla/5.0+(Windows+NT+6.1;+Win64;+x64;+rv:57.0)+Gecko/20100101+Firefox/57.0 - - example.com 200 0 0 123 456 789", "offset": 1204, "prospector.type": "log" } diff --git a/filebeat/module/iis/error/test/test.log-expected.json b/filebeat/module/iis/error/test/test.log-expected.json index ad14babac55..aaea72919f8 100644 --- a/filebeat/module/iis/error/test/test.log-expected.json +++ b/filebeat/module/iis/error/test/test.log-expected.json @@ -14,6 +14,7 @@ "iis.error.server_port": "80", "iis.error.url": "/qos/1kbfile.txt", "input.type": "log", + "log.original": "2018-01-01 08:09:10 172.31.77.6 2094 172.31.77.6 80 HTTP/1.1 GET /qos/1kbfile.txt 503 - ConnLimit -", "offset": 186, "prospector.type": "log" }, @@ -39,6 +40,7 @@ "iis.error.server_port": "80", "iis.error.url": "/ThisIsMyUrl.htm", "input.type": "log", + "log.original": "2018-01-01 09:10:11 85.181.35.98 2780 127.0.0.1 80 HTTP/1.1 GET /ThisIsMyUrl.htm 400 - Hostname -", "offset": 286, "prospector.type": "log" }, @@ -64,6 +66,7 @@ "iis.error.server_port": "80", "iis.error.url": "/", "input.type": "log", + "log.original": "2018-01-01 10:11:12 85.181.35.98 2894 127.0.0.1 80 HTTP/2.0 GET / 505 - Version_N/S -", "offset": 384, "prospector.type": "log" }, @@ -85,6 +88,7 @@ "iis.error.server_ip": "127.0.0.1", "iis.error.server_port": "80", "input.type": "log", + "log.original": "2018-01-01 11:12:13 85.181.35.98 64388 127.0.0.1 80 - - - - - Timer_MinBytesPerSecond -", "offset": 470, "prospector.type": "log" } diff --git a/filebeat/module/kafka/log/test/controller.log-expected.json b/filebeat/module/kafka/log/test/controller.log-expected.json index 698fde1e230..57829118ee6 100644 --- a/filebeat/module/kafka/log/test/controller.log-expected.json +++ b/filebeat/module/kafka/log/test/controller.log-expected.json @@ -8,6 +8,7 @@ "kafka.log.component": "controller-event-thread", "kafka.log.level": "INFO", "kafka.log.message": "Starting", + "log.original": "[2017-08-04 10:48:21,048] INFO [controller-event-thread]: Starting (kafka.controller.ControllerEventManager$ControllerEventThread)", "message": "[2017-08-04 10:48:21,048] INFO [controller-event-thread]: Starting (kafka.controller.ControllerEventManager$ControllerEventThread)", "offset": 0, "prospector.type": "log" @@ -21,6 +22,7 @@ "kafka.log.component": "Controller 0", "kafka.log.level": "INFO", "kafka.log.message": "0 successfully elected as the controller", + "log.original": "[2017-08-04 10:48:21,063] INFO [Controller 0]: 0 successfully elected as the controller (kafka.controller.KafkaController)", "message": "[2017-08-04 10:48:21,063] INFO [Controller 0]: 0 successfully elected as the controller (kafka.controller.KafkaController)", "offset": 131, "prospector.type": "log" @@ -34,6 +36,7 @@ "kafka.log.component": "Controller 0", "kafka.log.level": "INFO", "kafka.log.message": "Broker 0 starting become controller state transition", + "log.original": "[2017-08-04 10:48:21,064] INFO [Controller 0]: Broker 0 starting become controller state transition (kafka.controller.KafkaController)", "message": "[2017-08-04 10:48:21,064] INFO [Controller 0]: Broker 0 starting become controller state transition (kafka.controller.KafkaController)", "offset": 254, "prospector.type": "log" @@ -47,6 +50,7 @@ "kafka.log.component": "Controller 0", "kafka.log.level": "INFO", "kafka.log.message": "Controller 0 incremented epoch to 1", + "log.original": "[2017-08-04 10:48:21,082] INFO [Controller 0]: Controller 0 incremented epoch to 1 (kafka.controller.KafkaController)", "message": "[2017-08-04 10:48:21,082] INFO [Controller 0]: Controller 0 incremented epoch to 1 (kafka.controller.KafkaController)", "offset": 389, "prospector.type": "log" @@ -60,6 +64,7 @@ "kafka.log.component": "Controller 0", "kafka.log.level": "DEBUG", "kafka.log.message": "Registering IsrChangeNotificationListener", + "log.original": "[2017-08-04 10:48:21,085] DEBUG [Controller 0]: Registering IsrChangeNotificationListener (kafka.controller.KafkaController)", "message": "[2017-08-04 10:48:21,085] DEBUG [Controller 0]: Registering IsrChangeNotificationListener (kafka.controller.KafkaController)", "offset": 507, "prospector.type": "log" @@ -73,6 +78,7 @@ "kafka.log.component": "Replica state machine on controller 0", "kafka.log.level": "INFO", "kafka.log.message": "Started replica state machine with initial state -> Map()", + "log.original": "[2017-08-04 10:48:21,154] INFO [Replica state machine on controller 0]: Started replica state machine with initial state -> Map() (kafka.controller.ReplicaStateMachine)", "message": "[2017-08-04 10:48:21,154] INFO [Replica state machine on controller 0]: Started replica state machine with initial state -> Map() (kafka.controller.ReplicaStateMachine)", "offset": 632, "prospector.type": "log" @@ -86,6 +92,7 @@ "kafka.log.component": "Partition state machine on Controller 0", "kafka.log.level": "INFO", "kafka.log.message": "Started partition state machine with initial state -> Map()", + "log.original": "[2017-08-04 10:48:21,156] INFO [Partition state machine on Controller 0]: Started partition state machine with initial state -> Map() (kafka.controller.PartitionStateMachine)", "message": "[2017-08-04 10:48:21,156] INFO [Partition state machine on Controller 0]: Started partition state machine with initial state -> Map() (kafka.controller.PartitionStateMachine)", "offset": 801, "prospector.type": "log" @@ -99,6 +106,7 @@ "kafka.log.component": "Controller 0", "kafka.log.level": "INFO", "kafka.log.message": "Broker 0 is ready to serve as the new controller with epoch 1", + "log.original": "[2017-08-04 10:48:21,157] INFO [Controller 0]: Broker 0 is ready to serve as the new controller with epoch 1 (kafka.controller.KafkaController)", "message": "[2017-08-04 10:48:21,157] INFO [Controller 0]: Broker 0 is ready to serve as the new controller with epoch 1 (kafka.controller.KafkaController)", "offset": 976, "prospector.type": "log" @@ -112,6 +120,7 @@ "kafka.log.component": "Partition state machine on Controller 0", "kafka.log.level": "INFO", "kafka.log.message": "Invoking state change to OnlinePartition for partitions ", + "log.original": "[2017-08-04 10:48:21,165] INFO [Partition state machine on Controller 0]: Invoking state change to OnlinePartition for partitions (kafka.controller.PartitionStateMachine)", "message": "[2017-08-04 10:48:21,165] INFO [Partition state machine on Controller 0]: Invoking state change to OnlinePartition for partitions (kafka.controller.PartitionStateMachine)", "offset": 1120, "prospector.type": "log" @@ -125,6 +134,7 @@ "kafka.log.component": "Controller 0", "kafka.log.level": "DEBUG", "kafka.log.message": "Live brokers: ", + "log.original": "[2017-08-04 11:44:22,588] DEBUG [Controller 0]: Live brokers: (kafka.controller.KafkaController)", "message": "[2017-08-04 11:44:22,588] DEBUG [Controller 0]: Live brokers: (kafka.controller.KafkaController)", "offset": 1292, "prospector.type": "log" @@ -138,6 +148,7 @@ "kafka.log.component": "controller-event-thread", "kafka.log.level": "INFO", "kafka.log.message": "Shutting down", + "log.original": "[2017-08-04 11:44:25,094] INFO [controller-event-thread]: Shutting down (kafka.controller.ControllerEventManager$ControllerEventThread)", "message": "[2017-08-04 11:44:25,094] INFO [controller-event-thread]: Shutting down (kafka.controller.ControllerEventManager$ControllerEventThread)", "offset": 1390, "prospector.type": "log" @@ -151,6 +162,7 @@ "kafka.log.component": "controller-event-thread", "kafka.log.level": "INFO", "kafka.log.message": "Stopped", + "log.original": "[2017-08-04 11:44:25,095] INFO [controller-event-thread]: Stopped (kafka.controller.ControllerEventManager$ControllerEventThread)", "message": "[2017-08-04 11:44:25,095] INFO [controller-event-thread]: Stopped (kafka.controller.ControllerEventManager$ControllerEventThread)", "offset": 1526, "prospector.type": "log" @@ -164,6 +176,7 @@ "kafka.log.component": "controller-event-thread", "kafka.log.level": "INFO", "kafka.log.message": "Shutdown completed", + "log.original": "[2017-08-04 11:44:25,097] INFO [controller-event-thread]: Shutdown completed (kafka.controller.ControllerEventManager$ControllerEventThread)", "message": "[2017-08-04 11:44:25,097] INFO [controller-event-thread]: Shutdown completed (kafka.controller.ControllerEventManager$ControllerEventThread)", "offset": 1656, "prospector.type": "log" @@ -177,6 +190,7 @@ "kafka.log.component": "Controller 0", "kafka.log.level": "DEBUG", "kafka.log.message": "Controller resigning, broker id 0", + "log.original": "[2017-08-04 11:44:25,099] DEBUG [Controller 0]: Controller resigning, broker id 0 (kafka.controller.KafkaController)", "message": "[2017-08-04 11:44:25,099] DEBUG [Controller 0]: Controller resigning, broker id 0 (kafka.controller.KafkaController)", "offset": 1797, "prospector.type": "log" @@ -190,6 +204,7 @@ "kafka.log.component": "Controller 0", "kafka.log.level": "DEBUG", "kafka.log.message": "De-registering IsrChangeNotificationListener", + "log.original": "[2017-08-04 11:44:25,100] DEBUG [Controller 0]: De-registering IsrChangeNotificationListener (kafka.controller.KafkaController)", "message": "[2017-08-04 11:44:25,100] DEBUG [Controller 0]: De-registering IsrChangeNotificationListener (kafka.controller.KafkaController)", "offset": 1914, "prospector.type": "log" @@ -203,6 +218,7 @@ "kafka.log.component": "Partition state machine on Controller 0", "kafka.log.level": "INFO", "kafka.log.message": "Stopped partition state machine", + "log.original": "[2017-08-04 11:44:25,105] INFO [Partition state machine on Controller 0]: Stopped partition state machine (kafka.controller.PartitionStateMachine)", "message": "[2017-08-04 11:44:25,105] INFO [Partition state machine on Controller 0]: Stopped partition state machine (kafka.controller.PartitionStateMachine)", "offset": 2042, "prospector.type": "log" @@ -216,6 +232,7 @@ "kafka.log.component": "Replica state machine on controller 0", "kafka.log.level": "INFO", "kafka.log.message": "Stopped replica state machine", + "log.original": "[2017-08-04 11:44:25,111] INFO [Replica state machine on controller 0]: Stopped replica state machine (kafka.controller.ReplicaStateMachine)", "message": "[2017-08-04 11:44:25,111] INFO [Replica state machine on controller 0]: Stopped replica state machine (kafka.controller.ReplicaStateMachine)", "offset": 2189, "prospector.type": "log" @@ -229,6 +246,7 @@ "kafka.log.component": "Controller-0-to-broker-0-send-thread", "kafka.log.level": "INFO", "kafka.log.message": "Shutting down", + "log.original": "[2017-08-04 11:44:25,112] INFO [Controller-0-to-broker-0-send-thread]: Shutting down (kafka.controller.RequestSendThread)", "message": "[2017-08-04 11:44:25,112] INFO [Controller-0-to-broker-0-send-thread]: Shutting down (kafka.controller.RequestSendThread)", "offset": 2330, "prospector.type": "log" @@ -242,6 +260,7 @@ "kafka.log.component": "Controller-0-to-broker-0-send-thread", "kafka.log.level": "INFO", "kafka.log.message": "Stopped", + "log.original": "[2017-08-04 11:44:25,112] INFO [Controller-0-to-broker-0-send-thread]: Stopped (kafka.controller.RequestSendThread)", "message": "[2017-08-04 11:44:25,112] INFO [Controller-0-to-broker-0-send-thread]: Stopped (kafka.controller.RequestSendThread)", "offset": 2452, "prospector.type": "log" @@ -255,6 +274,7 @@ "kafka.log.component": "Controller-0-to-broker-0-send-thread", "kafka.log.level": "INFO", "kafka.log.message": "Shutdown completed", + "log.original": "[2017-08-04 11:44:25,113] INFO [Controller-0-to-broker-0-send-thread]: Shutdown completed (kafka.controller.RequestSendThread)", "message": "[2017-08-04 11:44:25,113] INFO [Controller-0-to-broker-0-send-thread]: Shutdown completed (kafka.controller.RequestSendThread)", "offset": 2568, "prospector.type": "log" diff --git a/filebeat/module/kafka/log/test/server.log-expected.json b/filebeat/module/kafka/log/test/server.log-expected.json index 15b904ad343..9637092e141 100644 --- a/filebeat/module/kafka/log/test/server.log-expected.json +++ b/filebeat/module/kafka/log/test/server.log-expected.json @@ -8,6 +8,7 @@ "kafka.log.component": "unknown", "kafka.log.level": "INFO", "kafka.log.message": "starting", + "log.original": "[2017-08-04 10:48:20,377] INFO starting (kafka.server.KafkaServer)", "message": "[2017-08-04 10:48:20,377] INFO starting (kafka.server.KafkaServer)", "offset": 0, "prospector.type": "log" @@ -21,6 +22,7 @@ "kafka.log.component": "unknown", "kafka.log.level": "INFO", "kafka.log.message": "Connecting to zookeeper on localhost:2181", + "log.original": "[2017-08-04 10:48:20,379] INFO Connecting to zookeeper on localhost:2181 (kafka.server.KafkaServer)", "message": "[2017-08-04 10:48:20,379] INFO Connecting to zookeeper on localhost:2181 (kafka.server.KafkaServer)", "offset": 67, "prospector.type": "log" @@ -34,6 +36,7 @@ "kafka.log.component": "unknown", "kafka.log.level": "INFO", "kafka.log.message": "Client environment:java.io.tmpdir=/tmp", + "log.original": "[2017-08-04 10:48:20,400] INFO Client environment:java.io.tmpdir=/tmp (org.apache.zookeeper.ZooKeeper)", "message": "[2017-08-04 10:48:20,400] INFO Client environment:java.io.tmpdir=/tmp (org.apache.zookeeper.ZooKeeper)", "offset": 167, "prospector.type": "log" @@ -47,6 +50,7 @@ "kafka.log.component": "unknown", "kafka.log.level": "INFO", "kafka.log.message": "Client environment:java.compiler=", + "log.original": "[2017-08-04 10:48:20,400] INFO Client environment:java.compiler= (org.apache.zookeeper.ZooKeeper)", "message": "[2017-08-04 10:48:20,400] INFO Client environment:java.compiler= (org.apache.zookeeper.ZooKeeper)", "offset": 270, "prospector.type": "log" @@ -60,6 +64,7 @@ "kafka.log.component": "unknown", "kafka.log.level": "INFO", "kafka.log.message": "Initiating client connection, connectString=localhost:2181 sessionTimeout=6000 watcher=org.I0Itec.zkclient.ZkClient@5ffead27", + "log.original": "[2017-08-04 10:48:20,401] INFO Initiating client connection, connectString=localhost:2181 sessionTimeout=6000 watcher=org.I0Itec.zkclient.ZkClient@5ffead27 (org.apache.zookeeper.ZooKeeper)", "message": "[2017-08-04 10:48:20,401] INFO Initiating client connection, connectString=localhost:2181 sessionTimeout=6000 watcher=org.I0Itec.zkclient.ZkClient@5ffead27 (org.apache.zookeeper.ZooKeeper)", "offset": 372, "prospector.type": "log" @@ -73,6 +78,7 @@ "kafka.log.component": "unknown", "kafka.log.level": "INFO", "kafka.log.message": "Waiting for keeper state SyncConnected", + "log.original": "[2017-08-04 10:48:20,413] INFO Waiting for keeper state SyncConnected (org.I0Itec.zkclient.ZkClient)", "message": "[2017-08-04 10:48:20,413] INFO Waiting for keeper state SyncConnected (org.I0Itec.zkclient.ZkClient)", "offset": 561, "prospector.type": "log" @@ -86,6 +92,7 @@ "kafka.log.component": "unknown", "kafka.log.level": "INFO", "kafka.log.message": "Opening socket connection to server localhost/0:0:0:0:0:0:0:1:2181. Will not attempt to authenticate using SASL (unknown error)", + "log.original": "[2017-08-04 10:48:20,415] INFO Opening socket connection to server localhost/0:0:0:0:0:0:0:1:2181. Will not attempt to authenticate using SASL (unknown error) (org.apache.zookeeper.ClientCnxn)", "message": "[2017-08-04 10:48:20,415] INFO Opening socket connection to server localhost/0:0:0:0:0:0:0:1:2181. Will not attempt to authenticate using SASL (unknown error) (org.apache.zookeeper.ClientCnxn)", "offset": 662, "prospector.type": "log" @@ -99,6 +106,7 @@ "kafka.log.component": "unknown", "kafka.log.level": "INFO", "kafka.log.message": "Socket connection established to localhost/0:0:0:0:0:0:0:1:2181, initiating session", + "log.original": "[2017-08-04 10:48:20,420] INFO Socket connection established to localhost/0:0:0:0:0:0:0:1:2181, initiating session (org.apache.zookeeper.ClientCnxn)", "message": "[2017-08-04 10:48:20,420] INFO Socket connection established to localhost/0:0:0:0:0:0:0:1:2181, initiating session (org.apache.zookeeper.ClientCnxn)", "offset": 855, "prospector.type": "log" @@ -112,6 +120,7 @@ "kafka.log.component": "unknown", "kafka.log.level": "INFO", "kafka.log.message": "Session establishment complete on server localhost/0:0:0:0:0:0:0:1:2181, sessionid = 0x15dabf8d4140000, negotiated timeout = 6000", + "log.original": "[2017-08-04 10:48:20,457] INFO Session establishment complete on server localhost/0:0:0:0:0:0:0:1:2181, sessionid = 0x15dabf8d4140000, negotiated timeout = 6000 (org.apache.zookeeper.ClientCnxn)", "message": "[2017-08-04 10:48:20,457] INFO Session establishment complete on server localhost/0:0:0:0:0:0:0:1:2181, sessionid = 0x15dabf8d4140000, negotiated timeout = 6000 (org.apache.zookeeper.ClientCnxn)", "offset": 1004, "prospector.type": "log" @@ -125,6 +134,7 @@ "kafka.log.component": "unknown", "kafka.log.level": "INFO", "kafka.log.message": "zookeeper state changed (SyncConnected)", + "log.original": "[2017-08-04 10:48:20,458] INFO zookeeper state changed (SyncConnected) (org.I0Itec.zkclient.ZkClient)", "message": "[2017-08-04 10:48:20,458] INFO zookeeper state changed (SyncConnected) (org.I0Itec.zkclient.ZkClient)", "offset": 1199, "prospector.type": "log" @@ -138,6 +148,7 @@ "kafka.log.component": "unknown", "kafka.log.level": "WARN", "kafka.log.message": "No meta.properties file under dir /tmp/kafka-logs/meta.properties", + "log.original": "[2017-08-04 10:48:20,748] WARN No meta.properties file under dir /tmp/kafka-logs/meta.properties (kafka.server.BrokerMetadataCheckpoint)", "message": "[2017-08-04 10:48:20,748] WARN No meta.properties file under dir /tmp/kafka-logs/meta.properties (kafka.server.BrokerMetadataCheckpoint)", "offset": 1301, "prospector.type": "log" @@ -151,6 +162,7 @@ "kafka.log.component": "ThrottledRequestReaper-Fetch", "kafka.log.level": "INFO", "kafka.log.message": "Starting", + "log.original": "[2017-08-04 10:48:20,800] INFO [ThrottledRequestReaper-Fetch]: Starting (kafka.server.ClientQuotaManager$ThrottledRequestReaper)", "message": "[2017-08-04 10:48:20,800] INFO [ThrottledRequestReaper-Fetch]: Starting (kafka.server.ClientQuotaManager$ThrottledRequestReaper)", "offset": 1438, "prospector.type": "log" @@ -164,6 +176,7 @@ "kafka.log.component": "unknown", "kafka.log.level": "INFO", "kafka.log.message": "Log directory '/tmp/kafka-logs' not found, creating it.", + "log.original": "[2017-08-04 10:48:20,866] INFO Log directory '/tmp/kafka-logs' not found, creating it. (kafka.log.LogManager)", "message": "[2017-08-04 10:48:20,866] INFO Log directory '/tmp/kafka-logs' not found, creating it. (kafka.log.LogManager)", "offset": 1567, "prospector.type": "log" @@ -177,6 +190,7 @@ "kafka.log.component": "unknown", "kafka.log.level": "INFO", "kafka.log.message": "Loading logs.", + "log.original": "[2017-08-04 10:48:20,873] INFO Loading logs. (kafka.log.LogManager)", "message": "[2017-08-04 10:48:20,873] INFO Loading logs. (kafka.log.LogManager)", "offset": 1677, "prospector.type": "log" @@ -190,6 +204,7 @@ "kafka.log.component": "ExpirationReaper-0-Heartbeat", "kafka.log.level": "INFO", "kafka.log.message": "Starting", + "log.original": "[2017-08-04 10:48:21,062] INFO [ExpirationReaper-0-Heartbeat]: Starting (kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper)", "message": "[2017-08-04 10:48:21,062] INFO [ExpirationReaper-0-Heartbeat]: Starting (kafka.server.DelayedOperationPurgatory$ExpiredOperationReaper)", "offset": 1745, "prospector.type": "log" @@ -203,6 +218,7 @@ "kafka.log.component": "unknown", "kafka.log.level": "INFO", "kafka.log.message": "Result of znode creation is: OK", + "log.original": "[2017-08-04 10:48:21,063] INFO Result of znode creation is: OK (kafka.utils.ZKCheckedEphemeral)", "message": "[2017-08-04 10:48:21,063] INFO Result of znode creation is: OK (kafka.utils.ZKCheckedEphemeral)", "offset": 1881, "prospector.type": "log" @@ -216,6 +232,7 @@ "kafka.log.component": "Group Metadata Manager on Broker 0", "kafka.log.level": "INFO", "kafka.log.message": "Removed 0 expired offsets in 1 milliseconds.", + "log.original": "[2017-08-04 10:48:21,095] INFO [Group Metadata Manager on Broker 0]: Removed 0 expired offsets in 1 milliseconds. (kafka.coordinator.group.GroupMetadataManager)", "message": "[2017-08-04 10:48:21,095] INFO [Group Metadata Manager on Broker 0]: Removed 0 expired offsets in 1 milliseconds. (kafka.coordinator.group.GroupMetadataManager)", "offset": 1977, "prospector.type": "log" @@ -229,6 +246,7 @@ "kafka.log.component": "ProducerId Manager 0", "kafka.log.level": "INFO", "kafka.log.message": "Acquired new producerId block (brokerId:0,blockStartProducerId:0,blockEndProducerId:999) by writing to Zk with path version 1", + "log.original": "[2017-08-04 10:48:21,127] INFO [ProducerId Manager 0]: Acquired new producerId block (brokerId:0,blockStartProducerId:0,blockEndProducerId:999) by writing to Zk with path version 1 (kafka.coordinator.transaction.ProducerIdManager)", "message": "[2017-08-04 10:48:21,127] INFO [ProducerId Manager 0]: Acquired new producerId block (brokerId:0,blockStartProducerId:0,blockEndProducerId:999) by writing to Zk with path version 1 (kafka.coordinator.transaction.ProducerIdManager)", "offset": 2138, "prospector.type": "log" @@ -242,6 +260,7 @@ "kafka.log.component": "Transaction Coordinator 0", "kafka.log.level": "INFO", "kafka.log.message": "Starting up.", + "log.original": "[2017-08-04 10:48:21,162] INFO [Transaction Coordinator 0]: Starting up. (kafka.coordinator.transaction.TransactionCoordinator)", "message": "[2017-08-04 10:48:21,162] INFO [Transaction Coordinator 0]: Starting up. (kafka.coordinator.transaction.TransactionCoordinator)", "offset": 2369, "prospector.type": "log" @@ -255,6 +274,7 @@ "kafka.log.component": "Transaction Marker Channel Manager 0", "kafka.log.level": "INFO", "kafka.log.message": "Starting", + "log.original": "[2017-08-04 10:48:21,167] INFO [Transaction Marker Channel Manager 0]: Starting (kafka.coordinator.transaction.TransactionMarkerChannelManager)", "message": "[2017-08-04 10:48:21,167] INFO [Transaction Marker Channel Manager 0]: Starting (kafka.coordinator.transaction.TransactionMarkerChannelManager)", "offset": 2497, "prospector.type": "log" diff --git a/filebeat/module/kafka/log/test/state-change-1.1.0.log-expected.json b/filebeat/module/kafka/log/test/state-change-1.1.0.log-expected.json index be011b17d23..e278bb8d83e 100644 --- a/filebeat/module/kafka/log/test/state-change-1.1.0.log-expected.json +++ b/filebeat/module/kafka/log/test/state-change-1.1.0.log-expected.json @@ -8,6 +8,7 @@ "kafka.log.component": "Broker id=30", "kafka.log.level": "TRACE", "kafka.log.message": "Cached leader info PartitionState(controllerEpoch=25, leader=-1, leaderEpoch=15, isr=[10], zkVersion=15, replicas=[10], offlineReplicas=[10]) for partition __consumer_offsets-16 in response to UpdateMetadata request sent by controller 20 epoch 25 with correlation id 8", + "log.original": "[2018-07-16 10:17:06,489] TRACE [Broker id=30] Cached leader info PartitionState(controllerEpoch=25, leader=-1, leaderEpoch=15, isr=[10], zkVersion=15, replicas=[10], offlineReplicas=[10]) for partition __consumer_offsets-16 in response to UpdateMetadata request sent by controller 20 epoch 25 with correlation id 8 (state.change.logger)", "message": "[2018-07-16 10:17:06,489] TRACE [Broker id=30] Cached leader info PartitionState(controllerEpoch=25, leader=-1, leaderEpoch=15, isr=[10], zkVersion=15, replicas=[10], offlineReplicas=[10]) for partition __consumer_offsets-16 in response to UpdateMetadata request sent by controller 20 epoch 25 with correlation id 8 (state.change.logger)", "offset": 0, "prospector.type": "log" diff --git a/filebeat/module/kafka/log/test/state-change.log-expected.json b/filebeat/module/kafka/log/test/state-change.log-expected.json index f6c4112aa1a..c1de02dead2 100644 --- a/filebeat/module/kafka/log/test/state-change.log-expected.json +++ b/filebeat/module/kafka/log/test/state-change.log-expected.json @@ -8,6 +8,7 @@ "kafka.log.component": "unknown", "kafka.log.level": "TRACE", "kafka.log.message": "Controller 0 epoch 1 received response {error_code=0} for a request sent to broker baldur:9092 (id: 0 rack: null)", + "log.original": "[2017-08-04 10:48:21,428] TRACE Controller 0 epoch 1 received response {error_code=0} for a request sent to broker baldur:9092 (id: 0 rack: null) (state.change.logger)", "message": "[2017-08-04 10:48:21,428] TRACE Controller 0 epoch 1 received response {error_code=0} for a request sent to broker baldur:9092 (id: 0 rack: null) (state.change.logger)", "offset": 0, "prospector.type": "log" diff --git a/filebeat/module/logstash/log/test/logstash-plain.log-expected.json b/filebeat/module/logstash/log/test/logstash-plain.log-expected.json index 2157da6003d..05d84e58264 100644 --- a/filebeat/module/logstash/log/test/logstash-plain.log-expected.json +++ b/filebeat/module/logstash/log/test/logstash-plain.log-expected.json @@ -4,6 +4,7 @@ "fileset.module": "logstash", "fileset.name": "log", "input.type": "log", + "log.original": "[2017-10-23T14:20:12,046][INFO ][logstash.modules.scaffold] Initializing module {:module_name=>\"fb_apache\", :directory=>\"/usr/share/logstash/modules/fb_apache/configuration\"}", "logstash.log.level": "INFO", "logstash.log.message": "Initializing module {:module_name=>\"fb_apache\", :directory=>\"/usr/share/logstash/modules/fb_apache/configuration\"}", "logstash.log.module": "logstash.modules.scaffold", diff --git a/filebeat/module/logstash/slowlog/test/slowlog-plain.log-expected.json b/filebeat/module/logstash/slowlog/test/slowlog-plain.log-expected.json index 835106bf975..b8949ecd31a 100644 --- a/filebeat/module/logstash/slowlog/test/slowlog-plain.log-expected.json +++ b/filebeat/module/logstash/slowlog/test/slowlog-plain.log-expected.json @@ -4,6 +4,7 @@ "fileset.module": "logstash", "fileset.name": "slowlog", "input.type": "log", + "log.original": "[2017-10-30T09:57:58,243][WARN ][slowlog.logstash.filters.sleep] event processing time {:plugin_params=>{\"time\"=>3, \"id\"=>\"e4e12a4e3082615c5427079bf4250dbfa338ebac10f8ea9912d7b98a14f56b8c\"}, :took_in_nanos=>3027675106, :took_in_millis=>3027, :event=>\"{\\\"@version\\\":\\\"1\\\",\\\"@timestamp\\\":\\\"2017-10-30T13:57:55.130Z\\\",\\\"host\\\":\\\"sashimi\\\",\\\"sequence\\\":0,\\\"message\\\":\\\"Hello world!\\\"}\"}", "logstash.slowlog.event": "\"{\\\"@version\\\":\\\"1\\\",\\\"@timestamp\\\":\\\"2017-10-30T13:57:55.130Z\\\",\\\"host\\\":\\\"sashimi\\\",\\\"sequence\\\":0,\\\"message\\\":\\\"Hello world!\\\"}\"", "logstash.slowlog.level": "WARN", "logstash.slowlog.message": "event processing time {:plugin_params=>{\"time\"=>3, \"id\"=>\"e4e12a4e3082615c5427079bf4250dbfa338ebac10f8ea9912d7b98a14f56b8c\"}, :took_in_nanos=>3027675106, :took_in_millis=>3027, :event=>\"{\\\"@version\\\":\\\"1\\\",\\\"@timestamp\\\":\\\"2017-10-30T13:57:55.130Z\\\",\\\"host\\\":\\\"sashimi\\\",\\\"sequence\\\":0,\\\"message\\\":\\\"Hello world!\\\"}\"}", diff --git a/filebeat/module/mongodb/_meta/config.reference.yml b/filebeat/module/mongodb/_meta/config.reference.yml index 86f1511ec35..615dc97e012 100644 --- a/filebeat/module/mongodb/_meta/config.reference.yml +++ b/filebeat/module/mongodb/_meta/config.reference.yml @@ -10,3 +10,5 @@ # Input configuration (advanced). Any input configuration option # can be added under this section. #input: + #Keeps the original message, so the data can be processed again on Ingest Node. + #keep_original_message: true diff --git a/filebeat/module/mongodb/log/test/mongodb-debian-3.2.11.log-expected.json b/filebeat/module/mongodb/log/test/mongodb-debian-3.2.11.log-expected.json index f0cd77888db..0b20762f993 100644 --- a/filebeat/module/mongodb/log/test/mongodb-debian-3.2.11.log-expected.json +++ b/filebeat/module/mongodb/log/test/mongodb-debian-3.2.11.log-expected.json @@ -4,6 +4,7 @@ "fileset.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.original": "2018-02-05T13:44:56.657+0100 I CONTROL [initandlisten] git version: 009580ad490190ba33d1c6253ebd8d91808923e4", "mongodb.log.component": "CONTROL", "mongodb.log.context": "initandlisten", "mongodb.log.message": "git version: 009580ad490190ba33d1c6253ebd8d91808923e4", @@ -16,6 +17,7 @@ "fileset.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.original": "2018-02-05T13:44:56.657+0100 I CONTROL [initandlisten] modules: none", "mongodb.log.component": "CONTROL", "mongodb.log.context": "initandlisten", "mongodb.log.message": "modules: none", @@ -28,6 +30,7 @@ "fileset.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.original": "2018-02-05T13:44:56.657+0100 I CONTROL [initandlisten] OpenSSL version: OpenSSL 1.0.2l 25 May 2017", "mongodb.log.component": "CONTROL", "mongodb.log.context": "initandlisten", "mongodb.log.message": "OpenSSL version: OpenSSL 1.0.2l 25 May 2017", @@ -40,6 +43,7 @@ "fileset.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.original": "2018-02-05T13:44:56.677+0100 I STORAGE [initandlisten] wiredtiger_open config: create,cache_size=8G,session_max=20000,eviction=(threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),file_manager=(close_idle_time=100000),checkpoint=(wait=60,log_size=2GB),statistics_log=(wait=0),", "mongodb.log.component": "STORAGE", "mongodb.log.context": "initandlisten", "mongodb.log.message": "wiredtiger_open config: create,cache_size=8G,session_max=20000,eviction=(threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,archive=true,path=journal,compressor=snappy),file_manager=(close_idle_time=100000),checkpoint=(wait=60,log_size=2GB),statistics_log=(wait=0),", @@ -52,6 +56,7 @@ "fileset.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.original": "2018-02-05T13:44:56.724+0100 I FTDC [initandlisten] Initializing full-time diagnostic data capture with directory '/var/lib/mongodb/diagnostic.data'", "mongodb.log.component": "FTDC", "mongodb.log.context": "initandlisten", "mongodb.log.message": "Initializing full-time diagnostic data capture with directory '/var/lib/mongodb/diagnostic.data'", @@ -64,6 +69,7 @@ "fileset.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.original": "2018-02-05T13:44:56.724+0100 I NETWORK [HostnameCanonicalizationWorker] Starting hostname canonicalization worker", "mongodb.log.component": "NETWORK", "mongodb.log.context": "HostnameCanonicalizationWorker", "mongodb.log.message": "Starting hostname canonicalization worker", @@ -76,6 +82,7 @@ "fileset.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.original": "2018-02-05T13:44:56.744+0100 I NETWORK [initandlisten] waiting for connections on port 27017", "mongodb.log.component": "NETWORK", "mongodb.log.context": "initandlisten", "mongodb.log.message": "waiting for connections on port 27017", @@ -88,6 +95,7 @@ "fileset.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.original": "2018-02-05T13:50:55.170+0100 I NETWORK [conn1] end connection 127.0.0.1:55404 (0 connections now open)", "mongodb.log.component": "NETWORK", "mongodb.log.context": "conn1", "mongodb.log.message": "end connection 127.0.0.1:55404 (0 connections now open)", @@ -100,6 +108,7 @@ "fileset.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.original": "2018-02-05T13:50:55.487+0100 I NETWORK [initandlisten] connection accepted from 127.0.0.1:55406 #2 (1 connection now open)", "mongodb.log.component": "NETWORK", "mongodb.log.context": "initandlisten", "mongodb.log.message": "connection accepted from 127.0.0.1:55406 #2 (1 connection now open)", @@ -112,6 +121,7 @@ "fileset.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.original": "2018-02-05T14:49:45.606+0100 I CONTROL [signalProcessingThread] now exiting", "mongodb.log.component": "CONTROL", "mongodb.log.context": "signalProcessingThread", "mongodb.log.message": "now exiting", @@ -124,6 +134,7 @@ "fileset.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.original": "2018-02-05T14:49:45.606+0100 I NETWORK [signalProcessingThread] closing listening socket: 7", "mongodb.log.component": "NETWORK", "mongodb.log.context": "signalProcessingThread", "mongodb.log.message": "closing listening socket: 7", @@ -136,6 +147,7 @@ "fileset.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.original": "2018-02-05T14:49:45.606+0100 I NETWORK [signalProcessingThread] removing socket file: /run/mongodb/mongodb-27017.sock", "mongodb.log.component": "NETWORK", "mongodb.log.context": "signalProcessingThread", "mongodb.log.message": "removing socket file: /run/mongodb/mongodb-27017.sock", @@ -148,6 +160,7 @@ "fileset.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.original": "2018-02-05T14:49:45.606+0100 I NETWORK [signalProcessingThread] shutdown: going to flush diaglog...", "mongodb.log.component": "NETWORK", "mongodb.log.context": "signalProcessingThread", "mongodb.log.message": "shutdown: going to flush diaglog...", @@ -160,6 +173,7 @@ "fileset.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.original": "2018-02-05T14:49:45.606+0100 I NETWORK [signalProcessingThread] shutdown: going to close sockets...", "mongodb.log.component": "NETWORK", "mongodb.log.context": "signalProcessingThread", "mongodb.log.message": "shutdown: going to close sockets...", @@ -172,6 +186,7 @@ "fileset.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.original": "2018-02-05T14:49:45.688+0100 I STORAGE [signalProcessingThread] shutdown: removing fs lock...", "mongodb.log.component": "STORAGE", "mongodb.log.context": "signalProcessingThread", "mongodb.log.message": "shutdown: removing fs lock...", @@ -184,6 +199,7 @@ "fileset.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.original": "2018-02-05T13:44:56.657+0100 I CONTROL [initandlisten] db version v3.2.11", "mongodb.log.component": "CONTROL", "mongodb.log.context": "initandlisten", "mongodb.log.message": "db version v3.2.11", @@ -196,6 +212,7 @@ "fileset.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.original": "2018-02-05T13:44:56.657+0100 I CONTROL [initandlisten] build environment:", "mongodb.log.component": "CONTROL", "mongodb.log.context": "initandlisten", "mongodb.log.message": "build environment:", @@ -208,6 +225,7 @@ "fileset.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.original": "2018-02-05T13:44:56.657+0100 I CONTROL [initandlisten] distarch: x86_64", "mongodb.log.component": "CONTROL", "mongodb.log.context": "initandlisten", "mongodb.log.message": " distarch: x86_64", @@ -220,6 +238,7 @@ "fileset.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.original": "2018-02-05T13:44:56.657+0100 I CONTROL [initandlisten] options: { config: \"/etc/mongodb.conf\", net: { bindIp: \"127.0.0.1\", unixDomainSocket: { pathPrefix: \"/run/mongodb\" } }, storage: { dbPath: \"/var/lib/mongodb\", journal: { enabled: true } }, systemLog: { destination: \"file\", logAppend: true, path: \"/var/log/mongodb/mongodb.log\" } }", "mongodb.log.component": "CONTROL", "mongodb.log.context": "initandlisten", "mongodb.log.message": "options: { config: \"/etc/mongodb.conf\", net: { bindIp: \"127.0.0.1\", unixDomainSocket: { pathPrefix: \"/run/mongodb\" } }, storage: { dbPath: \"/var/lib/mongodb\", journal: { enabled: true } }, systemLog: { destination: \"file\", logAppend: true, path: \"/var/log/mongodb/mongodb.log\" } }", @@ -232,6 +251,7 @@ "fileset.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.original": "2018-02-05T13:50:55.170+0100 I NETWORK [initandlisten] connection accepted from 127.0.0.1:55404 #1 (1 connection now open)", "mongodb.log.component": "NETWORK", "mongodb.log.context": "initandlisten", "mongodb.log.message": "connection accepted from 127.0.0.1:55404 #1 (1 connection now open)", @@ -244,6 +264,7 @@ "fileset.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.original": "2018-02-05T13:50:56.180+0100 I NETWORK [conn3] end connection 127.0.0.1:55414 (0 connections now open)", "mongodb.log.component": "NETWORK", "mongodb.log.context": "conn3", "mongodb.log.message": "end connection 127.0.0.1:55414 (0 connections now open)", @@ -256,6 +277,7 @@ "fileset.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.original": "2018-02-05T14:15:42.095+0100 I NETWORK [conn4] end connection 127.0.0.1:58336 (0 connections now open)", "mongodb.log.component": "NETWORK", "mongodb.log.context": "conn4", "mongodb.log.message": "end connection 127.0.0.1:58336 (0 connections now open)", @@ -268,6 +290,7 @@ "fileset.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.original": "2018-02-05T14:49:45.606+0100 I NETWORK [signalProcessingThread] shutdown: going to close listening sockets...", "mongodb.log.component": "NETWORK", "mongodb.log.context": "signalProcessingThread", "mongodb.log.message": "shutdown: going to close listening sockets...", @@ -280,6 +303,7 @@ "fileset.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.original": "2018-02-05T14:49:45.606+0100 I STORAGE [signalProcessingThread] WiredTigerKVEngine shutting down", "mongodb.log.component": "STORAGE", "mongodb.log.context": "signalProcessingThread", "mongodb.log.message": "WiredTigerKVEngine shutting down", @@ -292,6 +316,7 @@ "fileset.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.original": "2018-02-05T14:49:45.688+0100 I CONTROL [signalProcessingThread] dbexit: rc: 0", "mongodb.log.component": "CONTROL", "mongodb.log.context": "signalProcessingThread", "mongodb.log.message": "dbexit: rc: 0", @@ -304,6 +329,7 @@ "fileset.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.original": "2018-02-05T13:44:56.657+0100 I CONTROL [initandlisten] MongoDB starting : pid=29803 port=27017 dbpath=/var/lib/mongodb 64-bit host=sleipnir", "mongodb.log.component": "CONTROL", "mongodb.log.context": "initandlisten", "mongodb.log.message": "MongoDB starting : pid=29803 port=27017 dbpath=/var/lib/mongodb 64-bit host=sleipnir", @@ -316,6 +342,7 @@ "fileset.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.original": "2018-02-05T13:44:56.657+0100 I CONTROL [initandlisten] allocator: tcmalloc", "mongodb.log.component": "CONTROL", "mongodb.log.context": "initandlisten", "mongodb.log.message": "allocator: tcmalloc", @@ -328,6 +355,7 @@ "fileset.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.original": "2018-02-05T13:44:56.657+0100 I CONTROL [initandlisten] target_arch: x86_64", "mongodb.log.component": "CONTROL", "mongodb.log.context": "initandlisten", "mongodb.log.message": " target_arch: x86_64", @@ -340,6 +368,7 @@ "fileset.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.original": "2018-02-05T13:50:55.487+0100 I NETWORK [conn2] end connection 127.0.0.1:55406 (0 connections now open)", "mongodb.log.component": "NETWORK", "mongodb.log.context": "conn2", "mongodb.log.message": "end connection 127.0.0.1:55406 (0 connections now open)", @@ -352,6 +381,7 @@ "fileset.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.original": "2018-02-05T13:50:56.180+0100 I NETWORK [initandlisten] connection accepted from 127.0.0.1:55414 #3 (1 connection now open)", "mongodb.log.component": "NETWORK", "mongodb.log.context": "initandlisten", "mongodb.log.message": "connection accepted from 127.0.0.1:55414 #3 (1 connection now open)", @@ -364,6 +394,7 @@ "fileset.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.original": "2018-02-05T14:11:41.401+0100 I NETWORK [initandlisten] connection accepted from 127.0.0.1:58336 #4 (1 connection now open)", "mongodb.log.component": "NETWORK", "mongodb.log.context": "initandlisten", "mongodb.log.message": "connection accepted from 127.0.0.1:58336 #4 (1 connection now open)", @@ -376,6 +407,7 @@ "fileset.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.original": "2018-02-05T14:49:45.605+0100 I CONTROL [signalProcessingThread] got signal 15 (Terminated), will terminate after current cmd ends", "mongodb.log.component": "CONTROL", "mongodb.log.context": "signalProcessingThread", "mongodb.log.message": "got signal 15 (Terminated), will terminate after current cmd ends", @@ -388,6 +420,7 @@ "fileset.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.original": "2018-02-05T14:49:45.605+0100 I FTDC [signalProcessingThread] Shutting down full-time diagnostic data capture", "mongodb.log.component": "FTDC", "mongodb.log.context": "signalProcessingThread", "mongodb.log.message": "Shutting down full-time diagnostic data capture", @@ -400,6 +433,7 @@ "fileset.module": "mongodb", "fileset.name": "log", "input.type": "log", + "log.original": "2018-02-05T14:49:45.606+0100 I NETWORK [signalProcessingThread] closing listening socket: 6", "mongodb.log.component": "NETWORK", "mongodb.log.context": "signalProcessingThread", "mongodb.log.message": "closing listening socket: 6", diff --git a/filebeat/module/mysql/_meta/config.reference.yml b/filebeat/module/mysql/_meta/config.reference.yml index 49f1db5e72b..e0615ca9204 100644 --- a/filebeat/module/mysql/_meta/config.reference.yml +++ b/filebeat/module/mysql/_meta/config.reference.yml @@ -10,6 +10,8 @@ # Input configuration (advanced). Any input configuration option # can be added under this section. #input: + #Keeps the original message, so the data can be processed again on Ingest Node. + #keep_original_message: true # Slow logs #slowlog: @@ -22,3 +24,5 @@ # Input configuration (advanced). Any input configuration option # can be added under this section. #input: + #Keeps the original message, so the data can be processed again on Ingest Node. + #keep_original_message: true diff --git a/filebeat/module/nginx/_meta/config.reference.yml b/filebeat/module/nginx/_meta/config.reference.yml index 572341217e6..080420be06d 100644 --- a/filebeat/module/nginx/_meta/config.reference.yml +++ b/filebeat/module/nginx/_meta/config.reference.yml @@ -10,6 +10,8 @@ # Input configuration (advanced). Any input configuration option # can be added under this section. #input: + #Keeps the original message, so the data can be processed again on Ingest Node. + #keep_original_message: true # Error logs #error: @@ -22,3 +24,5 @@ # Input configuration (advanced). Any input configuration option # can be added under this section. #input: + #Keeps the original message, so the data can be processed again on Ingest Node. + #keep_original_message: true diff --git a/filebeat/module/nginx/access/test/test.log-expected.json b/filebeat/module/nginx/access/test/test.log-expected.json index 0b7cc707111..946290c8e89 100644 --- a/filebeat/module/nginx/access/test/test.log-expected.json +++ b/filebeat/module/nginx/access/test/test.log-expected.json @@ -4,6 +4,7 @@ "fileset.module": "nginx", "fileset.name": "access", "input.type": "log", + "log.original": "10.0.0.2, 10.0.0.1, 127.0.0.1 - - [07/Dec/2016:11:05:07 +0100] \"GET /ocelot HTTP/1.1\" 200 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"", "nginx.access.body_sent.bytes": "571", "nginx.access.http_version": "1.1", "nginx.access.method": "GET", @@ -34,6 +35,7 @@ "fileset.module": "nginx", "fileset.name": "access", "input.type": "log", + "log.original": "172.17.0.1 - - [29/May/2017:19:02:48 +0000] \"GET /stringpatch HTTP/1.1\" 404 612 \"-\" \"Mozilla/5.0 (Windows NT 6.1; rv:15.0) Gecko/20120716 Firefox/15.0a2\" \"-\"", "nginx.access.body_sent.bytes": "612", "nginx.access.http_version": "1.1", "nginx.access.method": "GET", @@ -61,6 +63,7 @@ "fileset.module": "nginx", "fileset.name": "access", "input.type": "log", + "log.original": "10.0.0.2, 10.0.0.1, 85.181.35.98 - - [07/Dec/2016:11:05:07 +0100] \"GET /ocelot HTTP/1.1\" 200 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"", "nginx.access.body_sent.bytes": "571", "nginx.access.geoip.city_name": "Berlin", "nginx.access.geoip.continent_name": "Europe", @@ -98,6 +101,7 @@ "fileset.module": "nginx", "fileset.name": "access", "input.type": "log", + "log.original": "85.181.35.98 - - [07/Dec/2016:11:05:07 +0100] \"GET /ocelot HTTP/1.1\" 200 571 \"-\" \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.12; rv:49.0) Gecko/20100101 Firefox/49.0\"", "nginx.access.body_sent.bytes": "571", "nginx.access.geoip.city_name": "Berlin", "nginx.access.geoip.continent_name": "Europe", @@ -133,6 +137,7 @@ "fileset.module": "nginx", "fileset.name": "access", "input.type": "log", + "log.original": "\"10.5.102.222, 199.96.1.1, 204.246.1.1\" 10.2.1.185 - - [22/Jan/2016:13:18:29 +0000] \"GET /assets/xxxx?q=100 HTTP/1.1\" 200 25507 \"-\" \"Amazon CloudFront\"", "nginx.access.body_sent.bytes": "25507", "nginx.access.geoip.city_name": "Springfield", "nginx.access.geoip.continent_name": "North America", @@ -167,6 +172,7 @@ "fileset.module": "nginx", "fileset.name": "access", "input.type": "log", + "log.original": "2a03:0000:10ff:f00f:0000:0000:0:8000, 10.225.192.17 10.2.2.121 - - [30/Dec/2016:06:47:09 +0000] \"GET /test.html HTTP/1.1\" 404 8571 \"-\" \"Mozilla/5.0 (compatible; Facebot 1.0; https://developers.facebook.com/docs/sharing/webmasters/crawler)\"", "nginx.access.body_sent.bytes": "8571", "nginx.access.geoip.continent_name": "Europe", "nginx.access.geoip.country_iso_code": "PT", @@ -199,6 +205,7 @@ "fileset.module": "nginx", "fileset.name": "access", "input.type": "log", + "log.original": "127.0.0.1 - - [12/Apr/2018:09:48:40 +0200] \"\" 400 0 \"-\" \"-\"", "nginx.access.body_sent.bytes": "0", "nginx.access.referrer": "-", "nginx.access.remote_ip": "127.0.0.1", diff --git a/filebeat/module/postgresql/_meta/config.reference.yml b/filebeat/module/postgresql/_meta/config.reference.yml index e1deee0e25c..3b0f394d12b 100644 --- a/filebeat/module/postgresql/_meta/config.reference.yml +++ b/filebeat/module/postgresql/_meta/config.reference.yml @@ -10,3 +10,5 @@ # Input configuration (advanced). Any input configuration option # can be added under this section. #input: + #Keeps the original message, so the data can be processed again on Ingest Node. + #keep_original_message: true diff --git a/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json b/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json index b17481ca76b..47eae945f66 100644 --- a/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json +++ b/filebeat/module/postgresql/log/test/postgresql-9.6-debian-with-slowlog.log-expected.json @@ -4,6 +4,7 @@ "fileset.module": "postgresql", "fileset.name": "log", "input.type": "log", + "log.original": "2017-07-31 13:36:42.585 CEST [4974] LOG: database system was shut down at 2017-06-17 16:58:04 CEST", "message": "2017-07-31 13:36:42.585 CEST [4974] LOG: database system was shut down at 2017-06-17 16:58:04 CEST", "offset": 0, "postgresql.log.level": "LOG", @@ -18,6 +19,7 @@ "fileset.module": "postgresql", "fileset.name": "log", "input.type": "log", + "log.original": "2017-07-31 13:36:42.605 CEST [4974] LOG: MultiXact member wraparound protections are now enabled", "message": "2017-07-31 13:36:42.605 CEST [4974] LOG: MultiXact member wraparound protections are now enabled", "offset": 100, "postgresql.log.level": "LOG", @@ -32,6 +34,7 @@ "fileset.module": "postgresql", "fileset.name": "log", "input.type": "log", + "log.original": "2017-07-31 13:36:42.615 CEST [4978] LOG: autovacuum launcher started", "message": "2017-07-31 13:36:42.615 CEST [4978] LOG: autovacuum launcher started", "offset": 198, "postgresql.log.level": "LOG", @@ -46,6 +49,7 @@ "fileset.module": "postgresql", "fileset.name": "log", "input.type": "log", + "log.original": "2017-07-31 13:36:42.616 CEST [4973] LOG: database system is ready to accept connections", "message": "2017-07-31 13:36:42.616 CEST [4973] LOG: database system is ready to accept connections", "offset": 268, "postgresql.log.level": "LOG", @@ -60,6 +64,7 @@ "fileset.module": "postgresql", "fileset.name": "log", "input.type": "log", + "log.original": "2017-07-31 13:36:42.956 CEST [4980] [unknown]@[unknown] LOG: incomplete startup packet", "message": "2017-07-31 13:36:42.956 CEST [4980] [unknown]@[unknown] LOG: incomplete startup packet", "offset": 357, "postgresql.log.database": "unknown", @@ -79,6 +84,7 @@ "log.flags": [ "multiline" ], + "log.original": "2017-07-31 13:36:43.557 CEST [4983] postgres@postgres LOG: duration: 37.118 ms statement: SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", "message": "2017-07-31 13:36:43.557 CEST [4983] postgres@postgres LOG: duration: 37.118 ms statement: SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", "offset": 445, "postgresql.log.database": "postgres", @@ -99,6 +105,7 @@ "log.flags": [ "multiline" ], + "log.original": "2017-07-31 13:36:44.104 CEST [4986] postgres@postgres LOG: duration: 2.895 ms statement: SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", "message": "2017-07-31 13:36:44.104 CEST [4986] postgres@postgres LOG: duration: 2.895 ms statement: SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", "offset": 873, "postgresql.log.database": "postgres", @@ -119,6 +126,7 @@ "log.flags": [ "multiline" ], + "log.original": "2017-07-31 13:36:44.642 CEST [4989] postgres@postgres LOG: duration: 2.809 ms statement: SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", "message": "2017-07-31 13:36:44.642 CEST [4989] postgres@postgres LOG: duration: 2.809 ms statement: SELECT d.datname as \"Name\",\n\t pg_catalog.pg_get_userbyid(d.datdba) as \"Owner\",\n\t pg_catalog.pg_encoding_to_char(d.encoding) as \"Encoding\",\n\t d.datcollate as \"Collate\",\n\t d.datctype as \"Ctype\",\n\t pg_catalog.array_to_string(d.datacl, E'\\n') AS \"Access privileges\"\n\tFROM pg_catalog.pg_database d\n\tORDER BY 1;", "offset": 1300, "postgresql.log.database": "postgres", @@ -136,6 +144,7 @@ "fileset.module": "postgresql", "fileset.name": "log", "input.type": "log", + "log.original": "2017-07-31 13:39:16.249 CEST [5407] postgres@users FATAL: database \"users\" does not exist", "message": "2017-07-31 13:39:16.249 CEST [5407] postgres@users FATAL: database \"users\" does not exist", "offset": 1727, "postgresql.log.database": "users", @@ -152,6 +161,7 @@ "fileset.module": "postgresql", "fileset.name": "log", "input.type": "log", + "log.original": "2017-07-31 13:39:17.945 CEST [5500] postgres@user FATAL: database \"user\" does not exist", "message": "2017-07-31 13:39:17.945 CEST [5500] postgres@user FATAL: database \"user\" does not exist", "offset": 1818, "postgresql.log.database": "user", @@ -171,6 +181,7 @@ "log.flags": [ "multiline" ], + "log.original": "2017-07-31 13:39:21.025 CEST [5404] postgres@postgres LOG: duration: 37.598 ms statement: SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", "message": "2017-07-31 13:39:21.025 CEST [5404] postgres@postgres LOG: duration: 37.598 ms statement: SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", "offset": 1907, "postgresql.log.database": "postgres", @@ -188,6 +199,7 @@ "fileset.module": "postgresql", "fileset.name": "log", "input.type": "log", + "log.original": "2017-07-31 13:39:31.619 CEST [5502] postgres@clients LOG: duration: 9.482 ms statement: select * from clients;", "message": "2017-07-31 13:39:31.619 CEST [5502] postgres@clients LOG: duration: 9.482 ms statement: select * from clients;", "offset": 2620, "postgresql.log.database": "clients", @@ -205,6 +217,7 @@ "fileset.module": "postgresql", "fileset.name": "log", "input.type": "log", + "log.original": "2017-07-31 13:39:40.147 CEST [5502] postgres@clients LOG: duration: 0.765 ms statement: select id from clients;", "message": "2017-07-31 13:39:40.147 CEST [5502] postgres@clients LOG: duration: 0.765 ms statement: select id from clients;", "offset": 2733, "postgresql.log.database": "clients", @@ -225,6 +238,7 @@ "log.flags": [ "multiline" ], + "log.original": "2017-07-31 13:40:54.310 CEST [5502] postgres@clients LOG: duration: 26.082 ms statement: SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", "message": "2017-07-31 13:40:54.310 CEST [5502] postgres@clients LOG: duration: 26.082 ms statement: SELECT n.nspname as \"Schema\",\n\t c.relname as \"Name\",\n\t CASE c.relkind WHEN 'r' THEN 'table' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' WHEN 'i' THEN 'index' WHEN 'S' THEN 'sequence' WHEN 's' THEN 'special' WHEN 'f' THEN 'foreign table' END as \"Type\",\n\t pg_catalog.pg_get_userbyid(c.relowner) as \"Owner\"\n\tFROM pg_catalog.pg_class c\n\t LEFT JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace\n\tWHERE c.relkind IN ('r','')\n\t AND n.nspname <> 'pg_catalog'\n\t AND n.nspname <> 'information_schema'\n\t AND n.nspname !~ '^pg_toast'\n\t AND pg_catalog.pg_table_is_visible(c.oid)\n\tORDER BY 1,2;", "offset": 2847, "postgresql.log.database": "clients", @@ -242,6 +256,7 @@ "fileset.module": "postgresql", "fileset.name": "log", "input.type": "log", + "log.original": "2017-07-31 13:43:22.645 CEST [5502] postgres@clients LOG: duration: 36.162 ms statement: create table cats(name varchar(50) primary key, toy varchar (50) not null, born timestamp not null);", "message": "2017-07-31 13:43:22.645 CEST [5502] postgres@clients LOG: duration: 36.162 ms statement: create table cats(name varchar(50) primary key, toy varchar (50) not null, born timestamp not null);", "offset": 3559, "postgresql.log.database": "clients", @@ -259,6 +274,7 @@ "fileset.module": "postgresql", "fileset.name": "log", "input.type": "log", + "log.original": "2017-07-31 13:46:02.670 CEST [5502] postgres@c$lients LOG: duration: 10.540 ms statement: insert into cats(name, toy, born) values('kate', 'ball', now());", "message": "2017-07-31 13:46:02.670 CEST [5502] postgres@c$lients LOG: duration: 10.540 ms statement: insert into cats(name, toy, born) values('kate', 'ball', now());", "offset": 3751, "postgresql.log.database": "c$lients", @@ -276,6 +292,7 @@ "fileset.module": "postgresql", "fileset.name": "log", "input.type": "log", + "log.original": "2017-07-31 13:46:23.016 CEST [5502] postgres@_clients$db LOG: duration: 5.156 ms statement: insert into cats(name, toy, born) values('frida', 'horse', now());", "message": "2017-07-31 13:46:23.016 CEST [5502] postgres@_clients$db LOG: duration: 5.156 ms statement: insert into cats(name, toy, born) values('frida', 'horse', now());", "offset": 3908, "postgresql.log.database": "_clients$db", @@ -293,6 +310,7 @@ "fileset.module": "postgresql", "fileset.name": "log", "input.type": "log", + "log.original": "2017-07-31 13:46:55.637 CEST [5502] postgres@clients_db LOG: duration: 25.871 ms statement: create table dogs(name varchar(50) primary key, owner varchar (50) not null, born timestamp not null);", "message": "2017-07-31 13:46:55.637 CEST [5502] postgres@clients_db LOG: duration: 25.871 ms statement: create table dogs(name varchar(50) primary key, owner varchar (50) not null, born timestamp not null);", "offset": 4069, "postgresql.log.database": "clients_db", diff --git a/filebeat/module/redis/log/test/test.log-expected.json b/filebeat/module/redis/log/test/test.log-expected.json index 3fd7f8f3454..654bce00928 100644 --- a/filebeat/module/redis/log/test/test.log-expected.json +++ b/filebeat/module/redis/log/test/test.log-expected.json @@ -4,6 +4,7 @@ "fileset.module": "redis", "fileset.name": "log", "input.type": "log", + "log.original": "98738:M 30 May 12:23:52.442 * Saving the final RDB snapshot before exiting.", "offset": 0, "prospector.type": "log", "redis.log.level": "notice", @@ -16,6 +17,7 @@ "fileset.module": "redis", "fileset.name": "log", "input.type": "log", + "log.original": "30 May 10:05:20 . 0 clients connected (0 slaves), 618932 bytes in use, 0 shared objects.", "offset": 76, "prospector.type": "log", "redis.log.level": "debug", @@ -26,6 +28,7 @@ "fileset.module": "redis", "fileset.name": "log", "input.type": "log", + "log.original": "[2932] 31 May 04:32:08 * The server is now ready to accept connections on port 6379\"", "offset": 165, "prospector.type": "log", "redis.log.level": "notice", @@ -36,6 +39,7 @@ "fileset.module": "redis", "fileset.name": "log", "input.type": "log", + "log.original": "5092:signal-handler (1496141844) Received SIGINT scheduling shutdown...", "offset": 250, "prospector.type": "log", "redis.log.message": "Received SIGINT scheduling shutdown...", diff --git a/filebeat/module/system/_meta/config.reference.yml b/filebeat/module/system/_meta/config.reference.yml index b4121ca8081..24bb3d77be7 100644 --- a/filebeat/module/system/_meta/config.reference.yml +++ b/filebeat/module/system/_meta/config.reference.yml @@ -13,6 +13,8 @@ # Input configuration (advanced). Any input configuration option # can be added under this section. #input: + #Keeps the original message, so the data can be processed again on Ingest Node. + #keep_original_message: true # Authorization logs #auth: @@ -28,3 +30,5 @@ # Input configuration (advanced). Any input configuration option # can be added under this section. #input: + #Keeps the original message, so the data can be processed again on Ingest Node. + #keep_original_message: true diff --git a/filebeat/module/system/auth/test/test.log-expected.json b/filebeat/module/system/auth/test/test.log-expected.json index c99cbeb2fa3..cd75299b6f8 100644 --- a/filebeat/module/system/auth/test/test.log-expected.json +++ b/filebeat/module/system/auth/test/test.log-expected.json @@ -4,6 +4,7 @@ "fileset.module": "system", "fileset.name": "auth", "input.type": "log", + "log.original": "Feb 21 21:54:44 localhost sshd[3402]: Accepted publickey for vagrant from 10.0.2.2 port 63673 ssh2: RSA 39:33:99:e9:a0:dc:f2:33:a3:e5:72:3b:7c:3a:56:84", "offset": 0, "prospector.type": "log", "system.auth.hostname": "localhost", @@ -21,6 +22,7 @@ "fileset.module": "system", "fileset.name": "auth", "input.type": "log", + "log.original": "Feb 23 00:13:35 localhost sshd[7483]: Accepted password for vagrant from 192.168.33.1 port 58803 ssh2", "offset": 152, "prospector.type": "log", "system.auth.hostname": "localhost", @@ -37,6 +39,7 @@ "fileset.module": "system", "fileset.name": "auth", "input.type": "log", + "log.original": "Feb 21 21:56:12 localhost sshd[3430]: Invalid user test from 10.0.2.2", "offset": 254, "prospector.type": "log", "system.auth.hostname": "localhost", @@ -51,6 +54,7 @@ "fileset.module": "system", "fileset.name": "auth", "input.type": "log", + "log.original": "Feb 20 08:35:22 slave22 sshd[5774]: Failed password for root from 116.31.116.24 port 29160 ssh2", "offset": 324, "prospector.type": "log", "system.auth.hostname": "slave22", @@ -73,6 +77,7 @@ "fileset.module": "system", "fileset.name": "auth", "input.type": "log", + "log.original": "Feb 21 23:35:33 localhost sudo: vagrant : TTY=pts/0 ; PWD=/home/vagrant ; USER=root ; COMMAND=/bin/ls", "offset": 420, "prospector.type": "log", "system.auth.hostname": "localhost", @@ -88,6 +93,7 @@ "fileset.module": "system", "fileset.name": "auth", "input.type": "log", + "log.original": "Feb 19 15:30:04 slave22 sshd[18406]: Did not receive identification string from 123.57.245.163", "offset": 522, "prospector.type": "log", "system.auth.hostname": "slave22", @@ -100,6 +106,7 @@ "fileset.module": "system", "fileset.name": "auth", "input.type": "log", + "log.original": "Feb 23 00:08:48 localhost sudo: vagrant : TTY=pts/1 ; PWD=/home/vagrant ; USER=root ; COMMAND=/bin/cat /var/log/secure", "offset": 617, "prospector.type": "log", "system.auth.hostname": "localhost", @@ -115,6 +122,7 @@ "fileset.module": "system", "fileset.name": "auth", "input.type": "log", + "log.original": "Feb 24 00:13:02 precise32 sudo: tsg : user NOT in sudoers ; TTY=pts/1 ; PWD=/home/vagrant ; USER=root ; COMMAND=/bin/ls", "offset": 736, "prospector.type": "log", "system.auth.hostname": "precise32", @@ -131,6 +139,7 @@ "fileset.module": "system", "fileset.name": "auth", "input.type": "log", + "log.original": "Feb 22 11:47:05 localhost groupadd[6991]: new group: name=apache, GID=48", "offset": 861, "prospector.type": "log", "system.auth.groupadd.gid": "48", @@ -144,6 +153,7 @@ "fileset.module": "system", "fileset.name": "auth", "input.type": "log", + "log.original": "Feb 22 11:47:05 localhost useradd[6995]: new user: name=apache, UID=48, GID=48, home=/usr/share/httpd, shell=/sbin/nologin", "offset": 934, "prospector.type": "log", "system.auth.hostname": "localhost", diff --git a/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json b/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json index 4d667d28a17..b34206989f4 100644 --- a/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json +++ b/filebeat/module/system/syslog/test/darwin-syslog-sample.log-expected.json @@ -7,6 +7,7 @@ "log.flags": [ "multiline" ], + "log.original": "Dec 13 11:35:28 a-mac-with-esc-key GoogleSoftwareUpdateAgent[21412]: 2016-12-13 11:35:28.420 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSAgentApp updateProductWithProductID:usingEngine:] Checking for updates for \"All Products\" using engine \n\t\t>>\n\t\tprocessor=\n\t\t\tisProcessing=NO actionsCompleted=0 progress=0.00\n\t\t\terrors=0 currentActionErrors=0\n\t\t\tevents=0 currentActionEvents=0\n\t\t\tactionQueue=( )\n\t\t>\n\t\tdelegate=(null)\n\t\tserverInfoStore=(null)\n\t\terrors=0\n\t>", "offset": 0, "prospector.type": "log", "system.syslog.hostname": "a-mac-with-esc-key", @@ -20,6 +21,7 @@ "fileset.module": "system", "fileset.name": "syslog", "input.type": "log", + "log.original": "Dec 13 11:35:28 a-mac-with-esc-key GoogleSoftwareUpdateAgent[21412]: 2016-12-13 11:35:28.421 GoogleSoftwareUpdateAgent[21412/0x700007399000] [lvl=2] -[KSUpdateEngine updateAllExceptProduct:] KSUpdateEngine updating all installed products, except:'com.google.Keystone'.", "offset": 907, "prospector.type": "log", "system.syslog.hostname": "a-mac-with-esc-key", @@ -33,6 +35,7 @@ "fileset.module": "system", "fileset.name": "syslog", "input.type": "log", + "log.original": "Apr 4 03:39:57 --- last message repeated 1 time ---", "offset": 1176, "prospector.type": "log", "system.syslog.message": "--- last message repeated 1 time ---", diff --git a/filebeat/module/traefik/_meta/config.reference.yml b/filebeat/module/traefik/_meta/config.reference.yml index e800f73557c..e5a722132a0 100644 --- a/filebeat/module/traefik/_meta/config.reference.yml +++ b/filebeat/module/traefik/_meta/config.reference.yml @@ -10,3 +10,5 @@ # Input configuration (advanced). Any input configuration option # can be added under this section. #input: + #Keeps the original message, so the data can be processed again on Ingest Node. + #keep_original_message: true diff --git a/filebeat/module/traefik/access/test/test.log-expected.json b/filebeat/module/traefik/access/test/test.log-expected.json index 5d9df6d2854..fb07bc96c6d 100644 --- a/filebeat/module/traefik/access/test/test.log-expected.json +++ b/filebeat/module/traefik/access/test/test.log-expected.json @@ -4,6 +4,7 @@ "fileset.module": "traefik", "fileset.name": "access", "input.type": "log", + "log.original": "192.168.33.1 - - [02/Oct/2017:20:22:07 +0000] \"GET /ui/favicons/favicon-16x16.png HTTP/1.1\" 304 0 \"http://example.com/login\" \"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36\" 262 \"Host-host-1\" \"http://172.19.0.3:5601\" 2ms", "offset": 0, "prospector.type": "log", "traefik.access.body_sent.bytes": "0", @@ -28,6 +29,7 @@ "fileset.module": "traefik", "fileset.name": "access", "input.type": "log", + "log.original": "85.181.35.98 - - [02/Oct/2017:20:22:08 +0000] \"GET /ui/favicons/favicon.ico HTTP/1.1\" 304 0 \"http://example.com/login\" \"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36\" 271 \"Host-host1 \"http://172.19.0.3:5601\" 3ms", "offset": 280, "prospector.type": "log", "traefik.access.body_sent.bytes": "0", diff --git a/libbeat/beat/pipeline.go b/libbeat/beat/pipeline.go index 823ae8de20b..de7d26535fc 100644 --- a/libbeat/beat/pipeline.go +++ b/libbeat/beat/pipeline.go @@ -54,6 +54,9 @@ type ClientConfig struct { // DynamicFields provides additional fields to be added to every event, supporting live updates DynamicFields *common.MapStrPointer + // KeepOriginalMsg determines whether an outgoing event needs to include the original log message. + KeepOriginalMsg bool + // Processors passes additional processor to the client, to be executed before // the pipeline processors. Processor ProcessorList diff --git a/libbeat/publisher/pipeline/processor.go b/libbeat/publisher/pipeline/processor.go index e2dcb7c73c0..a7943ca6029 100644 --- a/libbeat/publisher/pipeline/processor.go +++ b/libbeat/publisher/pipeline/processor.go @@ -48,14 +48,15 @@ type processorFn struct { // // 1. (P) generalize/normalize event // 2. (C) add Meta from client Config to event.Meta -// 3. (C) add Fields from client config to event.Fields -// 4. (P) add pipeline fields + tags -// 5. (C) add client fields + tags -// 6. (C) client processors list -// 7. (P) add beats metadata -// 8. (P) pipeline processors list -// 9. (P) (if publish/debug enabled) log event -// 10. (P) (if output disabled) dropEvent +// 3. (P) copy contents of message to `log.original` +// 4. (C) add Fields from client config to event.Fields +// 5. (P) add pipeline fields + tags +// 6. (C) add client fields + tags +// 7. (C) client processors list +// 8. (P) add beats metadata +// 9. (P) pipeline processors list +// 10. (P) (if publish/debug enabled) log event +// 11. (P) (if output disabled) dropEvent func newProcessorPipeline( info beat.Info, global pipelineProcessors, @@ -82,6 +83,11 @@ func newProcessorPipeline( processors.add(clientEventMeta(m, needsCopy)) } + if config.KeepOriginalMsg { + // setup 3: keep original message + processors.add(keepOriginalMsgProcessor) + } + // setup 4, 5: pipeline tags + client tags var tags []string tags = append(tags, global.tags...) @@ -217,6 +223,17 @@ var dropDisabledProcessor = newProcessor("dropDisabled", func(event *beat.Event) return nil, nil }) +var keepOriginalMsgProcessor = newProcessor("keepOriginalMsgEvent", func(event *beat.Event) (*beat.Event, error) { + // skip event if there is no message + original, ok := event.Fields["message"] + if !ok { + return event, nil + } + + event.PutValue("log.original", original) + return event, nil +}) + func beatAnnotateProcessor(beatMeta common.MapStr) *processorFn { const key = "beat" return newAnnotateProcessor("annotateBeat", func(event *beat.Event) {