Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Filebeat] Removing duplicate modules (aliases) Observability #27919

Merged
merged 8 commits into from
Sep 16, 2021
Merged

[Filebeat] Removing duplicate modules (aliases) Observability #27919

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
aa2f2dd
[Filebeat] Removing duplicate modules (aliases) Observability
P1llus Sep 14, 2021
57b3361
update changelog
P1llus Sep 14, 2021
40b0498
removing more references and fixing some old wrong documentation
P1llus Sep 14, 2021
9f6c8c7
reverting regex test
P1llus Sep 15, 2021
8b122c9
Merge branch 'master' into remove_duplicate_observability_modules
P1llus Sep 15, 2021
b5ef5d6
merge master
P1llus Sep 16, 2021
1ce3ec8
Merge branch 'remove_duplicate_observability_modules' of https://gith…
P1llus Sep 16, 2021
be8906e
merge conflict
P1llus Sep 16, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.next.asciidoc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
- Add option for S3 input to work without SQS notification {issue}18205[18205] {pull}27332[27332]
- Fix Crowdstrike ingest pipeline that was creating flattened `process` fields. {issue}27622[27622] {pull}27623[27623]
- Rename `log.path` to `log.file.path` in filestream to be consistent with `log` input and ECS. {pull}27761[27761]
- Removes old module aliases for `googlecloud` (moved to gcp) and `apache2` (moved to apache). {pull}27919[27919]
- Only filesets that are explicitly configured will be enabled. {issue}17256[17256] {pull}27526[27526]
- All filesets are disabled in the default configuration. {issue}17256[17256] {pull}27762[27762]

Expand Down
160 changes: 5 additions & 155 deletions dev-tools/ecs-migration.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@
alias6: true
alias: true

- from: docker.container.labels # TODO: How to map these?
- from: docker.container.labels # TODO: How to map these?
to: container.labels
alias6: false
alias: true
Expand Down Expand Up @@ -115,8 +115,8 @@

- from: source
to:
- log.file.path
- log.source.address
- log.file.path
- log.source.address
alias: false
beat: filebeat

Expand Down Expand Up @@ -428,7 +428,7 @@
beat: filebeat

- from: suricata.eve.timestamp
to: '@timestamp'
to: "@timestamp"
alias: true
beat: filebeat

Expand Down Expand Up @@ -476,7 +476,7 @@
beat: filebeat

- from: system.auth.timestamp
to: '@timestamp'
to: "@timestamp"
alias: true
beat: filebeat

Expand Down Expand Up @@ -560,155 +560,6 @@
alias: true
beat: filebeat

## Apache module

- from: apache2.access.remote_ip
to: source.address
alias: true
beat: filebeat

- from: apache2.access.user_name
to: user.name
alias: true
beat: filebeat

- from: apache2.access.method
to: http.request.method
alias: true
beat: filebeat

- from: apache2.access.url
to: url.original
alias: true
beat: filebeat

- from: apache2.access.http_version
to: http.version
alias: true
beat: filebeat

- from: apache2.access.response_code
to: http.response.status_code
alias: true
beat: filebeat

- from: apache2.access.referrer
to: http.request.referrer
alias: true
beat: filebeat

- from: apache2.access.agent
to: user_agent.original
alias: true
beat: filebeat

- from: apache2.access.body_sent.bytes
to: http.response.body.bytes
alias: true
beat: filebeat

- from: apache2.access.geoip.continent_name
to: source.geo.continent_name
alias: true
beat: filebeat

- from: apache2.access.geoip.country_iso_code
to: source.geo.country_iso_code
alias: true
beat: filebeat

- from: apache2.access.geoip.location
to: source.geo.location
alias: true
beat: filebeat

- from: apache2.access.geoip.region_name
to: source.geo.region_name
alias: true
beat: filebeat

- from: apache2.access.geoip.city_name
to: source.geo.city_name
alias: true
beat: filebeat

- from: apache2.access.geoip.region_iso_code
to: source.geo.region_iso_code
alias: true
beat: filebeat

- from: apache2.access.user_agent.original
to: user_agent.original
alias: true
beat: filebeat
- from: apache2.access.user_agent.device
to: user_agent.device.name
alias: true
beat: filebeat
- from: apache2.access.user_agent.name
to: user_agent.name
alias: true
beat: filebeat
- from: apache2.access.user_agent.os
to: user_agent.os.full_name
alias: true
beat: filebeat
- from: apache2.access.user_agent.os_name
to: user_agent.os.name
alias: true
beat: filebeat

- from: apache2.access.user_agent.major
to: user_agent.version
alias: false
beat: filebeat
- from: apache2.access.user_agent.minor
to: user_agent.version
alias: false
beat: filebeat
- from: apache2.access.user_agent.patch
to: user_agent.version
alias: false
beat: filebeat
- from: apache2.access.user_agent.os_major
to: user_agent.os.version
alias: false
beat: filebeat
- from: apache2.access.user_agent.os_minor
to: user_agent.os.version
alias: false
beat: filebeat
- from: apache2.access.user_agent.os_patch
to: user_agent.os.version
alias: false
beat: filebeat

### Error fileset
- from: apache2.error.message
to: message
alias: true
beat: filebeat

- from: apache2.error.level
to: log.level
alias: true
beat: filebeat

- from: apache2.error.client
to: source.address
alias: true
beat: filebeat

- from: apache2.error.pid
to: process.pid
alias: true
beat: filebeat

- from: apache2.error.tid
to: process.thread.id
alias: true
beat: filebeat

## Elasticsearch module

- from: elasticsearch.audit.origin_address
Expand Down Expand Up @@ -1748,7 +1599,6 @@
alias: true
beat: metricbeat


### Redis

- from: php_fpm.status.pid
Expand Down
32 changes: 16 additions & 16 deletions filebeat/autodiscover/builder/hints/logs_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -405,14 +405,14 @@ func TestGenerateHints(t *testing.T) {
},
"hints": common.MapStr{
"logs": common.MapStr{
"module": "apache2",
"module": "apache",
},
},
},
len: 1,
result: []common.MapStr{
{
"module": "apache2",
"module": "apache",
"error": map[string]interface{}{
"enabled": true,
"input": map[string]interface{}{
Expand Down Expand Up @@ -455,15 +455,15 @@ func TestGenerateHints(t *testing.T) {
},
"hints": common.MapStr{
"logs": common.MapStr{
"module": "apache2",
"module": "apache",
"fileset": "access",
},
},
},
len: 1,
result: []common.MapStr{
{
"module": "apache2",
"module": "apache",
"access": map[string]interface{}{
"enabled": true,
"input": map[string]interface{}{
Expand Down Expand Up @@ -506,7 +506,7 @@ func TestGenerateHints(t *testing.T) {
},
"hints": common.MapStr{
"logs": common.MapStr{
"module": "apache2",
"module": "apache",
"fileset.stdout": "access",
"fileset.stderr": "error",
},
Expand All @@ -515,7 +515,7 @@ func TestGenerateHints(t *testing.T) {
len: 1,
result: []common.MapStr{
{
"module": "apache2",
"module": "apache",
"access": map[string]interface{}{
"enabled": true,
"input": map[string]interface{}{
Expand Down Expand Up @@ -558,14 +558,14 @@ func TestGenerateHints(t *testing.T) {
},
"hints": common.MapStr{
"logs": common.MapStr{
"module": "apache2",
"module": "apache",
},
},
},
len: 1,
result: []common.MapStr{
{
"module": "apache2",
"module": "apache",
"error": map[string]interface{}{
"enabled": true,
"input": map[string]interface{}{
Expand Down Expand Up @@ -606,15 +606,15 @@ func TestGenerateHints(t *testing.T) {
},
"hints": common.MapStr{
"logs": common.MapStr{
"module": "apache2",
"module": "apache",
"fileset": "access",
},
},
},
len: 1,
result: []common.MapStr{
{
"module": "apache2",
"module": "apache",
"access": map[string]interface{}{
"enabled": true,
"input": map[string]interface{}{
Expand Down Expand Up @@ -655,7 +655,7 @@ func TestGenerateHints(t *testing.T) {
},
"hints": common.MapStr{
"logs": common.MapStr{
"module": "apache2",
"module": "apache",
"fileset.stdout": "access",
"fileset.stderr": "error",
},
Expand All @@ -664,7 +664,7 @@ func TestGenerateHints(t *testing.T) {
len: 1,
result: []common.MapStr{
{
"module": "apache2",
"module": "apache",
"access": map[string]interface{}{
"enabled": true,
"input": map[string]interface{}{
Expand Down Expand Up @@ -826,14 +826,14 @@ func TestGenerateHintsWithPaths(t *testing.T) {
},
"hints": common.MapStr{
"logs": common.MapStr{
"module": "apache2",
"module": "apache",
},
},
},
len: 1,
path: "/var/log/pods/${data.kubernetes.pod.uid}/${data.kubernetes.container.name}/*.log",
result: common.MapStr{
"module": "apache2",
"module": "apache",
"error": map[string]interface{}{
"enabled": true,
"input": map[string]interface{}{
Expand Down Expand Up @@ -878,15 +878,15 @@ func TestGenerateHintsWithPaths(t *testing.T) {
},
"hints": common.MapStr{
"logs": common.MapStr{
"module": "apache2",
"module": "apache",
"fileset": "access",
},
},
},
len: 1,
path: "/var/log/pods/${data.kubernetes.pod.uid}/${data.kubernetes.container.name}/*.log",
result: common.MapStr{
"module": "apache2",
"module": "apache",
"access": map[string]interface{}{
"enabled": true,
"input": map[string]interface{}{
Expand Down
Loading