From c57e35bdcc4a08e1104c1cfcfd5c3d71240b0103 Mon Sep 17 00:00:00 2001 From: Andrew Kroh Date: Thu, 28 Feb 2019 20:22:34 -0500 Subject: [PATCH] Trim project from GCE cloud metadata (#10987) This removes the project info from the cloud.machine.type and cloud.availability_zone fields that are added to the add_cloud_metadata processor for Google Compute Engine (GCE). Fixes #10968, #10775 (docs) (cherry picked from commit f527cd53ef8d1fb92cf30814aeed94a15a9dd5ff) --- CHANGELOG.next.asciidoc | 3 + libbeat/docs/processors-using.asciidoc | 90 ++++++++----------- .../add_cloud_metadata/provider_google_gce.go | 16 ++++ .../provider_google_gce_test.go | 4 +- 4 files changed, 59 insertions(+), 54 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 11f916ddfce6..cdb5e7022648 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -10,6 +10,9 @@ https://github.com/elastic/beats/compare/v7.0.0-beta1...master[Check the HEAD di *Affecting all Beats* +- On Google Cloud Engine (GCE) the add_cloud_metadata will now trim the project + info from the cloud.machine.type and cloud.availability_zone. {issue}10968[10968] + *Auditbeat* *Filebeat* diff --git a/libbeat/docs/processors-using.asciidoc b/libbeat/docs/processors-using.asciidoc index 4332cd741325..966be470d35e 100644 --- a/libbeat/docs/processors-using.asciidoc +++ b/libbeat/docs/processors-using.asciidoc @@ -442,14 +442,12 @@ _EC2_ [source,json] ------------------------------------------------------------------------------- { - "meta": { - "cloud": { - "availability_zone": "us-east-1c", - "instance_id": "i-4e123456", - "machine_type": "t2.medium", - "provider": "ec2", - "region": "us-east-1" - } + "cloud": { + "availability_zone": "us-east-1c", + "instance_id": "i-4e123456", + "machine_type": "t2.medium", + "provider": "ec2", + "region": "us-east-1" } } ------------------------------------------------------------------------------- @@ -459,12 +457,10 @@ _Digital Ocean_ [source,json] ------------------------------------------------------------------------------- { - "meta": { - "cloud": { - "instance_id": "1234567", - "provider": "digitalocean", - "region": "nyc2" - } + "cloud": { + "instance_id": "1234567", + "provider": "digitalocean", + "region": "nyc2" } } ------------------------------------------------------------------------------- @@ -474,14 +470,12 @@ _GCE_ [source,json] ------------------------------------------------------------------------------- { - "meta": { - "cloud": { - "availability_zone": "projects/1234567890/zones/us-east1-b", - "instance_id": "1234556778987654321", - "machine_type": "projects/1234567890/machineTypes/f1-micro", - "project_id": "my-dev", - "provider": "gce" - } + "cloud": { + "availability_zone": "us-east1-b", + "instance_id": "1234556778987654321", + "machine_type": "f1-micro", + "project_id": "my-dev", + "provider": "gce" } } ------------------------------------------------------------------------------- @@ -491,13 +485,11 @@ _Tencent Cloud_ [source,json] ------------------------------------------------------------------------------- { - "meta": { - "cloud": { - "availability_zone": "gz-azone2", - "instance_id": "ins-qcloudv5", - "provider": "qcloud", - "region": "china-south-gz" - } + "cloud": { + "availability_zone": "gz-azone2", + "instance_id": "ins-qcloudv5", + "provider": "qcloud", + "region": "china-south-gz" } } ------------------------------------------------------------------------------- @@ -510,13 +502,11 @@ ECS instance. [source,json] ------------------------------------------------------------------------------- { - "meta": { - "cloud": { - "availability_zone": "cn-shenzhen", - "instance_id": "i-wz9g2hqiikg0aliyun2b", - "provider": "ecs", - "region": "cn-shenzhen-a" - } + "cloud": { + "availability_zone": "cn-shenzhen", + "instance_id": "i-wz9g2hqiikg0aliyun2b", + "provider": "ecs", + "region": "cn-shenzhen-a" } } ------------------------------------------------------------------------------- @@ -526,14 +516,12 @@ _Azure Virtual Machine_ [source,json] ------------------------------------------------------------------------------- { - "meta": { - "cloud": { - "provider": "az", - "instance_id": "04ab04c3-63de-4709-a9f9-9ab8c0411d5e", - "instance_name": "test-az-vm", - "machine_type": "Standard_D3_v2", - "region": "eastus2" - } + "cloud": { + "provider": "az", + "instance_id": "04ab04c3-63de-4709-a9f9-9ab8c0411d5e", + "instance_name": "test-az-vm", + "machine_type": "Standard_D3_v2", + "region": "eastus2" } } ------------------------------------------------------------------------------- @@ -543,14 +531,12 @@ _Openstack Nova_ [source,json] ------------------------------------------------------------------------------- { - "meta": { - "cloud": { - "provider": "openstack", - "instance_name": "test-998d932195.mycloud.tld", - "availability_zone": "xxxx-az-c", - "instance_id": "i-00011a84", - "machine_type": "m2.large" - } + "cloud": { + "provider": "openstack", + "instance_name": "test-998d932195.mycloud.tld", + "availability_zone": "xxxx-az-c", + "instance_id": "i-00011a84", + "machine_type": "m2.large" } } ------------------------------------------------------------------------------- diff --git a/libbeat/processors/add_cloud_metadata/provider_google_gce.go b/libbeat/processors/add_cloud_metadata/provider_google_gce.go index 5eeefff901ef..d1f3bec2da94 100644 --- a/libbeat/processors/add_cloud_metadata/provider_google_gce.go +++ b/libbeat/processors/add_cloud_metadata/provider_google_gce.go @@ -18,6 +18,8 @@ package add_cloud_metadata import ( + "path" + "github.com/elastic/beats/libbeat/common" s "github.com/elastic/beats/libbeat/common/schema" c "github.com/elastic/beats/libbeat/common/schema/mapstriface" @@ -30,6 +32,18 @@ func newGceMetadataFetcher(config *common.Config) (*metadataFetcher, error) { gceSchema := func(m map[string]interface{}) common.MapStr { out := common.MapStr{} + trimLeadingPath := func(key string) { + v, err := out.GetValue(key) + if err != nil { + return + } + p, ok := v.(string) + if !ok { + return + } + out.Put(key, path.Base(p)) + } + if instance, ok := m["instance"].(map[string]interface{}); ok { s.Schema{ "instance": s.Object{ @@ -41,6 +55,8 @@ func newGceMetadataFetcher(config *common.Config) (*metadataFetcher, error) { }, "availability_zone": c.Str("zone"), }.ApplyTo(out, instance) + trimLeadingPath("machine.type") + trimLeadingPath("availability_zone") } if project, ok := m["project"].(map[string]interface{}); ok { diff --git a/libbeat/processors/add_cloud_metadata/provider_google_gce_test.go b/libbeat/processors/add_cloud_metadata/provider_google_gce_test.go index d5f9bc8d6a47..b76dbc9d4c60 100644 --- a/libbeat/processors/add_cloud_metadata/provider_google_gce_test.go +++ b/libbeat/processors/add_cloud_metadata/provider_google_gce_test.go @@ -158,9 +158,9 @@ func TestRetrieveGCEMetadata(t *testing.T) { "name": "test-gce-dev", }, "machine": common.MapStr{ - "type": "projects/111111111111/machineTypes/f1-micro", + "type": "f1-micro", }, - "availability_zone": "projects/111111111111/zones/us-east1-b", + "availability_zone": "us-east1-b", "project": common.MapStr{ "id": "test-dev", },