From 9d4522682940a8aaace9f3dc68941234f06d5425 Mon Sep 17 00:00:00 2001 From: Linu Elias Date: Thu, 8 Aug 2024 15:59:07 +0530 Subject: [PATCH 01/15] issue fix --- x-pack/metricbeat/module/gcp/constants.go | 1 + .../module/gcp/metrics/cloudsql/metadata.go | 32 ++++++++------ .../module/gcp/metrics/compute/metadata.go | 8 +++- .../module/gcp/metrics/metadata_services.go | 6 +-- .../module/gcp/metrics/metricset.go | 43 ++++++++++++++++++- .../module/gcp/metrics/redis/metadata.go | 32 ++++++++------ .../module/gcp/metrics/timeseries.go | 2 +- .../gcp/timeseries_metadata_collector.go | 19 +++++--- 8 files changed, 102 insertions(+), 41 deletions(-) diff --git a/x-pack/metricbeat/module/gcp/constants.go b/x-pack/metricbeat/module/gcp/constants.go index 787cb63f56aa..95e6a8374f7f 100644 --- a/x-pack/metricbeat/module/gcp/constants.go +++ b/x-pack/metricbeat/module/gcp/constants.go @@ -63,6 +63,7 @@ const ( ECSCloudMachineKey = ECSCloud + "." + ECSCloudMachine ECSCloudMachineType = "type" ECSCloudMachineTypeKey = ECSCloudMachineKey + "." + ECSCloudMachineType + ECSCloudProject = "project" ) // Metadata keys used for events. They follow GCP structure. diff --git a/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go b/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go index 0c21127d81f3..ba0a14b18ffe 100644 --- a/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go +++ b/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go @@ -19,15 +19,17 @@ import ( ) // NewMetadataService returns the specific Metadata service for a GCP CloudSQL resource. -func NewMetadataService(projectID, zone string, region string, regions []string, opt ...option.ClientOption) (gcp.MetadataService, error) { +func NewMetadataService(projectID, zone string, region string, regions []string, organizationID, organizationName string, opt ...option.ClientOption) (gcp.MetadataService, error) { return &metadataCollector{ - projectID: projectID, - zone: zone, - region: region, - regions: regions, - opt: opt, - instances: make(map[string]*sqladmin.DatabaseInstance), - logger: logp.NewLogger("metrics-cloudsql"), + projectID: projectID, + organizationID: organizationID, + organizationName: organizationName, + zone: zone, + region: region, + regions: regions, + opt: opt, + instances: make(map[string]*sqladmin.DatabaseInstance), + logger: logp.NewLogger("metrics-cloudsql"), }, nil } @@ -46,11 +48,13 @@ type cloudsqlMetadata struct { } type metadataCollector struct { - projectID string - zone string - region string - regions []string - opt []option.ClientOption + projectID string + organizationID string + organizationName string + zone string + region string + regions []string + opt []option.ClientOption // NOTE: instances holds data used for all metrics collected in a given period // this avoids calling the remote endpoint for each metric, which would take a long time overall instances map[string]*sqladmin.DatabaseInstance @@ -91,7 +95,7 @@ func (s *metadataCollector) Metadata(ctx context.Context, resp *monitoringpb.Tim return gcp.MetadataCollectorData{}, err } - stackdriverLabels := gcp.NewStackdriverMetadataServiceForTimeSeries(resp) + stackdriverLabels := gcp.NewStackdriverMetadataServiceForTimeSeries(resp, s.organizationID, s.organizationName) metadataCollectorData, err := stackdriverLabels.Metadata(ctx, resp) if err != nil { diff --git a/x-pack/metricbeat/module/gcp/metrics/compute/metadata.go b/x-pack/metricbeat/module/gcp/metrics/compute/metadata.go index c215e471b454..2f7e52aa009c 100644 --- a/x-pack/metricbeat/module/gcp/metrics/compute/metadata.go +++ b/x-pack/metricbeat/module/gcp/metrics/compute/metadata.go @@ -22,9 +22,11 @@ import ( ) // NewMetadataService returns the specific Metadata service for a GCP Compute resource -func NewMetadataService(projectID, zone string, region string, regions []string, opt ...option.ClientOption) (gcp.MetadataService, error) { +func NewMetadataService(projectID, zone string, region string, regions []string, organizationID, organizationName string, opt ...option.ClientOption) (gcp.MetadataService, error) { return &metadataCollector{ projectID: projectID, + organizationID: organizationID, + organizationName: organizationName, zone: zone, region: region, regions: regions, @@ -49,6 +51,8 @@ type computeMetadata struct { type metadataCollector struct { projectID string + organizationID string + organizationName string zone string region string regions []string @@ -63,7 +67,7 @@ func (s *metadataCollector) Metadata(ctx context.Context, resp *monitoringpb.Tim if err != nil { return gcp.MetadataCollectorData{}, err } - stackdriverLabels := gcp.NewStackdriverMetadataServiceForTimeSeries(resp) + stackdriverLabels := gcp.NewStackdriverMetadataServiceForTimeSeries(resp, s.organizationID, s.organizationName) metadataCollectorData, err := stackdriverLabels.Metadata(ctx, resp) if err != nil { return gcp.MetadataCollectorData{}, err diff --git a/x-pack/metricbeat/module/gcp/metrics/metadata_services.go b/x-pack/metricbeat/module/gcp/metrics/metadata_services.go index 3722157eab98..1778bdd76af7 100644 --- a/x-pack/metricbeat/module/gcp/metrics/metadata_services.go +++ b/x-pack/metricbeat/module/gcp/metrics/metadata_services.go @@ -16,11 +16,11 @@ import ( func NewMetadataServiceForConfig(c config, serviceName string) (gcp.MetadataService, error) { switch serviceName { case gcp.ServiceCompute: - return compute.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.opt...) + return compute.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions,c.OrganizationID,c.OrganizationName, c.opt...) case gcp.ServiceCloudSQL: - return cloudsql.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.opt...) + return cloudsql.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions,c.OrganizationID,c.OrganizationName, c.opt...) case gcp.ServiceRedis: - return redis.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.opt...) + return redis.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions,c.OrganizationID,c.OrganizationName, c.opt...) default: return nil, nil } diff --git a/x-pack/metricbeat/module/gcp/metrics/metricset.go b/x-pack/metricbeat/module/gcp/metrics/metricset.go index 0ac5f6d76a85..013befa7487d 100644 --- a/x-pack/metricbeat/module/gcp/metrics/metricset.go +++ b/x-pack/metricbeat/module/gcp/metrics/metricset.go @@ -18,6 +18,8 @@ import ( "google.golang.org/genproto/googleapis/api/metric" "google.golang.org/protobuf/types/known/durationpb" + cloudresourcemanager "google.golang.org/api/cloudresourcemanager/v1" + "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/x-pack/metricbeat/module/gcp" "github.com/elastic/elastic-agent-libs/logp" @@ -102,6 +104,8 @@ type config struct { Region string `config:"region"` Regions []string `config:"regions"` ProjectID string `config:"project_id" validate:"required"` + OrganizationID string `config:"organization_id"` + OrganizationName string `config:"organization_name"` ExcludeLabels bool `config:"exclude_labels"` CredentialsFilePath string `config:"credentials_file_path"` CredentialsJSON string `config:"credentials_json"` @@ -149,7 +153,10 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { Seconds: int64(gcp.MonitoringMetricsSamplingRate), } } - + // set organization id + if err := m.setOrganization(); err != nil { + m.Logger().Warnf("Organization has not been set: %s", err) + } // Get ingest delay and sample period for each metric type ctx := context.Background() client, err := monitoring.NewMetricClient(ctx, m.config.opt...) @@ -215,7 +222,7 @@ func (m *MetricSet) Fetch(ctx context.Context, reporter mb.ReporterV2) (err erro func (m *MetricSet) mapToEvents(ctx context.Context, timeSeries []timeSeriesWithAligner, sdc metricsConfig) ([]mb.Event, error) { mapper := newIncomingFieldMapper(m.Logger(), sdc) - var metadataService = gcp.NewStackdriverMetadataServiceForTimeSeries(nil) + var metadataService = gcp.NewStackdriverMetadataServiceForTimeSeries(nil, "", "") var err error if !m.config.ExcludeLabels { @@ -352,3 +359,35 @@ func addHostFields(groupedEvents []KeyValuePoint) mapstr.M { } return hostRootFields } + +func (m *MetricSet) setOrganization() error { + // get project details + ctx := context.Background() + // Initialize the Cloud Resource Manager service + srv, err := cloudresourcemanager.NewService(ctx, m.config.opt...) + if err != nil { + return fmt.Errorf("failed to create service: %v", err) + } + + // Get the project ancestor details + ancestryResponse, err := srv.Projects.GetAncestry(m.config.ProjectID, &cloudresourcemanager.GetAncestryRequest{}).Do() + if err != nil { + return fmt.Errorf("failed to get project: %v", err) + } + + ancestor := ancestryResponse.Ancestor[len(ancestryResponse.Ancestor)-1] + if ancestor.ResourceId.Type == "organization" { + m.config.OrganizationID = ancestor.ResourceId.Id + orgReq := srv.Organizations.Get("organizations/" + m.config.OrganizationID) + + orgDetails, err := orgReq.Do() + if err != nil { + return fmt.Errorf("failed to get org: %v", err) + } + if orgDetails.DisplayName != "" { + m.config.OrganizationName = orgDetails.DisplayName + } + } + + return nil +} diff --git a/x-pack/metricbeat/module/gcp/metrics/redis/metadata.go b/x-pack/metricbeat/module/gcp/metrics/redis/metadata.go index 1a553cc543f1..f129a7dd6fd6 100644 --- a/x-pack/metricbeat/module/gcp/metrics/redis/metadata.go +++ b/x-pack/metricbeat/module/gcp/metrics/redis/metadata.go @@ -21,15 +21,17 @@ import ( ) // NewMetadataService returns the specific Metadata service for a GCP Redis resource -func NewMetadataService(projectID, zone string, region string, regions []string, opt ...option.ClientOption) (gcp.MetadataService, error) { +func NewMetadataService(projectID, zone string, region string, regions []string, organizationID, organizationName string, opt ...option.ClientOption) (gcp.MetadataService, error) { return &metadataCollector{ - projectID: projectID, - zone: zone, - region: region, - regions: regions, - opt: opt, - instances: make(map[string]*redispb.Instance), - logger: logp.NewLogger("metrics-redis"), + projectID: projectID, + organizationID: organizationID, + organizationName: organizationName, + zone: zone, + region: region, + regions: regions, + opt: opt, + instances: make(map[string]*redispb.Instance), + logger: logp.NewLogger("metrics-redis"), }, nil } @@ -48,11 +50,13 @@ type redisMetadata struct { } type metadataCollector struct { - projectID string - zone string - region string - regions []string - opt []option.ClientOption + projectID string + organizationID string + organizationName string + zone string + region string + regions []string + opt []option.ClientOption // NOTE: instances holds data used for all metrics collected in a given period // this avoids calling the remote endpoint for each metric, which would take a long time overall instances map[string]*redispb.Instance @@ -66,7 +70,7 @@ func (s *metadataCollector) Metadata(ctx context.Context, resp *monitoringpb.Tim return gcp.MetadataCollectorData{}, err } - stackdriverLabels := gcp.NewStackdriverMetadataServiceForTimeSeries(resp) + stackdriverLabels := gcp.NewStackdriverMetadataServiceForTimeSeries(resp, s.organizationID, s.organizationName) metadataCollectorData, err := stackdriverLabels.Metadata(ctx, resp) if err != nil { diff --git a/x-pack/metricbeat/module/gcp/metrics/timeseries.go b/x-pack/metricbeat/module/gcp/metrics/timeseries.go index 0118df395386..4a3de51470bc 100644 --- a/x-pack/metricbeat/module/gcp/metrics/timeseries.go +++ b/x-pack/metricbeat/module/gcp/metrics/timeseries.go @@ -116,7 +116,7 @@ func (m *MetricSet) groupTimeSeries(ctx context.Context, timeSeries []timeSeries aligner := tsa.aligner for _, ts := range tsa.timeSeries { if defaultMetadataService == nil { - metadataService = gcp.NewStackdriverMetadataServiceForTimeSeries(ts) + metadataService = gcp.NewStackdriverMetadataServiceForTimeSeries(ts, m.config.OrganizationID, m.config.OrganizationName) } sdCollectorInputData := gcp.NewStackdriverCollectorInputData(ts, m.config.ProjectID, m.config.Zone, m.config.Region, m.config.Regions) keyValues := mapper.mapTimeSeriesToKeyValuesPoints(ts, aligner) diff --git a/x-pack/metricbeat/module/gcp/timeseries_metadata_collector.go b/x-pack/metricbeat/module/gcp/timeseries_metadata_collector.go index 028f3d713e6a..b5482a6f247d 100644 --- a/x-pack/metricbeat/module/gcp/timeseries_metadata_collector.go +++ b/x-pack/metricbeat/module/gcp/timeseries_metadata_collector.go @@ -27,16 +27,20 @@ func NewStackdriverCollectorInputData(ts *monitoringpb.TimeSeries, projectID, zo // NewStackdriverMetadataServiceForTimeSeries apart from having a long name takes a time series object to return the // Stackdriver canonical Metadata extractor -func NewStackdriverMetadataServiceForTimeSeries(ts *monitoringpb.TimeSeries) MetadataService { +func NewStackdriverMetadataServiceForTimeSeries(ts *monitoringpb.TimeSeries, organizationID, organizationName string) MetadataService { return &StackdriverTimeSeriesMetadataCollector{ - timeSeries: ts, + timeSeries: ts, + organizationID: organizationID, + organizationName: organizationName, } } // StackdriverTimeSeriesMetadataCollector is the implementation of MetadataCollector to collect metrics from Stackdriver // common TimeSeries objects type StackdriverTimeSeriesMetadataCollector struct { - timeSeries *monitoringpb.TimeSeries + timeSeries *monitoringpb.TimeSeries + organizationID string + organizationName string } // Metadata parses a Timeseries object to return its metadata divided into "unknown" (first object) and ECS (second @@ -53,14 +57,19 @@ func (s *StackdriverTimeSeriesMetadataCollector) Metadata(ctx context.Context, i ecs := mapstr.M{ ECSCloud: mapstr.M{ - ECSCloudAccount: mapstr.M{ + ECSCloudProject: mapstr.M{ ECSCloudAccountID: accountID, ECSCloudAccountName: accountID, }, ECSCloudProvider: "gcp", }, } - + if s.organizationID != "" { + _, _ = ecs.Put(ECSCloud+"."+ECSCloudAccount+"."+ECSCloudAccountID, s.organizationID) + } + if s.organizationName != "" { + _, _ = ecs.Put(ECSCloud+"."+ECSCloudAccount+"."+ECSCloudAccountName, s.organizationName) + } if availabilityZone != "" { _, _ = ecs.Put(ECSCloud+"."+ECSCloudAvailabilityZone, availabilityZone) From 9f2752d301c272a8aa1f18c6ab1a3a9892ab916c Mon Sep 17 00:00:00 2001 From: Linu Elias Date: Thu, 8 Aug 2024 16:47:58 +0530 Subject: [PATCH 02/15] add change.log --- CHANGELOG.next.asciidoc | 1 + x-pack/metricbeat/module/gcp/metrics/metricset.go | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 9c6e506a9fc6..dd94ab6ea0ed 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -193,6 +193,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Fix missing metrics from CloudWatch when include_linked_accounts set to false. {issue}40071[40071] {pull}40135[40135] - Update beat module with apm-server monitoring metrics fields {pull}40127[40127] - Fix Azure Monitor metric timespan to restore Storage Account PT1H metrics {issue}40376[40376] {pull}40367[40367] +- Fix to collect the Google Cloud organization ID and display name, and assign them to `cloud.account.id` and `cloud.account.name`, respectively {issue}39203[39203] {pull}40454[40454] *Osquerybeat* diff --git a/x-pack/metricbeat/module/gcp/metrics/metricset.go b/x-pack/metricbeat/module/gcp/metrics/metricset.go index 013befa7487d..409d0de38317 100644 --- a/x-pack/metricbeat/module/gcp/metrics/metricset.go +++ b/x-pack/metricbeat/module/gcp/metrics/metricset.go @@ -155,7 +155,7 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { } // set organization id if err := m.setOrganization(); err != nil { - m.Logger().Warnf("Organization has not been set: %s", err) + m.Logger().Warnf("organization details has not been set: %s", err) } // Get ingest delay and sample period for each metric type ctx := context.Background() @@ -366,23 +366,23 @@ func (m *MetricSet) setOrganization() error { // Initialize the Cloud Resource Manager service srv, err := cloudresourcemanager.NewService(ctx, m.config.opt...) if err != nil { - return fmt.Errorf("failed to create service: %v", err) + return fmt.Errorf("failed to create cloudresourcemanager service: %v", err) } // Get the project ancestor details ancestryResponse, err := srv.Projects.GetAncestry(m.config.ProjectID, &cloudresourcemanager.GetAncestryRequest{}).Do() if err != nil { - return fmt.Errorf("failed to get project: %v", err) + return fmt.Errorf("failed to get project ancestors: %v", err) } ancestor := ancestryResponse.Ancestor[len(ancestryResponse.Ancestor)-1] if ancestor.ResourceId.Type == "organization" { m.config.OrganizationID = ancestor.ResourceId.Id - orgReq := srv.Organizations.Get("organizations/" + m.config.OrganizationID) + orgReq := srv.Organizations.Get(fmt.Sprintf("organizations/%s", m.config.OrganizationID)) orgDetails, err := orgReq.Do() if err != nil { - return fmt.Errorf("failed to get org: %v", err) + return fmt.Errorf("failed to get organization details: %v", err) } if orgDetails.DisplayName != "" { m.config.OrganizationName = orgDetails.DisplayName From 7a56bd9f93cf4a266fae6721b67aec6b639ddf1c Mon Sep 17 00:00:00 2001 From: Linu Elias Date: Thu, 8 Aug 2024 17:25:36 +0530 Subject: [PATCH 03/15] formatting --- x-pack/metricbeat/module/gcp/metrics/metadata_services.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/metricbeat/module/gcp/metrics/metadata_services.go b/x-pack/metricbeat/module/gcp/metrics/metadata_services.go index 1778bdd76af7..d7e1f1325b23 100644 --- a/x-pack/metricbeat/module/gcp/metrics/metadata_services.go +++ b/x-pack/metricbeat/module/gcp/metrics/metadata_services.go @@ -16,11 +16,11 @@ import ( func NewMetadataServiceForConfig(c config, serviceName string) (gcp.MetadataService, error) { switch serviceName { case gcp.ServiceCompute: - return compute.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions,c.OrganizationID,c.OrganizationName, c.opt...) + return compute.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.OrganizationID, c.OrganizationName, c.opt...) case gcp.ServiceCloudSQL: - return cloudsql.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions,c.OrganizationID,c.OrganizationName, c.opt...) + return cloudsql.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.OrganizationID, c.OrganizationName, c.opt...) case gcp.ServiceRedis: - return redis.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions,c.OrganizationID,c.OrganizationName, c.opt...) + return redis.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.OrganizationID, c.OrganizationName, c.opt...) default: return nil, nil } From 7228ac41c647eb1c528dddb78538314e310756c8 Mon Sep 17 00:00:00 2001 From: Linu Elias Date: Fri, 9 Aug 2024 10:08:27 +0530 Subject: [PATCH 04/15] change.log --- CHANGELOG.next.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index dd94ab6ea0ed..7a89cc413821 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -193,7 +193,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Fix missing metrics from CloudWatch when include_linked_accounts set to false. {issue}40071[40071] {pull}40135[40135] - Update beat module with apm-server monitoring metrics fields {pull}40127[40127] - Fix Azure Monitor metric timespan to restore Storage Account PT1H metrics {issue}40376[40376] {pull}40367[40367] -- Fix to collect the Google Cloud organization ID and display name, and assign them to `cloud.account.id` and `cloud.account.name`, respectively {issue}39203[39203] {pull}40454[40454] +- Fix to collect the Google Cloud organization ID and display name, and assign them to `cloud.account.id` and `cloud.account.name`, respectively {issue}39203[39203] {pull}40461[40461] *Osquerybeat* From be57da1e11a7cb49eda519e7f1d6eb50fca16529 Mon Sep 17 00:00:00 2001 From: Linu Elias Date: Mon, 12 Aug 2024 14:54:55 +0530 Subject: [PATCH 05/15] error-fix --- x-pack/metricbeat/module/gcp/metrics/metricset.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/metricbeat/module/gcp/metrics/metricset.go b/x-pack/metricbeat/module/gcp/metrics/metricset.go index 409d0de38317..d4cb0b0a78c5 100644 --- a/x-pack/metricbeat/module/gcp/metrics/metricset.go +++ b/x-pack/metricbeat/module/gcp/metrics/metricset.go @@ -366,13 +366,13 @@ func (m *MetricSet) setOrganization() error { // Initialize the Cloud Resource Manager service srv, err := cloudresourcemanager.NewService(ctx, m.config.opt...) if err != nil { - return fmt.Errorf("failed to create cloudresourcemanager service: %v", err) + return fmt.Errorf("failed to create cloudresourcemanager service: %w", err) } // Get the project ancestor details ancestryResponse, err := srv.Projects.GetAncestry(m.config.ProjectID, &cloudresourcemanager.GetAncestryRequest{}).Do() if err != nil { - return fmt.Errorf("failed to get project ancestors: %v", err) + return fmt.Errorf("failed to get project ancestors: %w", err) } ancestor := ancestryResponse.Ancestor[len(ancestryResponse.Ancestor)-1] @@ -382,7 +382,7 @@ func (m *MetricSet) setOrganization() error { orgDetails, err := orgReq.Do() if err != nil { - return fmt.Errorf("failed to get organization details: %v", err) + return fmt.Errorf("failed to get organization details: %w", err) } if orgDetails.DisplayName != "" { m.config.OrganizationName = orgDetails.DisplayName From 67e8aa6bfcc7292ae46c821d0fc42c17ca002a23 Mon Sep 17 00:00:00 2001 From: Linu Elias Date: Tue, 13 Aug 2024 00:36:54 +0530 Subject: [PATCH 06/15] resolved comments --- CHANGELOG.next.asciidoc | 2 +- .../module/gcp/metrics/metadata_services.go | 6 ++--- .../module/gcp/metrics/metricset.go | 26 +++++++++---------- .../module/gcp/metrics/timeseries.go | 2 +- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 7a89cc413821..e0615d938bcc 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -193,7 +193,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Fix missing metrics from CloudWatch when include_linked_accounts set to false. {issue}40071[40071] {pull}40135[40135] - Update beat module with apm-server monitoring metrics fields {pull}40127[40127] - Fix Azure Monitor metric timespan to restore Storage Account PT1H metrics {issue}40376[40376] {pull}40367[40367] -- Fix to collect the Google Cloud organization ID and display name, and assign them to `cloud.account.id` and `cloud.account.name`, respectively {issue}39203[39203] {pull}40461[40461] +- Add GCP organization ID and display name to ECS cloud fields. {issue}39203[39203] {pull}40461[40461] *Osquerybeat* diff --git a/x-pack/metricbeat/module/gcp/metrics/metadata_services.go b/x-pack/metricbeat/module/gcp/metrics/metadata_services.go index d7e1f1325b23..faf4b02efef3 100644 --- a/x-pack/metricbeat/module/gcp/metrics/metadata_services.go +++ b/x-pack/metricbeat/module/gcp/metrics/metadata_services.go @@ -16,11 +16,11 @@ import ( func NewMetadataServiceForConfig(c config, serviceName string) (gcp.MetadataService, error) { switch serviceName { case gcp.ServiceCompute: - return compute.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.OrganizationID, c.OrganizationName, c.opt...) + return compute.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.organizationID, c.organizationName, c.opt...) case gcp.ServiceCloudSQL: - return cloudsql.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.OrganizationID, c.OrganizationName, c.opt...) + return cloudsql.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.organizationID, c.organizationName, c.opt...) case gcp.ServiceRedis: - return redis.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.OrganizationID, c.OrganizationName, c.opt...) + return redis.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.organizationID, c.organizationName, c.opt...) default: return nil, nil } diff --git a/x-pack/metricbeat/module/gcp/metrics/metricset.go b/x-pack/metricbeat/module/gcp/metrics/metricset.go index d4cb0b0a78c5..32474520839b 100644 --- a/x-pack/metricbeat/module/gcp/metrics/metricset.go +++ b/x-pack/metricbeat/module/gcp/metrics/metricset.go @@ -104,14 +104,14 @@ type config struct { Region string `config:"region"` Regions []string `config:"regions"` ProjectID string `config:"project_id" validate:"required"` - OrganizationID string `config:"organization_id"` - OrganizationName string `config:"organization_name"` ExcludeLabels bool `config:"exclude_labels"` CredentialsFilePath string `config:"credentials_file_path"` CredentialsJSON string `config:"credentials_json"` - opt []option.ClientOption - period *durationpb.Duration + opt []option.ClientOption + period *durationpb.Duration + organizationID string + organizationName string } // New creates a new instance of the MetricSet. New is responsible for unpacking @@ -153,12 +153,13 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { Seconds: int64(gcp.MonitoringMetricsSamplingRate), } } + + // Get ingest delay and sample period for each metric type + ctx := context.Background() // set organization id - if err := m.setOrganization(); err != nil { + if err := m.setOrganization(ctx); err != nil { m.Logger().Warnf("organization details has not been set: %s", err) } - // Get ingest delay and sample period for each metric type - ctx := context.Background() client, err := monitoring.NewMetricClient(ctx, m.config.opt...) if err != nil { return nil, fmt.Errorf("error creating Stackdriver client: %w", err) @@ -360,9 +361,8 @@ func addHostFields(groupedEvents []KeyValuePoint) mapstr.M { return hostRootFields } -func (m *MetricSet) setOrganization() error { - // get project details - ctx := context.Background() +func (m *MetricSet) setOrganization(ctx context.Context) error { + // Initialize the Cloud Resource Manager service srv, err := cloudresourcemanager.NewService(ctx, m.config.opt...) if err != nil { @@ -377,15 +377,15 @@ func (m *MetricSet) setOrganization() error { ancestor := ancestryResponse.Ancestor[len(ancestryResponse.Ancestor)-1] if ancestor.ResourceId.Type == "organization" { - m.config.OrganizationID = ancestor.ResourceId.Id - orgReq := srv.Organizations.Get(fmt.Sprintf("organizations/%s", m.config.OrganizationID)) + m.config.organizationID = ancestor.ResourceId.Id + orgReq := srv.Organizations.Get(fmt.Sprintf("organizations/%s", m.config.organizationID)) orgDetails, err := orgReq.Do() if err != nil { return fmt.Errorf("failed to get organization details: %w", err) } if orgDetails.DisplayName != "" { - m.config.OrganizationName = orgDetails.DisplayName + m.config.organizationName = orgDetails.DisplayName } } diff --git a/x-pack/metricbeat/module/gcp/metrics/timeseries.go b/x-pack/metricbeat/module/gcp/metrics/timeseries.go index 4a3de51470bc..882efdcea487 100644 --- a/x-pack/metricbeat/module/gcp/metrics/timeseries.go +++ b/x-pack/metricbeat/module/gcp/metrics/timeseries.go @@ -116,7 +116,7 @@ func (m *MetricSet) groupTimeSeries(ctx context.Context, timeSeries []timeSeries aligner := tsa.aligner for _, ts := range tsa.timeSeries { if defaultMetadataService == nil { - metadataService = gcp.NewStackdriverMetadataServiceForTimeSeries(ts, m.config.OrganizationID, m.config.OrganizationName) + metadataService = gcp.NewStackdriverMetadataServiceForTimeSeries(ts, m.config.organizationID, m.config.organizationName) } sdCollectorInputData := gcp.NewStackdriverCollectorInputData(ts, m.config.ProjectID, m.config.Zone, m.config.Region, m.config.Regions) keyValues := mapper.mapTimeSeriesToKeyValuesPoints(ts, aligner) From 6b55a2b16565ab42c991bea7b656e6d4aa6c3d40 Mon Sep 17 00:00:00 2001 From: Linu Elias Date: Wed, 14 Aug 2024 16:48:03 +0530 Subject: [PATCH 07/15] resolved comments --- x-pack/metricbeat/module/gcp/metrics/metricset.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/x-pack/metricbeat/module/gcp/metrics/metricset.go b/x-pack/metricbeat/module/gcp/metrics/metricset.go index 32474520839b..d14f63501670 100644 --- a/x-pack/metricbeat/module/gcp/metrics/metricset.go +++ b/x-pack/metricbeat/module/gcp/metrics/metricset.go @@ -374,8 +374,11 @@ func (m *MetricSet) setOrganization(ctx context.Context) error { if err != nil { return fmt.Errorf("failed to get project ancestors: %w", err) } - + if len(ancestryResponse.Ancestor) == 0 { + return fmt.Errorf("no ancestors found for project '%s'", m.config.ProjectID) + } ancestor := ancestryResponse.Ancestor[len(ancestryResponse.Ancestor)-1] + if ancestor.ResourceId.Type == "organization" { m.config.organizationID = ancestor.ResourceId.Id orgReq := srv.Organizations.Get(fmt.Sprintf("organizations/%s", m.config.organizationID)) @@ -384,9 +387,8 @@ func (m *MetricSet) setOrganization(ctx context.Context) error { if err != nil { return fmt.Errorf("failed to get organization details: %w", err) } - if orgDetails.DisplayName != "" { - m.config.organizationName = orgDetails.DisplayName - } + + m.config.organizationName = orgDetails.DisplayName } return nil From de598db2ae952e731f15c0aa3afda63e9bf546f5 Mon Sep 17 00:00:00 2001 From: Linu Elias Date: Tue, 20 Aug 2024 17:19:25 +0530 Subject: [PATCH 08/15] context added --- x-pack/metricbeat/module/gcp/metrics/metricset.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/metricbeat/module/gcp/metrics/metricset.go b/x-pack/metricbeat/module/gcp/metrics/metricset.go index d14f63501670..3157919e1625 100644 --- a/x-pack/metricbeat/module/gcp/metrics/metricset.go +++ b/x-pack/metricbeat/module/gcp/metrics/metricset.go @@ -370,7 +370,7 @@ func (m *MetricSet) setOrganization(ctx context.Context) error { } // Get the project ancestor details - ancestryResponse, err := srv.Projects.GetAncestry(m.config.ProjectID, &cloudresourcemanager.GetAncestryRequest{}).Do() + ancestryResponse, err := srv.Projects.GetAncestry(m.config.ProjectID, &cloudresourcemanager.GetAncestryRequest{}).Context(ctx).Do() if err != nil { return fmt.Errorf("failed to get project ancestors: %w", err) } @@ -383,7 +383,7 @@ func (m *MetricSet) setOrganization(ctx context.Context) error { m.config.organizationID = ancestor.ResourceId.Id orgReq := srv.Organizations.Get(fmt.Sprintf("organizations/%s", m.config.organizationID)) - orgDetails, err := orgReq.Do() + orgDetails, err := orgReq.Context(ctx).Do() if err != nil { return fmt.Errorf("failed to get organization details: %w", err) } From 31d8ec61a5e3bd1f5424f0658f7f494f7385f683 Mon Sep 17 00:00:00 2001 From: Linu Elias Date: Wed, 4 Sep 2024 12:51:29 +0530 Subject: [PATCH 09/15] resolve comments --- x-pack/metricbeat/module/gcp/constants.go | 6 +++--- .../module/gcp/metrics/cloudsql/metadata.go | 6 ++++-- .../module/gcp/metrics/compute/metadata.go | 6 ++++-- .../module/gcp/metrics/metadata_services.go | 6 +++--- .../metricbeat/module/gcp/metrics/metricset.go | 16 ++++++++++++---- .../module/gcp/metrics/redis/metadata.go | 6 ++++-- .../metricbeat/module/gcp/metrics/timeseries.go | 2 +- .../module/gcp/timeseries_metadata_collector.go | 12 +++++++----- 8 files changed, 38 insertions(+), 22 deletions(-) diff --git a/x-pack/metricbeat/module/gcp/constants.go b/x-pack/metricbeat/module/gcp/constants.go index 95e6a8374f7f..bba1171bb58c 100644 --- a/x-pack/metricbeat/module/gcp/constants.go +++ b/x-pack/metricbeat/module/gcp/constants.go @@ -48,9 +48,9 @@ const ( ECSCloudRegion = "region" - ECSCloudAccount = "account" - ECSCloudAccountID = "id" - ECSCloudAccountName = "name" + ECSCloudAccount = "account" + ECSCloudID = "id" + ECSCloudName = "name" ECSCloudInstance = "instance" ECSCloudInstanceKey = ECSCloud + "." + ECSCloudInstance diff --git a/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go b/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go index ba0a14b18ffe..353f77b831d8 100644 --- a/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go +++ b/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go @@ -19,9 +19,10 @@ import ( ) // NewMetadataService returns the specific Metadata service for a GCP CloudSQL resource. -func NewMetadataService(projectID, zone string, region string, regions []string, organizationID, organizationName string, opt ...option.ClientOption) (gcp.MetadataService, error) { +func NewMetadataService(projectID, zone string, region string, regions []string, organizationID, organizationName string, projectName string, opt ...option.ClientOption) (gcp.MetadataService, error) { return &metadataCollector{ projectID: projectID, + projectName: projectName, organizationID: organizationID, organizationName: organizationName, zone: zone, @@ -49,6 +50,7 @@ type cloudsqlMetadata struct { type metadataCollector struct { projectID string + projectName string organizationID string organizationName string zone string @@ -95,7 +97,7 @@ func (s *metadataCollector) Metadata(ctx context.Context, resp *monitoringpb.Tim return gcp.MetadataCollectorData{}, err } - stackdriverLabels := gcp.NewStackdriverMetadataServiceForTimeSeries(resp, s.organizationID, s.organizationName) + stackdriverLabels := gcp.NewStackdriverMetadataServiceForTimeSeries(resp, s.organizationID, s.organizationName, s.projectName) metadataCollectorData, err := stackdriverLabels.Metadata(ctx, resp) if err != nil { diff --git a/x-pack/metricbeat/module/gcp/metrics/compute/metadata.go b/x-pack/metricbeat/module/gcp/metrics/compute/metadata.go index 2f7e52aa009c..3a7b12986814 100644 --- a/x-pack/metricbeat/module/gcp/metrics/compute/metadata.go +++ b/x-pack/metricbeat/module/gcp/metrics/compute/metadata.go @@ -22,9 +22,10 @@ import ( ) // NewMetadataService returns the specific Metadata service for a GCP Compute resource -func NewMetadataService(projectID, zone string, region string, regions []string, organizationID, organizationName string, opt ...option.ClientOption) (gcp.MetadataService, error) { +func NewMetadataService(projectID, zone string, region string, regions []string, organizationID, organizationName string, projectName string, opt ...option.ClientOption) (gcp.MetadataService, error) { return &metadataCollector{ projectID: projectID, + projectName: projectName, organizationID: organizationID, organizationName: organizationName, zone: zone, @@ -51,6 +52,7 @@ type computeMetadata struct { type metadataCollector struct { projectID string + projectName string organizationID string organizationName string zone string @@ -67,7 +69,7 @@ func (s *metadataCollector) Metadata(ctx context.Context, resp *monitoringpb.Tim if err != nil { return gcp.MetadataCollectorData{}, err } - stackdriverLabels := gcp.NewStackdriverMetadataServiceForTimeSeries(resp, s.organizationID, s.organizationName) + stackdriverLabels := gcp.NewStackdriverMetadataServiceForTimeSeries(resp, s.organizationID, s.organizationName, s.projectName) metadataCollectorData, err := stackdriverLabels.Metadata(ctx, resp) if err != nil { return gcp.MetadataCollectorData{}, err diff --git a/x-pack/metricbeat/module/gcp/metrics/metadata_services.go b/x-pack/metricbeat/module/gcp/metrics/metadata_services.go index faf4b02efef3..95c7971915a4 100644 --- a/x-pack/metricbeat/module/gcp/metrics/metadata_services.go +++ b/x-pack/metricbeat/module/gcp/metrics/metadata_services.go @@ -16,11 +16,11 @@ import ( func NewMetadataServiceForConfig(c config, serviceName string) (gcp.MetadataService, error) { switch serviceName { case gcp.ServiceCompute: - return compute.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.organizationID, c.organizationName, c.opt...) + return compute.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.organizationID, c.organizationName, c.projectName, c.opt...) case gcp.ServiceCloudSQL: - return cloudsql.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.organizationID, c.organizationName, c.opt...) + return cloudsql.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.organizationID, c.organizationName, c.projectName, c.opt...) case gcp.ServiceRedis: - return redis.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.organizationID, c.organizationName, c.opt...) + return redis.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.organizationID, c.organizationName, c.projectName, c.opt...) default: return nil, nil } diff --git a/x-pack/metricbeat/module/gcp/metrics/metricset.go b/x-pack/metricbeat/module/gcp/metrics/metricset.go index 69bf897ce101..b0d3af57ec78 100644 --- a/x-pack/metricbeat/module/gcp/metrics/metricset.go +++ b/x-pack/metricbeat/module/gcp/metrics/metricset.go @@ -112,6 +112,7 @@ type config struct { period *durationpb.Duration organizationID string organizationName string + projectName string } // New creates a new instance of the MetricSet. New is responsible for unpacking @@ -157,8 +158,8 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Get ingest delay and sample period for each metric type ctx := context.Background() // set organization id - if err := m.setOrganization(ctx); err != nil { - m.Logger().Warnf("organization details has not been set: %s", err) + if err := m.setOrgAndProjectDetails(ctx); err != nil { + m.Logger().Warnf("error occurred while fetching organization and project name: %s", err) } client, err := monitoring.NewMetricClient(ctx, m.config.opt...) if err != nil { @@ -361,14 +362,21 @@ func addHostFields(groupedEvents []KeyValuePoint) mapstr.M { return hostRootFields } -func (m *MetricSet) setOrganization(ctx context.Context) error { +func (m *MetricSet) setOrgAndProjectDetails(ctx context.Context) error { // Initialize the Cloud Resource Manager service srv, err := cloudresourcemanager.NewService(ctx, m.config.opt...) if err != nil { return fmt.Errorf("failed to create cloudresourcemanager service: %w", err) } - + // Get Project name + project, err := srv.Projects.Get(m.config.ProjectID).Context(ctx).Do() + if err != nil { + return fmt.Errorf("failed to get project name: %w", err) + } + if project != nil { + m.config.projectName = project.Name + } // Get the project ancestor details ancestryResponse, err := srv.Projects.GetAncestry(m.config.ProjectID, &cloudresourcemanager.GetAncestryRequest{}).Context(ctx).Do() if err != nil { diff --git a/x-pack/metricbeat/module/gcp/metrics/redis/metadata.go b/x-pack/metricbeat/module/gcp/metrics/redis/metadata.go index f129a7dd6fd6..705b86ec838b 100644 --- a/x-pack/metricbeat/module/gcp/metrics/redis/metadata.go +++ b/x-pack/metricbeat/module/gcp/metrics/redis/metadata.go @@ -21,9 +21,10 @@ import ( ) // NewMetadataService returns the specific Metadata service for a GCP Redis resource -func NewMetadataService(projectID, zone string, region string, regions []string, organizationID, organizationName string, opt ...option.ClientOption) (gcp.MetadataService, error) { +func NewMetadataService(projectID, zone string, region string, regions []string, organizationID, organizationName string, projectName string, opt ...option.ClientOption) (gcp.MetadataService, error) { return &metadataCollector{ projectID: projectID, + projectName: projectName, organizationID: organizationID, organizationName: organizationName, zone: zone, @@ -51,6 +52,7 @@ type redisMetadata struct { type metadataCollector struct { projectID string + projectName string organizationID string organizationName string zone string @@ -70,7 +72,7 @@ func (s *metadataCollector) Metadata(ctx context.Context, resp *monitoringpb.Tim return gcp.MetadataCollectorData{}, err } - stackdriverLabels := gcp.NewStackdriverMetadataServiceForTimeSeries(resp, s.organizationID, s.organizationName) + stackdriverLabels := gcp.NewStackdriverMetadataServiceForTimeSeries(resp, s.organizationID, s.organizationName, s.projectName) metadataCollectorData, err := stackdriverLabels.Metadata(ctx, resp) if err != nil { diff --git a/x-pack/metricbeat/module/gcp/metrics/timeseries.go b/x-pack/metricbeat/module/gcp/metrics/timeseries.go index 882efdcea487..0a0d3115450f 100644 --- a/x-pack/metricbeat/module/gcp/metrics/timeseries.go +++ b/x-pack/metricbeat/module/gcp/metrics/timeseries.go @@ -116,7 +116,7 @@ func (m *MetricSet) groupTimeSeries(ctx context.Context, timeSeries []timeSeries aligner := tsa.aligner for _, ts := range tsa.timeSeries { if defaultMetadataService == nil { - metadataService = gcp.NewStackdriverMetadataServiceForTimeSeries(ts, m.config.organizationID, m.config.organizationName) + metadataService = gcp.NewStackdriverMetadataServiceForTimeSeries(ts, m.config.organizationID, m.config.organizationName, m.config.projectName) } sdCollectorInputData := gcp.NewStackdriverCollectorInputData(ts, m.config.ProjectID, m.config.Zone, m.config.Region, m.config.Regions) keyValues := mapper.mapTimeSeriesToKeyValuesPoints(ts, aligner) diff --git a/x-pack/metricbeat/module/gcp/timeseries_metadata_collector.go b/x-pack/metricbeat/module/gcp/timeseries_metadata_collector.go index b5482a6f247d..7315cc32b8af 100644 --- a/x-pack/metricbeat/module/gcp/timeseries_metadata_collector.go +++ b/x-pack/metricbeat/module/gcp/timeseries_metadata_collector.go @@ -27,11 +27,12 @@ func NewStackdriverCollectorInputData(ts *monitoringpb.TimeSeries, projectID, zo // NewStackdriverMetadataServiceForTimeSeries apart from having a long name takes a time series object to return the // Stackdriver canonical Metadata extractor -func NewStackdriverMetadataServiceForTimeSeries(ts *monitoringpb.TimeSeries, organizationID, organizationName string) MetadataService { +func NewStackdriverMetadataServiceForTimeSeries(ts *monitoringpb.TimeSeries, organizationID, organizationName string, projectName string) MetadataService { return &StackdriverTimeSeriesMetadataCollector{ timeSeries: ts, organizationID: organizationID, organizationName: organizationName, + projectName: projectName, } } @@ -41,6 +42,7 @@ type StackdriverTimeSeriesMetadataCollector struct { timeSeries *monitoringpb.TimeSeries organizationID string organizationName string + projectName string } // Metadata parses a Timeseries object to return its metadata divided into "unknown" (first object) and ECS (second @@ -58,17 +60,17 @@ func (s *StackdriverTimeSeriesMetadataCollector) Metadata(ctx context.Context, i ecs := mapstr.M{ ECSCloud: mapstr.M{ ECSCloudProject: mapstr.M{ - ECSCloudAccountID: accountID, - ECSCloudAccountName: accountID, + ECSCloudID: accountID, + ECSCloudName: s.projectName, }, ECSCloudProvider: "gcp", }, } if s.organizationID != "" { - _, _ = ecs.Put(ECSCloud+"."+ECSCloudAccount+"."+ECSCloudAccountID, s.organizationID) + _, _ = ecs.Put(ECSCloud+"."+ECSCloudAccount+"."+ECSCloudID, s.organizationID) } if s.organizationName != "" { - _, _ = ecs.Put(ECSCloud+"."+ECSCloudAccount+"."+ECSCloudAccountName, s.organizationName) + _, _ = ecs.Put(ECSCloud+"."+ECSCloudAccount+"."+ECSCloudName, s.organizationName) } if availabilityZone != "" { _, _ = ecs.Put(ECSCloud+"."+ECSCloudAvailabilityZone, availabilityZone) From 0bff706d031d74a9ef9f1339b968f87468554c33 Mon Sep 17 00:00:00 2001 From: Linu Elias Date: Tue, 10 Sep 2024 18:11:54 +0530 Subject: [PATCH 10/15] error handling --- CHANGELOG.next.asciidoc | 2 +- .../metricbeat/module/gcp/_meta/docs.asciidoc | 10 ++++++ .../module/gcp/metrics/metricset.go | 36 ++++++++++++++----- 3 files changed, 38 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index e5ab220db79f..212c6fbc279e 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -60,6 +60,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Update metrics for the vSphere Host metricset. {pull}40429[40429] - Mark system process metricsets as running if metrics are partially available {pull}40565[40565] - Added back `elasticsearch.node.stats.jvm.mem.pools.*` to the `node_stats` metricset {pull}40571[40571] +- Add GCP organization and project details to ECS cloud fields. {pull}40461[40461] *Osquerybeat* @@ -193,7 +194,6 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Fix Azure Monitor metric timespan to restore Storage Account PT1H metrics {issue}40376[40376] {pull}40367[40367] - Remove excessive info-level logs in cgroups setup {pull}40491[40491] - Add missing ECS Cloud fields in GCP `metrics` metricset when using `exclude_labels: true` {issue}40437[40437] {pull}40467[40467] -- Add GCP organization ID and display name to ECS cloud fields. {issue}39203[39203] {pull}40461[40461] *Osquerybeat* diff --git a/x-pack/metricbeat/module/gcp/_meta/docs.asciidoc b/x-pack/metricbeat/module/gcp/_meta/docs.asciidoc index ae9431a23fd6..16f1901bf452 100644 --- a/x-pack/metricbeat/module/gcp/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/gcp/_meta/docs.asciidoc @@ -122,6 +122,10 @@ Generally, you have to create a Service Account and assign it the following role - `compute.instances.get` - `compute.instances.list` +* `Resource Manager`(To get ECS Cloud Labels): +- `resourcemanager.projects.get` +- `resourcemanager.organizations.get` + You can play in IAM pretty much with your service accounts and Instance level access to your resources (for example, allowing that everything running in an Instance is authorized to use the Compute API). The module uses Google Cloud Platform libraries for authentication so many possibilities are open but the Module is only supported by using the method mentioned above. [float] @@ -131,6 +135,12 @@ Google Cloud Platform offers the https://cloud.google.com/monitoring/api/metrics If you also want to *extract service labels* (by setting `exclude_labels` to false, which is the default state). You also make a new API check on the corresponding service. Service labels requires a new API call to extract those metrics. In the worst case the number of API calls will be doubled. In the best case, all metrics come from the same GCP entity and 100% of the required information is included in the first API call (which is cached for subsequent calls). +We have updated our field names to align with ECS semantics. As part of this change: + +* `cloud.account.id` will now contain the Google Cloud Organization ID (previously, it contained the project ID). +* `cloud.account.name` will now contain the Google Cloud Organization Display Name (previously, it contained the project name). +* New fields `cloud.project.id` and `cloud.project.name` will be added to store the actual project ID and project name, respectively. + If `period` value is set to 5-minute and sample period of the metric type is 60-second, then this module will collect data from this metric type once every 5 minutes with aggregation. GCP monitoring data has a up to 240 seconds latency, which means latest monitoring data will be up to 4 minutes old. Please see https://cloud.google.com/monitoring/api/v3/latency-n-retention[Latency of GCP Monitoring Metric Data] for more details. In `gcp` module, metrics are collected based on this ingest delay, which is also obtained from ListMetricDescriptors API. diff --git a/x-pack/metricbeat/module/gcp/metrics/metricset.go b/x-pack/metricbeat/module/gcp/metrics/metricset.go index b0d3af57ec78..604488ab21d6 100644 --- a/x-pack/metricbeat/module/gcp/metrics/metricset.go +++ b/x-pack/metricbeat/module/gcp/metrics/metricset.go @@ -158,8 +158,8 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Get ingest delay and sample period for each metric type ctx := context.Background() // set organization id - if err := m.setOrgAndProjectDetails(ctx); err != nil { - m.Logger().Warnf("error occurred while fetching organization and project name: %s", err) + if errs := m.setOrgAndProjectDetails(ctx); errs != nil { + m.Logger().Warnf("error occurred while fetching organization and project details: %s", errs) } client, err := monitoring.NewMetricClient(ctx, m.config.opt...) if err != nil { @@ -362,23 +362,42 @@ func addHostFields(groupedEvents []KeyValuePoint) mapstr.M { return hostRootFields } -func (m *MetricSet) setOrgAndProjectDetails(ctx context.Context) error { +func (m *MetricSet) setOrgAndProjectDetails(ctx context.Context) []error { + var errs []error // Initialize the Cloud Resource Manager service srv, err := cloudresourcemanager.NewService(ctx, m.config.opt...) if err != nil { - return fmt.Errorf("failed to create cloudresourcemanager service: %w", err) + errs = append(errs, fmt.Errorf("failed to create cloudresourcemanager service: %w", err)) + return errs } - // Get Project name - project, err := srv.Projects.Get(m.config.ProjectID).Context(ctx).Do() + // Set Project name + err = m.setProjectDetails(ctx, srv) + if err != nil { + errs = append(errs, err) + } + //Set Organization Details + err = m.setOrganizationDetails(ctx, srv) + if err != nil { + errs = append(errs, err) + } + return errs +} + +func (m *MetricSet) setProjectDetails(ctx context.Context, service *cloudresourcemanager.Service) error { + project, err := service.Projects.Get(m.config.ProjectID).Context(ctx).Do() if err != nil { return fmt.Errorf("failed to get project name: %w", err) } if project != nil { m.config.projectName = project.Name } + return nil +} + +func (m *MetricSet) setOrganizationDetails(ctx context.Context, service *cloudresourcemanager.Service) error { // Get the project ancestor details - ancestryResponse, err := srv.Projects.GetAncestry(m.config.ProjectID, &cloudresourcemanager.GetAncestryRequest{}).Context(ctx).Do() + ancestryResponse, err := service.Projects.GetAncestry(m.config.ProjectID, &cloudresourcemanager.GetAncestryRequest{}).Context(ctx).Do() if err != nil { return fmt.Errorf("failed to get project ancestors: %w", err) } @@ -389,7 +408,7 @@ func (m *MetricSet) setOrgAndProjectDetails(ctx context.Context) error { if ancestor.ResourceId.Type == "organization" { m.config.organizationID = ancestor.ResourceId.Id - orgReq := srv.Organizations.Get(fmt.Sprintf("organizations/%s", m.config.organizationID)) + orgReq := service.Organizations.Get(fmt.Sprintf("organizations/%s", m.config.organizationID)) orgDetails, err := orgReq.Context(ctx).Do() if err != nil { @@ -398,6 +417,5 @@ func (m *MetricSet) setOrgAndProjectDetails(ctx context.Context) error { m.config.organizationName = orgDetails.DisplayName } - return nil } From 06ca307b4f557439bdc8df71c851c7484d845da0 Mon Sep 17 00:00:00 2001 From: Linu Elias Date: Tue, 10 Sep 2024 18:45:57 +0530 Subject: [PATCH 11/15] doc changes --- metricbeat/docs/modules/gcp.asciidoc | 10 ++++++++++ x-pack/metricbeat/module/gcp/_meta/docs.asciidoc | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/metricbeat/docs/modules/gcp.asciidoc b/metricbeat/docs/modules/gcp.asciidoc index 5f81455412ac..3f6323a32256 100644 --- a/metricbeat/docs/modules/gcp.asciidoc +++ b/metricbeat/docs/modules/gcp.asciidoc @@ -134,6 +134,10 @@ Generally, you have to create a Service Account and assign it the following role - `compute.instances.get` - `compute.instances.list` +* `Browser`: +- `resourcemanager.projects.get` +- `resourcemanager.organizations.get` + You can play in IAM pretty much with your service accounts and Instance level access to your resources (for example, allowing that everything running in an Instance is authorized to use the Compute API). The module uses Google Cloud Platform libraries for authentication so many possibilities are open but the Module is only supported by using the method mentioned above. [float] @@ -143,6 +147,12 @@ Google Cloud Platform offers the https://cloud.google.com/monitoring/api/metrics If you also want to *extract service labels* (by setting `exclude_labels` to false, which is the default state). You also make a new API check on the corresponding service. Service labels requires a new API call to extract those metrics. In the worst case the number of API calls will be doubled. In the best case, all metrics come from the same GCP entity and 100% of the required information is included in the first API call (which is cached for subsequent calls). +We have updated our field names to align with ECS semantics. As part of this change: + +* `cloud.account.id` will now contain the Google Cloud Organization ID (previously, it contained the project ID). +* `cloud.account.name` will now contain the Google Cloud Organization Display Name (previously, it contained the project name). +* New fields `cloud.project.id` and `cloud.project.name` will be added to store the actual project ID and project name, respectively. + If `period` value is set to 5-minute and sample period of the metric type is 60-second, then this module will collect data from this metric type once every 5 minutes with aggregation. GCP monitoring data has a up to 240 seconds latency, which means latest monitoring data will be up to 4 minutes old. Please see https://cloud.google.com/monitoring/api/v3/latency-n-retention[Latency of GCP Monitoring Metric Data] for more details. In `gcp` module, metrics are collected based on this ingest delay, which is also obtained from ListMetricDescriptors API. diff --git a/x-pack/metricbeat/module/gcp/_meta/docs.asciidoc b/x-pack/metricbeat/module/gcp/_meta/docs.asciidoc index 16f1901bf452..795df412421e 100644 --- a/x-pack/metricbeat/module/gcp/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/gcp/_meta/docs.asciidoc @@ -122,7 +122,7 @@ Generally, you have to create a Service Account and assign it the following role - `compute.instances.get` - `compute.instances.list` -* `Resource Manager`(To get ECS Cloud Labels): +* `Browser`: - `resourcemanager.projects.get` - `resourcemanager.organizations.get` From bef89eaf02466417991f64c4dd41bf00af0f0215 Mon Sep 17 00:00:00 2001 From: Linu Elias Date: Tue, 17 Sep 2024 12:35:30 +0530 Subject: [PATCH 12/15] change.log --- CHANGELOG.asciidoc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 339720a41b85..b3916467cd4f 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -7,6 +7,12 @@ === Beats version 8.15.1 https://github.com/elastic/beats/compare/v8.15.0\...v8.15.1[View commits] +==== Breaking changes + +*Metricbeat* + +- Add GCP organization and project details to ECS cloud fields. {pull}40461[40461] + ==== Bugfixes *Affecting all Beats* From 5bc85003ab3a3176c141c938189ba1b2176d2518 Mon Sep 17 00:00:00 2001 From: Linu-Elias Date: Wed, 18 Sep 2024 18:31:08 +0530 Subject: [PATCH 13/15] Update gcp.asciidoc --- metricbeat/docs/modules/gcp.asciidoc | 33 ++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/metricbeat/docs/modules/gcp.asciidoc b/metricbeat/docs/modules/gcp.asciidoc index 3f6323a32256..205ef1a1de6d 100644 --- a/metricbeat/docs/modules/gcp.asciidoc +++ b/metricbeat/docs/modules/gcp.asciidoc @@ -153,6 +153,39 @@ We have updated our field names to align with ECS semantics. As part of this cha * `cloud.account.name` will now contain the Google Cloud Organization Display Name (previously, it contained the project name). * New fields `cloud.project.id` and `cloud.project.name` will be added to store the actual project ID and project name, respectively. +To restore the previous version, you can add a custom ingest pipeline to the Elastic Integration: +[source,json] +---- +{ + "processors": [ + { + "set": { + "field": "cloud.account.id", + "value": "{{cloud.project.id}}", + "if": "ctx?.cloud?.project?.id != null" + } + }, + { + "set": { + "field": "cloud.account.name", + "value": "{{cloud.project.name}}", + "if": "ctx?.cloud?.project?.name != null" + } + }, + { + "remove": { + "field": [ + "cloud.project.id", + "cloud.project.name" + ], + "ignore_missing": true + } + } + ] +} +---- +For more information on creating custom ingest pipelines and processors, please see the https://www.elastic.co/guide/en/fleet/current/data-streams-pipeline-tutorial.html#data-streams-pipeline-two[Custom Ingest Pipelines] guide. + If `period` value is set to 5-minute and sample period of the metric type is 60-second, then this module will collect data from this metric type once every 5 minutes with aggregation. GCP monitoring data has a up to 240 seconds latency, which means latest monitoring data will be up to 4 minutes old. Please see https://cloud.google.com/monitoring/api/v3/latency-n-retention[Latency of GCP Monitoring Metric Data] for more details. In `gcp` module, metrics are collected based on this ingest delay, which is also obtained from ListMetricDescriptors API. From 4522f7195c69edf8ebcac041cafb7deacae7a731 Mon Sep 17 00:00:00 2001 From: Linu-Elias Date: Wed, 18 Sep 2024 18:33:41 +0530 Subject: [PATCH 14/15] Update CHANGELOG.asciidoc --- CHANGELOG.asciidoc | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index b3916467cd4f..254b1a590632 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -3,9 +3,9 @@ :issue: https://github.com/elastic/beats/issues/ :pull: https://github.com/elastic/beats/pull/ -[[release-notes-8.15.1]] -=== Beats version 8.15.1 -https://github.com/elastic/beats/compare/v8.15.0\...v8.15.1[View commits] +[[release-notes-8.15.2]] +=== Beats version 8.15.2 +https://github.com/elastic/beats/compare/v8.15.0\...v8.15.2[View commits] ==== Breaking changes @@ -13,6 +13,10 @@ https://github.com/elastic/beats/compare/v8.15.0\...v8.15.1[View commits] - Add GCP organization and project details to ECS cloud fields. {pull}40461[40461] +[[release-notes-8.15.1]] +=== Beats version 8.15.1 +https://github.com/elastic/beats/compare/v8.15.0\...v8.15.1[View commits] + ==== Bugfixes *Affecting all Beats* From e75868df8b5a9dd7fc610e13a45081af0f9a3342 Mon Sep 17 00:00:00 2001 From: Linu Elias Date: Thu, 19 Sep 2024 11:06:14 +0530 Subject: [PATCH 15/15] doc changes --- .../metricbeat/module/gcp/_meta/docs.asciidoc | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/x-pack/metricbeat/module/gcp/_meta/docs.asciidoc b/x-pack/metricbeat/module/gcp/_meta/docs.asciidoc index 795df412421e..a33735c5bd87 100644 --- a/x-pack/metricbeat/module/gcp/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/gcp/_meta/docs.asciidoc @@ -141,6 +141,39 @@ We have updated our field names to align with ECS semantics. As part of this cha * `cloud.account.name` will now contain the Google Cloud Organization Display Name (previously, it contained the project name). * New fields `cloud.project.id` and `cloud.project.name` will be added to store the actual project ID and project name, respectively. +To restore the previous version, you can add a custom ingest pipeline to the Elastic Integration: +[source,json] +---- +{ + "processors": [ + { + "set": { + "field": "cloud.account.id", + "value": "{{cloud.project.id}}", + "if": "ctx?.cloud?.project?.id != null" + } + }, + { + "set": { + "field": "cloud.account.name", + "value": "{{cloud.project.name}}", + "if": "ctx?.cloud?.project?.name != null" + } + }, + { + "remove": { + "field": [ + "cloud.project.id", + "cloud.project.name" + ], + "ignore_missing": true + } + } + ] +} +---- +For more information on creating custom ingest pipelines and processors, please see the https://www.elastic.co/guide/en/fleet/current/data-streams-pipeline-tutorial.html#data-streams-pipeline-two[Custom Ingest Pipelines] guide. + If `period` value is set to 5-minute and sample period of the metric type is 60-second, then this module will collect data from this metric type once every 5 minutes with aggregation. GCP monitoring data has a up to 240 seconds latency, which means latest monitoring data will be up to 4 minutes old. Please see https://cloud.google.com/monitoring/api/v3/latency-n-retention[Latency of GCP Monitoring Metric Data] for more details. In `gcp` module, metrics are collected based on this ingest delay, which is also obtained from ListMetricDescriptors API.