diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 339720a41b85..254b1a590632 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -3,6 +3,16 @@ :issue: https://github.com/elastic/beats/issues/ :pull: https://github.com/elastic/beats/pull/ +[[release-notes-8.15.2]] +=== Beats version 8.15.2 +https://github.com/elastic/beats/compare/v8.15.0\...v8.15.2[View commits] + +==== Breaking changes + +*Metricbeat* + +- Add GCP organization and project details to ECS cloud fields. {pull}40461[40461] + [[release-notes-8.15.1]] === Beats version 8.15.1 https://github.com/elastic/beats/compare/v8.15.0\...v8.15.1[View commits] diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index bd2b258c63f3..e36ea3396a9e 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -58,6 +58,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff] - Add support for Kibana status metricset in v8 format {pull}40275[40275] - Mark system process metricsets as running if metrics are partially available {pull}40565[40565] - Added back `elasticsearch.node.stats.jvm.mem.pools.*` to the `node_stats` metricset {pull}40571[40571] +- Add GCP organization and project details to ECS cloud fields. {pull}40461[40461] *Osquerybeat* diff --git a/metricbeat/docs/modules/gcp.asciidoc b/metricbeat/docs/modules/gcp.asciidoc index 5f81455412ac..205ef1a1de6d 100644 --- a/metricbeat/docs/modules/gcp.asciidoc +++ b/metricbeat/docs/modules/gcp.asciidoc @@ -134,6 +134,10 @@ Generally, you have to create a Service Account and assign it the following role - `compute.instances.get` - `compute.instances.list` +* `Browser`: +- `resourcemanager.projects.get` +- `resourcemanager.organizations.get` + You can play in IAM pretty much with your service accounts and Instance level access to your resources (for example, allowing that everything running in an Instance is authorized to use the Compute API). The module uses Google Cloud Platform libraries for authentication so many possibilities are open but the Module is only supported by using the method mentioned above. [float] @@ -143,6 +147,45 @@ Google Cloud Platform offers the https://cloud.google.com/monitoring/api/metrics If you also want to *extract service labels* (by setting `exclude_labels` to false, which is the default state). You also make a new API check on the corresponding service. Service labels requires a new API call to extract those metrics. In the worst case the number of API calls will be doubled. In the best case, all metrics come from the same GCP entity and 100% of the required information is included in the first API call (which is cached for subsequent calls). +We have updated our field names to align with ECS semantics. As part of this change: + +* `cloud.account.id` will now contain the Google Cloud Organization ID (previously, it contained the project ID). +* `cloud.account.name` will now contain the Google Cloud Organization Display Name (previously, it contained the project name). +* New fields `cloud.project.id` and `cloud.project.name` will be added to store the actual project ID and project name, respectively. + +To restore the previous version, you can add a custom ingest pipeline to the Elastic Integration: +[source,json] +---- +{ + "processors": [ + { + "set": { + "field": "cloud.account.id", + "value": "{{cloud.project.id}}", + "if": "ctx?.cloud?.project?.id != null" + } + }, + { + "set": { + "field": "cloud.account.name", + "value": "{{cloud.project.name}}", + "if": "ctx?.cloud?.project?.name != null" + } + }, + { + "remove": { + "field": [ + "cloud.project.id", + "cloud.project.name" + ], + "ignore_missing": true + } + } + ] +} +---- +For more information on creating custom ingest pipelines and processors, please see the https://www.elastic.co/guide/en/fleet/current/data-streams-pipeline-tutorial.html#data-streams-pipeline-two[Custom Ingest Pipelines] guide. + If `period` value is set to 5-minute and sample period of the metric type is 60-second, then this module will collect data from this metric type once every 5 minutes with aggregation. GCP monitoring data has a up to 240 seconds latency, which means latest monitoring data will be up to 4 minutes old. Please see https://cloud.google.com/monitoring/api/v3/latency-n-retention[Latency of GCP Monitoring Metric Data] for more details. In `gcp` module, metrics are collected based on this ingest delay, which is also obtained from ListMetricDescriptors API. diff --git a/x-pack/metricbeat/module/gcp/_meta/docs.asciidoc b/x-pack/metricbeat/module/gcp/_meta/docs.asciidoc index ae9431a23fd6..a33735c5bd87 100644 --- a/x-pack/metricbeat/module/gcp/_meta/docs.asciidoc +++ b/x-pack/metricbeat/module/gcp/_meta/docs.asciidoc @@ -122,6 +122,10 @@ Generally, you have to create a Service Account and assign it the following role - `compute.instances.get` - `compute.instances.list` +* `Browser`: +- `resourcemanager.projects.get` +- `resourcemanager.organizations.get` + You can play in IAM pretty much with your service accounts and Instance level access to your resources (for example, allowing that everything running in an Instance is authorized to use the Compute API). The module uses Google Cloud Platform libraries for authentication so many possibilities are open but the Module is only supported by using the method mentioned above. [float] @@ -131,6 +135,45 @@ Google Cloud Platform offers the https://cloud.google.com/monitoring/api/metrics If you also want to *extract service labels* (by setting `exclude_labels` to false, which is the default state). You also make a new API check on the corresponding service. Service labels requires a new API call to extract those metrics. In the worst case the number of API calls will be doubled. In the best case, all metrics come from the same GCP entity and 100% of the required information is included in the first API call (which is cached for subsequent calls). +We have updated our field names to align with ECS semantics. As part of this change: + +* `cloud.account.id` will now contain the Google Cloud Organization ID (previously, it contained the project ID). +* `cloud.account.name` will now contain the Google Cloud Organization Display Name (previously, it contained the project name). +* New fields `cloud.project.id` and `cloud.project.name` will be added to store the actual project ID and project name, respectively. + +To restore the previous version, you can add a custom ingest pipeline to the Elastic Integration: +[source,json] +---- +{ + "processors": [ + { + "set": { + "field": "cloud.account.id", + "value": "{{cloud.project.id}}", + "if": "ctx?.cloud?.project?.id != null" + } + }, + { + "set": { + "field": "cloud.account.name", + "value": "{{cloud.project.name}}", + "if": "ctx?.cloud?.project?.name != null" + } + }, + { + "remove": { + "field": [ + "cloud.project.id", + "cloud.project.name" + ], + "ignore_missing": true + } + } + ] +} +---- +For more information on creating custom ingest pipelines and processors, please see the https://www.elastic.co/guide/en/fleet/current/data-streams-pipeline-tutorial.html#data-streams-pipeline-two[Custom Ingest Pipelines] guide. + If `period` value is set to 5-minute and sample period of the metric type is 60-second, then this module will collect data from this metric type once every 5 minutes with aggregation. GCP monitoring data has a up to 240 seconds latency, which means latest monitoring data will be up to 4 minutes old. Please see https://cloud.google.com/monitoring/api/v3/latency-n-retention[Latency of GCP Monitoring Metric Data] for more details. In `gcp` module, metrics are collected based on this ingest delay, which is also obtained from ListMetricDescriptors API. diff --git a/x-pack/metricbeat/module/gcp/constants.go b/x-pack/metricbeat/module/gcp/constants.go index 787cb63f56aa..bba1171bb58c 100644 --- a/x-pack/metricbeat/module/gcp/constants.go +++ b/x-pack/metricbeat/module/gcp/constants.go @@ -48,9 +48,9 @@ const ( ECSCloudRegion = "region" - ECSCloudAccount = "account" - ECSCloudAccountID = "id" - ECSCloudAccountName = "name" + ECSCloudAccount = "account" + ECSCloudID = "id" + ECSCloudName = "name" ECSCloudInstance = "instance" ECSCloudInstanceKey = ECSCloud + "." + ECSCloudInstance @@ -63,6 +63,7 @@ const ( ECSCloudMachineKey = ECSCloud + "." + ECSCloudMachine ECSCloudMachineType = "type" ECSCloudMachineTypeKey = ECSCloudMachineKey + "." + ECSCloudMachineType + ECSCloudProject = "project" ) // Metadata keys used for events. They follow GCP structure. diff --git a/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go b/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go index 0c21127d81f3..353f77b831d8 100644 --- a/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go +++ b/x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go @@ -19,15 +19,18 @@ import ( ) // NewMetadataService returns the specific Metadata service for a GCP CloudSQL resource. -func NewMetadataService(projectID, zone string, region string, regions []string, opt ...option.ClientOption) (gcp.MetadataService, error) { +func NewMetadataService(projectID, zone string, region string, regions []string, organizationID, organizationName string, projectName string, opt ...option.ClientOption) (gcp.MetadataService, error) { return &metadataCollector{ - projectID: projectID, - zone: zone, - region: region, - regions: regions, - opt: opt, - instances: make(map[string]*sqladmin.DatabaseInstance), - logger: logp.NewLogger("metrics-cloudsql"), + projectID: projectID, + projectName: projectName, + organizationID: organizationID, + organizationName: organizationName, + zone: zone, + region: region, + regions: regions, + opt: opt, + instances: make(map[string]*sqladmin.DatabaseInstance), + logger: logp.NewLogger("metrics-cloudsql"), }, nil } @@ -46,11 +49,14 @@ type cloudsqlMetadata struct { } type metadataCollector struct { - projectID string - zone string - region string - regions []string - opt []option.ClientOption + projectID string + projectName string + organizationID string + organizationName string + zone string + region string + regions []string + opt []option.ClientOption // NOTE: instances holds data used for all metrics collected in a given period // this avoids calling the remote endpoint for each metric, which would take a long time overall instances map[string]*sqladmin.DatabaseInstance @@ -91,7 +97,7 @@ func (s *metadataCollector) Metadata(ctx context.Context, resp *monitoringpb.Tim return gcp.MetadataCollectorData{}, err } - stackdriverLabels := gcp.NewStackdriverMetadataServiceForTimeSeries(resp) + stackdriverLabels := gcp.NewStackdriverMetadataServiceForTimeSeries(resp, s.organizationID, s.organizationName, s.projectName) metadataCollectorData, err := stackdriverLabels.Metadata(ctx, resp) if err != nil { diff --git a/x-pack/metricbeat/module/gcp/metrics/compute/metadata.go b/x-pack/metricbeat/module/gcp/metrics/compute/metadata.go index eb9f59713035..9780561fdf2f 100644 --- a/x-pack/metricbeat/module/gcp/metrics/compute/metadata.go +++ b/x-pack/metricbeat/module/gcp/metrics/compute/metadata.go @@ -23,9 +23,12 @@ import ( ) // NewMetadataService returns the specific Metadata service for a GCP Compute resource -func NewMetadataService(projectID, zone string, region string, regions []string, opt ...option.ClientOption) (gcp.MetadataService, error) { +func NewMetadataService(projectID, zone string, region string, regions []string, organizationID, organizationName string, projectName string, opt ...option.ClientOption) (gcp.MetadataService, error) { return &metadataCollector{ projectID: projectID, + projectName: projectName, + organizationID: organizationID, + organizationName: organizationName, zone: zone, region: region, regions: regions, @@ -50,6 +53,9 @@ type computeMetadata struct { type metadataCollector struct { projectID string + projectName string + organizationID string + organizationName string zone string region string regions []string @@ -64,7 +70,7 @@ func (s *metadataCollector) Metadata(ctx context.Context, resp *monitoringpb.Tim if err != nil { return gcp.MetadataCollectorData{}, err } - stackdriverLabels := gcp.NewStackdriverMetadataServiceForTimeSeries(resp) + stackdriverLabels := gcp.NewStackdriverMetadataServiceForTimeSeries(resp, s.organizationID, s.organizationName, s.projectName) metadataCollectorData, err := stackdriverLabels.Metadata(ctx, resp) if err != nil { return gcp.MetadataCollectorData{}, err diff --git a/x-pack/metricbeat/module/gcp/metrics/metadata_services.go b/x-pack/metricbeat/module/gcp/metrics/metadata_services.go index 3722157eab98..95c7971915a4 100644 --- a/x-pack/metricbeat/module/gcp/metrics/metadata_services.go +++ b/x-pack/metricbeat/module/gcp/metrics/metadata_services.go @@ -16,11 +16,11 @@ import ( func NewMetadataServiceForConfig(c config, serviceName string) (gcp.MetadataService, error) { switch serviceName { case gcp.ServiceCompute: - return compute.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.opt...) + return compute.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.organizationID, c.organizationName, c.projectName, c.opt...) case gcp.ServiceCloudSQL: - return cloudsql.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.opt...) + return cloudsql.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.organizationID, c.organizationName, c.projectName, c.opt...) case gcp.ServiceRedis: - return redis.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.opt...) + return redis.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.organizationID, c.organizationName, c.projectName, c.opt...) default: return nil, nil } diff --git a/x-pack/metricbeat/module/gcp/metrics/metricset.go b/x-pack/metricbeat/module/gcp/metrics/metricset.go index d56abf27acbf..604488ab21d6 100644 --- a/x-pack/metricbeat/module/gcp/metrics/metricset.go +++ b/x-pack/metricbeat/module/gcp/metrics/metricset.go @@ -18,6 +18,8 @@ import ( "google.golang.org/genproto/googleapis/api/metric" "google.golang.org/protobuf/types/known/durationpb" + cloudresourcemanager "google.golang.org/api/cloudresourcemanager/v1" + "github.com/elastic/beats/v7/metricbeat/mb" "github.com/elastic/beats/v7/x-pack/metricbeat/module/gcp" "github.com/elastic/elastic-agent-libs/logp" @@ -106,8 +108,11 @@ type config struct { CredentialsFilePath string `config:"credentials_file_path"` CredentialsJSON string `config:"credentials_json"` - opt []option.ClientOption - period *durationpb.Duration + opt []option.ClientOption + period *durationpb.Duration + organizationID string + organizationName string + projectName string } // New creates a new instance of the MetricSet. New is responsible for unpacking @@ -152,6 +157,10 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) { // Get ingest delay and sample period for each metric type ctx := context.Background() + // set organization id + if errs := m.setOrgAndProjectDetails(ctx); errs != nil { + m.Logger().Warnf("error occurred while fetching organization and project details: %s", errs) + } client, err := monitoring.NewMetricClient(ctx, m.config.opt...) if err != nil { return nil, fmt.Errorf("error creating Stackdriver client: %w", err) @@ -352,3 +361,61 @@ func addHostFields(groupedEvents []KeyValuePoint) mapstr.M { } return hostRootFields } + +func (m *MetricSet) setOrgAndProjectDetails(ctx context.Context) []error { + var errs []error + + // Initialize the Cloud Resource Manager service + srv, err := cloudresourcemanager.NewService(ctx, m.config.opt...) + if err != nil { + errs = append(errs, fmt.Errorf("failed to create cloudresourcemanager service: %w", err)) + return errs + } + // Set Project name + err = m.setProjectDetails(ctx, srv) + if err != nil { + errs = append(errs, err) + } + //Set Organization Details + err = m.setOrganizationDetails(ctx, srv) + if err != nil { + errs = append(errs, err) + } + return errs +} + +func (m *MetricSet) setProjectDetails(ctx context.Context, service *cloudresourcemanager.Service) error { + project, err := service.Projects.Get(m.config.ProjectID).Context(ctx).Do() + if err != nil { + return fmt.Errorf("failed to get project name: %w", err) + } + if project != nil { + m.config.projectName = project.Name + } + return nil +} + +func (m *MetricSet) setOrganizationDetails(ctx context.Context, service *cloudresourcemanager.Service) error { + // Get the project ancestor details + ancestryResponse, err := service.Projects.GetAncestry(m.config.ProjectID, &cloudresourcemanager.GetAncestryRequest{}).Context(ctx).Do() + if err != nil { + return fmt.Errorf("failed to get project ancestors: %w", err) + } + if len(ancestryResponse.Ancestor) == 0 { + return fmt.Errorf("no ancestors found for project '%s'", m.config.ProjectID) + } + ancestor := ancestryResponse.Ancestor[len(ancestryResponse.Ancestor)-1] + + if ancestor.ResourceId.Type == "organization" { + m.config.organizationID = ancestor.ResourceId.Id + orgReq := service.Organizations.Get(fmt.Sprintf("organizations/%s", m.config.organizationID)) + + orgDetails, err := orgReq.Context(ctx).Do() + if err != nil { + return fmt.Errorf("failed to get organization details: %w", err) + } + + m.config.organizationName = orgDetails.DisplayName + } + return nil +} diff --git a/x-pack/metricbeat/module/gcp/metrics/redis/metadata.go b/x-pack/metricbeat/module/gcp/metrics/redis/metadata.go index 1a553cc543f1..705b86ec838b 100644 --- a/x-pack/metricbeat/module/gcp/metrics/redis/metadata.go +++ b/x-pack/metricbeat/module/gcp/metrics/redis/metadata.go @@ -21,15 +21,18 @@ import ( ) // NewMetadataService returns the specific Metadata service for a GCP Redis resource -func NewMetadataService(projectID, zone string, region string, regions []string, opt ...option.ClientOption) (gcp.MetadataService, error) { +func NewMetadataService(projectID, zone string, region string, regions []string, organizationID, organizationName string, projectName string, opt ...option.ClientOption) (gcp.MetadataService, error) { return &metadataCollector{ - projectID: projectID, - zone: zone, - region: region, - regions: regions, - opt: opt, - instances: make(map[string]*redispb.Instance), - logger: logp.NewLogger("metrics-redis"), + projectID: projectID, + projectName: projectName, + organizationID: organizationID, + organizationName: organizationName, + zone: zone, + region: region, + regions: regions, + opt: opt, + instances: make(map[string]*redispb.Instance), + logger: logp.NewLogger("metrics-redis"), }, nil } @@ -48,11 +51,14 @@ type redisMetadata struct { } type metadataCollector struct { - projectID string - zone string - region string - regions []string - opt []option.ClientOption + projectID string + projectName string + organizationID string + organizationName string + zone string + region string + regions []string + opt []option.ClientOption // NOTE: instances holds data used for all metrics collected in a given period // this avoids calling the remote endpoint for each metric, which would take a long time overall instances map[string]*redispb.Instance @@ -66,7 +72,7 @@ func (s *metadataCollector) Metadata(ctx context.Context, resp *monitoringpb.Tim return gcp.MetadataCollectorData{}, err } - stackdriverLabels := gcp.NewStackdriverMetadataServiceForTimeSeries(resp) + stackdriverLabels := gcp.NewStackdriverMetadataServiceForTimeSeries(resp, s.organizationID, s.organizationName, s.projectName) metadataCollectorData, err := stackdriverLabels.Metadata(ctx, resp) if err != nil { diff --git a/x-pack/metricbeat/module/gcp/metrics/timeseries.go b/x-pack/metricbeat/module/gcp/metrics/timeseries.go index 0118df395386..0a0d3115450f 100644 --- a/x-pack/metricbeat/module/gcp/metrics/timeseries.go +++ b/x-pack/metricbeat/module/gcp/metrics/timeseries.go @@ -116,7 +116,7 @@ func (m *MetricSet) groupTimeSeries(ctx context.Context, timeSeries []timeSeries aligner := tsa.aligner for _, ts := range tsa.timeSeries { if defaultMetadataService == nil { - metadataService = gcp.NewStackdriverMetadataServiceForTimeSeries(ts) + metadataService = gcp.NewStackdriverMetadataServiceForTimeSeries(ts, m.config.organizationID, m.config.organizationName, m.config.projectName) } sdCollectorInputData := gcp.NewStackdriverCollectorInputData(ts, m.config.ProjectID, m.config.Zone, m.config.Region, m.config.Regions) keyValues := mapper.mapTimeSeriesToKeyValuesPoints(ts, aligner) diff --git a/x-pack/metricbeat/module/gcp/timeseries_metadata_collector.go b/x-pack/metricbeat/module/gcp/timeseries_metadata_collector.go index 028f3d713e6a..7315cc32b8af 100644 --- a/x-pack/metricbeat/module/gcp/timeseries_metadata_collector.go +++ b/x-pack/metricbeat/module/gcp/timeseries_metadata_collector.go @@ -27,16 +27,22 @@ func NewStackdriverCollectorInputData(ts *monitoringpb.TimeSeries, projectID, zo // NewStackdriverMetadataServiceForTimeSeries apart from having a long name takes a time series object to return the // Stackdriver canonical Metadata extractor -func NewStackdriverMetadataServiceForTimeSeries(ts *monitoringpb.TimeSeries) MetadataService { +func NewStackdriverMetadataServiceForTimeSeries(ts *monitoringpb.TimeSeries, organizationID, organizationName string, projectName string) MetadataService { return &StackdriverTimeSeriesMetadataCollector{ - timeSeries: ts, + timeSeries: ts, + organizationID: organizationID, + organizationName: organizationName, + projectName: projectName, } } // StackdriverTimeSeriesMetadataCollector is the implementation of MetadataCollector to collect metrics from Stackdriver // common TimeSeries objects type StackdriverTimeSeriesMetadataCollector struct { - timeSeries *monitoringpb.TimeSeries + timeSeries *monitoringpb.TimeSeries + organizationID string + organizationName string + projectName string } // Metadata parses a Timeseries object to return its metadata divided into "unknown" (first object) and ECS (second @@ -53,14 +59,19 @@ func (s *StackdriverTimeSeriesMetadataCollector) Metadata(ctx context.Context, i ecs := mapstr.M{ ECSCloud: mapstr.M{ - ECSCloudAccount: mapstr.M{ - ECSCloudAccountID: accountID, - ECSCloudAccountName: accountID, + ECSCloudProject: mapstr.M{ + ECSCloudID: accountID, + ECSCloudName: s.projectName, }, ECSCloudProvider: "gcp", }, } - + if s.organizationID != "" { + _, _ = ecs.Put(ECSCloud+"."+ECSCloudAccount+"."+ECSCloudID, s.organizationID) + } + if s.organizationName != "" { + _, _ = ecs.Put(ECSCloud+"."+ECSCloudAccount+"."+ECSCloudName, s.organizationName) + } if availabilityZone != "" { _, _ = ecs.Put(ECSCloud+"."+ECSCloudAvailabilityZone, availabilityZone)