x-pack/metricbeat/module/gcp: Add Organization ID and display name to cloud labels

CHANGELOG.next.asciidoc

@@ -193,6 +193,7 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]
 - Fix Azure Monitor metric timespan to restore Storage Account PT1H metrics {issue}40376[40376] {pull}40367[40367]
 - Remove excessive info-level logs in cgroups setup {pull}40491[40491]
 - Add missing ECS Cloud fields in GCP `metrics` metricset when using `exclude_labels: true` {issue}40437[40437] {pull}40467[40467]
+- Add GCP organization ID and display name to ECS cloud fields. {issue}39203[39203] {pull}40461[40461]
 
 *Osquerybeat*
 

x-pack/metricbeat/module/gcp/constants.go

@@ -48,9 +48,9 @@ const (
 
 	ECSCloudRegion = "region"
 
-	ECSCloudAccount     = "account"
-	ECSCloudAccountID   = "id"
-	ECSCloudAccountName = "name"
+	ECSCloudAccount = "account"
+	ECSCloudID      = "id"
+	ECSCloudName    = "name"
 
 	ECSCloudInstance        = "instance"
 	ECSCloudInstanceKey     = ECSCloud + "." + ECSCloudInstance
@@ -63,6 +63,7 @@ const (
 	ECSCloudMachineKey     = ECSCloud + "." + ECSCloudMachine
 	ECSCloudMachineType    = "type"
 	ECSCloudMachineTypeKey = ECSCloudMachineKey + "." + ECSCloudMachineType
+	ECSCloudProject        = "project"
 )
 
 // Metadata keys used for events. They follow GCP structure.

x-pack/metricbeat/module/gcp/metrics/cloudsql/metadata.go

@@ -19,15 +19,18 @@ import (
 )
 
 // NewMetadataService returns the specific Metadata service for a GCP CloudSQL resource.
-func NewMetadataService(projectID, zone string, region string, regions []string, opt ...option.ClientOption) (gcp.MetadataService, error) {
+func NewMetadataService(projectID, zone string, region string, regions []string, organizationID, organizationName string, projectName string, opt ...option.ClientOption) (gcp.MetadataService, error) {
 	return &metadataCollector{
-		projectID: projectID,
-		zone:      zone,
-		region:    region,
-		regions:   regions,
-		opt:       opt,
-		instances: make(map[string]*sqladmin.DatabaseInstance),
-		logger:    logp.NewLogger("metrics-cloudsql"),
+		projectID:        projectID,
+		projectName:      projectName,
+		organizationID:   organizationID,
+		organizationName: organizationName,
+		zone:             zone,
+		region:           region,
+		regions:          regions,
+		opt:              opt,
+		instances:        make(map[string]*sqladmin.DatabaseInstance),
+		logger:           logp.NewLogger("metrics-cloudsql"),
 	}, nil
 }
 
@@ -46,11 +49,14 @@ type cloudsqlMetadata struct {
 }
 
 type metadataCollector struct {
-	projectID string
-	zone      string
-	region    string
-	regions   []string
-	opt       []option.ClientOption
+	projectID        string
+	projectName      string
+	organizationID   string
+	organizationName string
+	zone             string
+	region           string
+	regions          []string
+	opt              []option.ClientOption
 	// NOTE: instances holds data used for all metrics collected in a given period
 	// this avoids calling the remote endpoint for each metric, which would take a long time overall
 	instances map[string]*sqladmin.DatabaseInstance
@@ -91,7 +97,7 @@ func (s *metadataCollector) Metadata(ctx context.Context, resp *monitoringpb.Tim
 		return gcp.MetadataCollectorData{}, err
 	}
 
-	stackdriverLabels := gcp.NewStackdriverMetadataServiceForTimeSeries(resp)
+	stackdriverLabels := gcp.NewStackdriverMetadataServiceForTimeSeries(resp, s.organizationID, s.organizationName, s.projectName)
 
 	metadataCollectorData, err := stackdriverLabels.Metadata(ctx, resp)
 	if err != nil {

x-pack/metricbeat/module/gcp/metrics/compute/metadata.go

@@ -22,9 +22,12 @@ import (
 )
 
 // NewMetadataService returns the specific Metadata service for a GCP Compute resource
-func NewMetadataService(projectID, zone string, region string, regions []string, opt ...option.ClientOption) (gcp.MetadataService, error) {
+func NewMetadataService(projectID, zone string, region string, regions []string, organizationID, organizationName string, projectName string, opt ...option.ClientOption) (gcp.MetadataService, error) {
 	return &metadataCollector{
 		projectID:        projectID,
+		projectName:      projectName,
+		organizationID:   organizationID,
+		organizationName: organizationName,
 		zone:             zone,
 		region:           region,
 		regions:          regions,
@@ -49,6 +52,9 @@ type computeMetadata struct {
 
 type metadataCollector struct {
 	projectID        string
+	projectName      string
+	organizationID   string
+	organizationName string
 	zone             string
 	region           string
 	regions          []string
@@ -63,7 +69,7 @@ func (s *metadataCollector) Metadata(ctx context.Context, resp *monitoringpb.Tim
 	if err != nil {
 		return gcp.MetadataCollectorData{}, err
 	}
-	stackdriverLabels := gcp.NewStackdriverMetadataServiceForTimeSeries(resp)
+	stackdriverLabels := gcp.NewStackdriverMetadataServiceForTimeSeries(resp, s.organizationID, s.organizationName, s.projectName)
 	metadataCollectorData, err := stackdriverLabels.Metadata(ctx, resp)
 	if err != nil {
 		return gcp.MetadataCollectorData{}, err

x-pack/metricbeat/module/gcp/metrics/metadata_services.go

@@ -16,11 +16,11 @@ import (
 func NewMetadataServiceForConfig(c config, serviceName string) (gcp.MetadataService, error) {
 	switch serviceName {
 	case gcp.ServiceCompute:
-		return compute.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.opt...)
+		return compute.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.organizationID, c.organizationName, c.projectName, c.opt...)
 	case gcp.ServiceCloudSQL:
-		return cloudsql.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.opt...)
+		return cloudsql.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.organizationID, c.organizationName, c.projectName, c.opt...)
 	case gcp.ServiceRedis:
-		return redis.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.opt...)
+		return redis.NewMetadataService(c.ProjectID, c.Zone, c.Region, c.Regions, c.organizationID, c.organizationName, c.projectName, c.opt...)
 	default:
 		return nil, nil
 	}

x-pack/metricbeat/module/gcp/metrics/metricset.go

@@ -18,6 +18,8 @@ import (
 	"google.golang.org/genproto/googleapis/api/metric"
 	"google.golang.org/protobuf/types/known/durationpb"
 
+	cloudresourcemanager "google.golang.org/api/cloudresourcemanager/v1"
+
 	"github.com/elastic/beats/v7/metricbeat/mb"
 	"github.com/elastic/beats/v7/x-pack/metricbeat/module/gcp"
 	"github.com/elastic/elastic-agent-libs/logp"
@@ -106,8 +108,11 @@ type config struct {
 	CredentialsFilePath string   `config:"credentials_file_path"`
 	CredentialsJSON     string   `config:"credentials_json"`
 
-	opt    []option.ClientOption
-	period *durationpb.Duration
+	opt              []option.ClientOption
+	period           *durationpb.Duration
+	organizationID   string
+	organizationName string
+	projectName      string
 }
 
 // New creates a new instance of the MetricSet. New is responsible for unpacking
@@ -152,6 +157,10 @@ func New(base mb.BaseMetricSet) (mb.MetricSet, error) {
 
 	// Get ingest delay and sample period for each metric type
 	ctx := context.Background()
+	// set organization id
+	if err := m.setOrgAndProjectDetails(ctx); err != nil {
+		m.Logger().Warnf("error occurred while fetching organization and project name: %s", err)
+	}
 	client, err := monitoring.NewMetricClient(ctx, m.config.opt...)
 	if err != nil {
 		return nil, fmt.Errorf("error creating Stackdriver client: %w", err)
@@ -352,3 +361,43 @@ func addHostFields(groupedEvents []KeyValuePoint) mapstr.M {
 	}
 	return hostRootFields
 }
+
+func (m *MetricSet) setOrgAndProjectDetails(ctx context.Context) error {
+
+	// Initialize the Cloud Resource Manager service
+	srv, err := cloudresourcemanager.NewService(ctx, m.config.opt...)
+	if err != nil {
+		return fmt.Errorf("failed to create cloudresourcemanager service: %w", err)
+	}
+	// Get Project name
+	project, err := srv.Projects.Get(m.config.ProjectID).Context(ctx).Do()
+	if err != nil {
+		return fmt.Errorf("failed to get project name: %w", err)
+	}
+	if project != nil {
+		m.config.projectName = project.Name
+	}
+	// Get the project ancestor details
+	ancestryResponse, err := srv.Projects.GetAncestry(m.config.ProjectID, &cloudresourcemanager.GetAncestryRequest{}).Context(ctx).Do()
+	if err != nil {
+		return fmt.Errorf("failed to get project ancestors: %w", err)
+	}
+	if len(ancestryResponse.Ancestor) == 0 {
+		return fmt.Errorf("no ancestors found for project '%s'", m.config.ProjectID)
+	}
+	ancestor := ancestryResponse.Ancestor[len(ancestryResponse.Ancestor)-1]
+
+	if ancestor.ResourceId.Type == "organization" {
+		m.config.organizationID = ancestor.ResourceId.Id
+		orgReq := srv.Organizations.Get(fmt.Sprintf("organizations/%s", m.config.organizationID))
+
+		orgDetails, err := orgReq.Context(ctx).Do()
+		if err != nil {
+			return fmt.Errorf("failed to get organization details: %w", err)
+		}
+
+		m.config.organizationName = orgDetails.DisplayName
+	}
+
+	return nil
+}

x-pack/metricbeat/module/gcp/metrics/redis/metadata.go

@@ -21,15 +21,18 @@ import (
 )
 
 // NewMetadataService returns the specific Metadata service for a GCP Redis resource
-func NewMetadataService(projectID, zone string, region string, regions []string, opt ...option.ClientOption) (gcp.MetadataService, error) {
+func NewMetadataService(projectID, zone string, region string, regions []string, organizationID, organizationName string, projectName string, opt ...option.ClientOption) (gcp.MetadataService, error) {
 	return &metadataCollector{
-		projectID: projectID,
-		zone:      zone,
-		region:    region,
-		regions:   regions,
-		opt:       opt,
-		instances: make(map[string]*redispb.Instance),
-		logger:    logp.NewLogger("metrics-redis"),
+		projectID:        projectID,
+		projectName:      projectName,
+		organizationID:   organizationID,
+		organizationName: organizationName,
+		zone:             zone,
+		region:           region,
+		regions:          regions,
+		opt:              opt,
+		instances:        make(map[string]*redispb.Instance),
+		logger:           logp.NewLogger("metrics-redis"),
 	}, nil
 }
 
@@ -48,11 +51,14 @@ type redisMetadata struct {
 }
 
 type metadataCollector struct {
-	projectID string
-	zone      string
-	region    string
-	regions   []string
-	opt       []option.ClientOption
+	projectID        string
+	projectName      string
+	organizationID   string
+	organizationName string
+	zone             string
+	region           string
+	regions          []string
+	opt              []option.ClientOption
 	// NOTE: instances holds data used for all metrics collected in a given period
 	// this avoids calling the remote endpoint for each metric, which would take a long time overall
 	instances map[string]*redispb.Instance
@@ -66,7 +72,7 @@ func (s *metadataCollector) Metadata(ctx context.Context, resp *monitoringpb.Tim
 		return gcp.MetadataCollectorData{}, err
 	}
 
-	stackdriverLabels := gcp.NewStackdriverMetadataServiceForTimeSeries(resp)
+	stackdriverLabels := gcp.NewStackdriverMetadataServiceForTimeSeries(resp, s.organizationID, s.organizationName, s.projectName)
 
 	metadataCollectorData, err := stackdriverLabels.Metadata(ctx, resp)
 	if err != nil {

x-pack/metricbeat/module/gcp/metrics/timeseries.go

@@ -116,7 +116,7 @@ func (m *MetricSet) groupTimeSeries(ctx context.Context, timeSeries []timeSeries
 		aligner := tsa.aligner
 		for _, ts := range tsa.timeSeries {
 			if defaultMetadataService == nil {
-				metadataService = gcp.NewStackdriverMetadataServiceForTimeSeries(ts)
+				metadataService = gcp.NewStackdriverMetadataServiceForTimeSeries(ts, m.config.organizationID, m.config.organizationName, m.config.projectName)
 			}
 			sdCollectorInputData := gcp.NewStackdriverCollectorInputData(ts, m.config.ProjectID, m.config.Zone, m.config.Region, m.config.Regions)
 			keyValues := mapper.mapTimeSeriesToKeyValuesPoints(ts, aligner)

x-pack/metricbeat/module/gcp/timeseries_metadata_collector.go

@@ -27,16 +27,22 @@ func NewStackdriverCollectorInputData(ts *monitoringpb.TimeSeries, projectID, zo
 
 // NewStackdriverMetadataServiceForTimeSeries apart from having a long name takes a time series object to return the
 // Stackdriver canonical Metadata extractor
-func NewStackdriverMetadataServiceForTimeSeries(ts *monitoringpb.TimeSeries) MetadataService {
+func NewStackdriverMetadataServiceForTimeSeries(ts *monitoringpb.TimeSeries, organizationID, organizationName string, projectName string) MetadataService {
 	return &StackdriverTimeSeriesMetadataCollector{
-		timeSeries: ts,
+		timeSeries:       ts,
+		organizationID:   organizationID,
+		organizationName: organizationName,
+		projectName:      projectName,
 	}
 }
 
 // StackdriverTimeSeriesMetadataCollector is the implementation of MetadataCollector to collect metrics from Stackdriver
 // common TimeSeries objects
 type StackdriverTimeSeriesMetadataCollector struct {
-	timeSeries *monitoringpb.TimeSeries
+	timeSeries       *monitoringpb.TimeSeries
+	organizationID   string
+	organizationName string
+	projectName      string
 }
 
 // Metadata parses a Timeseries object to return its metadata divided into "unknown" (first object) and ECS (second
@@ -53,14 +59,19 @@ func (s *StackdriverTimeSeriesMetadataCollector) Metadata(ctx context.Context, i
 
 	ecs := mapstr.M{
 		ECSCloud: mapstr.M{
-			ECSCloudAccount: mapstr.M{
-				ECSCloudAccountID:   accountID,
-				ECSCloudAccountName: accountID,
+			ECSCloudProject: mapstr.M{
+				ECSCloudID:   accountID,
+				ECSCloudName: s.projectName,
 			},
 			ECSCloudProvider: "gcp",
 		},
 	}
-
+	if s.organizationID != "" {
+		_, _ = ecs.Put(ECSCloud+"."+ECSCloudAccount+"."+ECSCloudID, s.organizationID)
+	}
+	if s.organizationName != "" {
+		_, _ = ecs.Put(ECSCloud+"."+ECSCloudAccount+"."+ECSCloudName, s.organizationName)
+	}
 	if availabilityZone != "" {
 		_, _ = ecs.Put(ECSCloud+"."+ECSCloudAvailabilityZone, availabilityZone)