[CIS GCP] Add a service-usage fetcher (#1340)

flavors/benchmark/gcp_test.go

@@ -59,6 +59,7 @@ func TestGCP_Initialize(t *testing.T) {
 			want: []string{
 				"gcp_cloud_assets_fetcher",
 				"gcp_monitoring_fetcher",
+				"gcp_service_usage_fetcher",
 			},
 		},
 		{
@@ -83,6 +84,7 @@ func TestGCP_Initialize(t *testing.T) {
 			want: []string{
 				"gcp_cloud_assets_fetcher",
 				"gcp_monitoring_fetcher",
+				"gcp_service_usage_fetcher",
 			},
 		},
 		{

resources/fetching/fetcher.go

@@ -53,6 +53,7 @@ const (
 	AccessAnalyzers           = "aws-access-analyzers"
 
 	GcpMonitoringType = "gcp-monitoring"
+	GcpServiceUsage   = "gcp-service-usage"
 
 	CloudIdentity          = "identity-management"
 	CloudCompute           = "cloud-compute"

resources/fetching/fetchers/gcp/service_usage_fetcher.go

@@ -0,0 +1,112 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package fetchers
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/elastic/elastic-agent-libs/logp"
+
+	"github.com/elastic/cloudbeat/resources/fetching"
+	"github.com/elastic/cloudbeat/resources/providers/gcplib"
+	"github.com/elastic/cloudbeat/resources/providers/gcplib/inventory"
+)
+
+type GcpServiceUsageFetcher struct {
+	log        *logp.Logger
+	resourceCh chan fetching.ResourceInfo
+	provider   inventory.ServiceAPI
+}
+
+type GcpServiceUsageAsset struct {
+	Type    string
+	subType string
+
+	Asset *inventory.ServiceUsageAsset `json:"assets,omitempty"`
+}
+
+func NewGcpServiceUsageFetcher(_ context.Context, log *logp.Logger, ch chan fetching.ResourceInfo, provider inventory.ServiceAPI) *GcpServiceUsageFetcher {
+	return &GcpServiceUsageFetcher{
+		log:        log,
+		resourceCh: ch,
+		provider:   provider,
+	}
+}
+
+func (f *GcpServiceUsageFetcher) Fetch(ctx context.Context, cMetadata fetching.CycleMetadata) error {
+	f.log.Info("Starting GcpServiceUsageFetcher.Fetch")
+
+	serviceUsageAssets, err := f.provider.ListServiceUsageAssets()
+	if err != nil {
+		return err
+	}
+
+	for _, serviceUsageAsset := range serviceUsageAssets {
+		select {
+		case <-ctx.Done():
+			f.log.Infof("GcpServiceUsageFetcher.ListMonitoringAssets context err: %s", ctx.Err().Error())
+			return nil
+		case f.resourceCh <- fetching.ResourceInfo{
+			CycleMetadata: cMetadata,
+			Resource: &GcpServiceUsageAsset{
+				Type:    fetching.MonitoringIdentity,
+				subType: fetching.GcpServiceUsage,
+				Asset:   serviceUsageAsset,
+			},
+		}:
+		}
+	}
+
+	return nil
+}
+
+func (f *GcpServiceUsageFetcher) Stop() {
+	f.provider.Close()
+}
+
+func (g *GcpServiceUsageAsset) GetMetadata() (fetching.ResourceMetadata, error) {
+	id := fmt.Sprintf("%s-%s", g.subType, g.Asset.Ecs.ProjectId)
+	return fetching.ResourceMetadata{
+		ID:      id,
+		Type:    g.Type,
+		SubType: g.subType,
+		Name:    id,
+		Region:  gcplib.GlobalRegion,
+	}, nil
+}
+
+func (g *GcpServiceUsageAsset) GetData() any {
+	return g.Asset
+}
+
+func (g *GcpServiceUsageAsset) GetElasticCommonData() (map[string]any, error) {
+	return map[string]any{
+		"cloud": map[string]any{
+			"provider": "gcp",
+			"account": map[string]any{
+				"id":   g.Asset.Ecs.ProjectId,
+				"name": g.Asset.Ecs.ProjectName,
+			},
+			"Organization": map[string]any{
+				"id":   g.Asset.Ecs.OrganizationId,
+				"name": g.Asset.Ecs.OrganizationName,
+			},
+		},
+	}, nil
+}

resources/fetching/fetchers/gcp/service_usage_fetcher_test.go

@@ -0,0 +1,204 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package fetchers
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"testing"
+
+	"cloud.google.com/go/asset/apiv1/assetpb"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/suite"
+
+	"github.com/elastic/cloudbeat/resources/fetching"
+	"github.com/elastic/cloudbeat/resources/providers/gcplib"
+	"github.com/elastic/cloudbeat/resources/providers/gcplib/inventory"
+	"github.com/elastic/cloudbeat/resources/utils/testhelper"
+)
+
+type GcpServiceUsageFetcherTestSuite struct {
+	suite.Suite
+
+	resourceCh chan fetching.ResourceInfo
+}
+
+func TestGcpServiceUsageFetcherTestSuite(t *testing.T) {
+	s := new(GcpServiceUsageFetcherTestSuite)
+
+	suite.Run(t, s)
+}
+
+func (s *GcpServiceUsageFetcherTestSuite) SetupTest() {
+	s.resourceCh = make(chan fetching.ResourceInfo, 50)
+}
+
+func (s *GcpServiceUsageFetcherTestSuite) TearDownTest() {
+	close(s.resourceCh)
+}
+
+func (s *GcpServiceUsageFetcherTestSuite) TestFetcher_Fetch_Success() {
+	ctx := context.Background()
+	mockInventoryService := &inventory.MockServiceAPI{}
+	fetcher := GcpServiceUsageFetcher{
+		log:        testhelper.NewLogger(s.T()),
+		resourceCh: s.resourceCh,
+		provider:   mockInventoryService,
+	}
+
+	mockInventoryService.On("ListServiceUsageAssets", mock.Anything).Return(
+		[]*inventory.ServiceUsageAsset{
+			{
+				Ecs: &fetching.EcsGcp{
+					Provider:         "gcp",
+					ProjectId:        "a",
+					ProjectName:      "a",
+					OrganizationId:   "a",
+					OrganizationName: "a",
+				},
+				Services: []*inventory.ExtendedGcpAsset{
+					{Asset: &assetpb.Asset{Name: "a", AssetType: "serviceusage.googleapis.com/Service"}},
+				},
+			},
+		}, nil,
+	)
+
+	err := fetcher.Fetch(ctx, fetching.CycleMetadata{})
+	s.NoError(err)
+	results := testhelper.CollectResources(s.resourceCh)
+
+	// ListMonitoringAssets mocked to return a single asset
+	s.Equal(1, len(results))
+}
+
+func (s *GcpServiceUsageFetcherTestSuite) TestFetcher_Fetch_Error() {
+	ctx := context.Background()
+	mockInventoryService := &inventory.MockServiceAPI{}
+	fetcher := GcpServiceUsageFetcher{
+		log:        testhelper.NewLogger(s.T()),
+		resourceCh: s.resourceCh,
+		provider:   mockInventoryService,
+	}
+
+	mockInventoryService.On("ListServiceUsageAssets", mock.Anything).Return(nil, errors.New("api call error"))
+
+	err := fetcher.Fetch(ctx, fetching.CycleMetadata{})
+	s.Error(err)
+}
+
+func TestServiceUsageResource_GetMetadata(t *testing.T) {
+	tests := []struct {
+		name     string
+		resource GcpServiceUsageAsset
+		want     fetching.ResourceMetadata
+		wantErr  bool
+	}{
+		{
+			name: "retrieve successfully service usage assets",
+			resource: GcpServiceUsageAsset{
+				Type:    fetching.MonitoringIdentity,
+				subType: fetching.GcpServiceUsage,
+				Asset: &inventory.ServiceUsageAsset{
+					Ecs: &fetching.EcsGcp{
+						ProjectId:        projectId,
+						ProjectName:      "a",
+						OrganizationId:   "a",
+						OrganizationName: "a",
+					},
+					Services: []*inventory.ExtendedGcpAsset{},
+				},
+			},
+			want: fetching.ResourceMetadata{
+				ID:      fmt.Sprintf("%s-%s", fetching.GcpServiceUsage, projectId),
+				Name:    fmt.Sprintf("%s-%s", fetching.GcpServiceUsage, projectId),
+				Type:    fetching.MonitoringIdentity,
+				SubType: fetching.GcpServiceUsage,
+				Region:  gcplib.GlobalRegion,
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := tt.resource.GetMetadata()
+
+			assert.NoError(t, err)
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}
+
+func TestGcpServiceUsageAsset_GetElasticCommonData(t *testing.T) {
+	type fields struct {
+		Type    string
+		subType string
+		Asset   *inventory.ServiceUsageAsset
+	}
+	tests := []struct {
+		name   string
+		fields fields
+		want   map[string]any
+	}{
+		{
+			name: "happy path",
+			fields: fields{
+				Type:    fetching.MonitoringIdentity,
+				subType: fetching.GcpServiceUsage,
+				Asset: &inventory.ServiceUsageAsset{
+					Ecs: &fetching.EcsGcp{
+						ProjectId:        projectId,
+						ProjectName:      "a",
+						OrganizationId:   "a",
+						OrganizationName: "a",
+					},
+					Services: []*inventory.ExtendedGcpAsset{
+						{Asset: &assetpb.Asset{Name: "a", AssetType: "serviceusage.googleapis.com/Service"}},
+					},
+				},
+			},
+			want: map[string]any{
+				"cloud": map[string]any{
+					"provider": "gcp",
+					"account": map[string]any{
+						"id":   projectId,
+						"name": "a",
+					},
+					"Organization": map[string]any{
+						"id":   "a",
+						"name": "a",
+					},
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			g := &GcpServiceUsageAsset{
+				Type:    tt.fields.Type,
+				subType: tt.fields.subType,
+				Asset:   tt.fields.Asset,
+			}
+
+			got, err := g.GetElasticCommonData()
+
+			assert.NoError(t, err)
+			assert.Equalf(t, tt.want, got, "GetElasticCommonData()")
+		})
+	}
+}

resources/fetching/preset/gcp_preset.go

@@ -38,5 +38,8 @@ func NewCisGcpFetchers(ctx context.Context, log *logp.Logger, ch chan fetching.R
 	monitoringFetcher := fetchers.NewGcpMonitoringFetcher(ctx, log, ch, inventory)
 	m["gcp_monitoring_fetcher"] = registry.RegisteredFetcher{Fetcher: monitoringFetcher}
 
+	serviceUsageFetcher := fetchers.NewGcpServiceUsageFetcher(ctx, log, ch, inventory)
+	m["gcp_service_usage_fetcher"] = registry.RegisteredFetcher{Fetcher: serviceUsageFetcher}
+
 	return m, nil
 }