diff --git a/flavors/benchmark/gcp_test.go b/flavors/benchmark/gcp_test.go index 27ef77b78d..9a146df8fd 100644 --- a/flavors/benchmark/gcp_test.go +++ b/flavors/benchmark/gcp_test.go @@ -59,6 +59,7 @@ func TestGCP_Initialize(t *testing.T) { want: []string{ "gcp_cloud_assets_fetcher", "gcp_monitoring_fetcher", + "gcp_service_usage_fetcher", }, }, { @@ -83,6 +84,7 @@ func TestGCP_Initialize(t *testing.T) { want: []string{ "gcp_cloud_assets_fetcher", "gcp_monitoring_fetcher", + "gcp_service_usage_fetcher", }, }, { diff --git a/resources/fetching/fetcher.go b/resources/fetching/fetcher.go index e251d3895b..71b8272563 100644 --- a/resources/fetching/fetcher.go +++ b/resources/fetching/fetcher.go @@ -53,6 +53,7 @@ const ( AccessAnalyzers = "aws-access-analyzers" GcpMonitoringType = "gcp-monitoring" + GcpServiceUsage = "gcp-service-usage" CloudIdentity = "identity-management" CloudCompute = "cloud-compute" diff --git a/resources/fetching/fetchers/gcp/service_usage_fetcher.go b/resources/fetching/fetchers/gcp/service_usage_fetcher.go new file mode 100644 index 0000000000..0f2278302a --- /dev/null +++ b/resources/fetching/fetchers/gcp/service_usage_fetcher.go @@ -0,0 +1,112 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package fetchers + +import ( + "context" + "fmt" + + "github.com/elastic/elastic-agent-libs/logp" + + "github.com/elastic/cloudbeat/resources/fetching" + "github.com/elastic/cloudbeat/resources/providers/gcplib" + "github.com/elastic/cloudbeat/resources/providers/gcplib/inventory" +) + +type GcpServiceUsageFetcher struct { + log *logp.Logger + resourceCh chan fetching.ResourceInfo + provider inventory.ServiceAPI +} + +type GcpServiceUsageAsset struct { + Type string + subType string + + Asset *inventory.ServiceUsageAsset `json:"assets,omitempty"` +} + +func NewGcpServiceUsageFetcher(_ context.Context, log *logp.Logger, ch chan fetching.ResourceInfo, provider inventory.ServiceAPI) *GcpServiceUsageFetcher { + return &GcpServiceUsageFetcher{ + log: log, + resourceCh: ch, + provider: provider, + } +} + +func (f *GcpServiceUsageFetcher) Fetch(ctx context.Context, cMetadata fetching.CycleMetadata) error { + f.log.Info("Starting GcpServiceUsageFetcher.Fetch") + + serviceUsageAssets, err := f.provider.ListServiceUsageAssets() + if err != nil { + return err + } + + for _, serviceUsageAsset := range serviceUsageAssets { + select { + case <-ctx.Done(): + f.log.Infof("GcpServiceUsageFetcher.ListMonitoringAssets context err: %s", ctx.Err().Error()) + return nil + case f.resourceCh <- fetching.ResourceInfo{ + CycleMetadata: cMetadata, + Resource: &GcpServiceUsageAsset{ + Type: fetching.MonitoringIdentity, + subType: fetching.GcpServiceUsage, + Asset: serviceUsageAsset, + }, + }: + } + } + + return nil +} + +func (f *GcpServiceUsageFetcher) Stop() { + f.provider.Close() +} + +func (g *GcpServiceUsageAsset) GetMetadata() (fetching.ResourceMetadata, error) { + id := fmt.Sprintf("%s-%s", g.subType, g.Asset.Ecs.ProjectId) + return fetching.ResourceMetadata{ + ID: id, + Type: g.Type, + SubType: g.subType, + Name: id, + Region: gcplib.GlobalRegion, + }, nil +} + +func (g *GcpServiceUsageAsset) GetData() any { + return g.Asset +} + +func (g *GcpServiceUsageAsset) GetElasticCommonData() (map[string]any, error) { + return map[string]any{ + "cloud": map[string]any{ + "provider": "gcp", + "account": map[string]any{ + "id": g.Asset.Ecs.ProjectId, + "name": g.Asset.Ecs.ProjectName, + }, + "Organization": map[string]any{ + "id": g.Asset.Ecs.OrganizationId, + "name": g.Asset.Ecs.OrganizationName, + }, + }, + }, nil +} diff --git a/resources/fetching/fetchers/gcp/service_usage_fetcher_test.go b/resources/fetching/fetchers/gcp/service_usage_fetcher_test.go new file mode 100644 index 0000000000..3a05d2ca04 --- /dev/null +++ b/resources/fetching/fetchers/gcp/service_usage_fetcher_test.go @@ -0,0 +1,204 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package fetchers + +import ( + "context" + "errors" + "fmt" + "testing" + + "cloud.google.com/go/asset/apiv1/assetpb" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" + + "github.com/elastic/cloudbeat/resources/fetching" + "github.com/elastic/cloudbeat/resources/providers/gcplib" + "github.com/elastic/cloudbeat/resources/providers/gcplib/inventory" + "github.com/elastic/cloudbeat/resources/utils/testhelper" +) + +type GcpServiceUsageFetcherTestSuite struct { + suite.Suite + + resourceCh chan fetching.ResourceInfo +} + +func TestGcpServiceUsageFetcherTestSuite(t *testing.T) { + s := new(GcpServiceUsageFetcherTestSuite) + + suite.Run(t, s) +} + +func (s *GcpServiceUsageFetcherTestSuite) SetupTest() { + s.resourceCh = make(chan fetching.ResourceInfo, 50) +} + +func (s *GcpServiceUsageFetcherTestSuite) TearDownTest() { + close(s.resourceCh) +} + +func (s *GcpServiceUsageFetcherTestSuite) TestFetcher_Fetch_Success() { + ctx := context.Background() + mockInventoryService := &inventory.MockServiceAPI{} + fetcher := GcpServiceUsageFetcher{ + log: testhelper.NewLogger(s.T()), + resourceCh: s.resourceCh, + provider: mockInventoryService, + } + + mockInventoryService.On("ListServiceUsageAssets", mock.Anything).Return( + []*inventory.ServiceUsageAsset{ + { + Ecs: &fetching.EcsGcp{ + Provider: "gcp", + ProjectId: "a", + ProjectName: "a", + OrganizationId: "a", + OrganizationName: "a", + }, + Services: []*inventory.ExtendedGcpAsset{ + {Asset: &assetpb.Asset{Name: "a", AssetType: "serviceusage.googleapis.com/Service"}}, + }, + }, + }, nil, + ) + + err := fetcher.Fetch(ctx, fetching.CycleMetadata{}) + s.NoError(err) + results := testhelper.CollectResources(s.resourceCh) + + // ListMonitoringAssets mocked to return a single asset + s.Equal(1, len(results)) +} + +func (s *GcpServiceUsageFetcherTestSuite) TestFetcher_Fetch_Error() { + ctx := context.Background() + mockInventoryService := &inventory.MockServiceAPI{} + fetcher := GcpServiceUsageFetcher{ + log: testhelper.NewLogger(s.T()), + resourceCh: s.resourceCh, + provider: mockInventoryService, + } + + mockInventoryService.On("ListServiceUsageAssets", mock.Anything).Return(nil, errors.New("api call error")) + + err := fetcher.Fetch(ctx, fetching.CycleMetadata{}) + s.Error(err) +} + +func TestServiceUsageResource_GetMetadata(t *testing.T) { + tests := []struct { + name string + resource GcpServiceUsageAsset + want fetching.ResourceMetadata + wantErr bool + }{ + { + name: "retrieve successfully service usage assets", + resource: GcpServiceUsageAsset{ + Type: fetching.MonitoringIdentity, + subType: fetching.GcpServiceUsage, + Asset: &inventory.ServiceUsageAsset{ + Ecs: &fetching.EcsGcp{ + ProjectId: projectId, + ProjectName: "a", + OrganizationId: "a", + OrganizationName: "a", + }, + Services: []*inventory.ExtendedGcpAsset{}, + }, + }, + want: fetching.ResourceMetadata{ + ID: fmt.Sprintf("%s-%s", fetching.GcpServiceUsage, projectId), + Name: fmt.Sprintf("%s-%s", fetching.GcpServiceUsage, projectId), + Type: fetching.MonitoringIdentity, + SubType: fetching.GcpServiceUsage, + Region: gcplib.GlobalRegion, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := tt.resource.GetMetadata() + + assert.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestGcpServiceUsageAsset_GetElasticCommonData(t *testing.T) { + type fields struct { + Type string + subType string + Asset *inventory.ServiceUsageAsset + } + tests := []struct { + name string + fields fields + want map[string]any + }{ + { + name: "happy path", + fields: fields{ + Type: fetching.MonitoringIdentity, + subType: fetching.GcpServiceUsage, + Asset: &inventory.ServiceUsageAsset{ + Ecs: &fetching.EcsGcp{ + ProjectId: projectId, + ProjectName: "a", + OrganizationId: "a", + OrganizationName: "a", + }, + Services: []*inventory.ExtendedGcpAsset{ + {Asset: &assetpb.Asset{Name: "a", AssetType: "serviceusage.googleapis.com/Service"}}, + }, + }, + }, + want: map[string]any{ + "cloud": map[string]any{ + "provider": "gcp", + "account": map[string]any{ + "id": projectId, + "name": "a", + }, + "Organization": map[string]any{ + "id": "a", + "name": "a", + }, + }, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := &GcpServiceUsageAsset{ + Type: tt.fields.Type, + subType: tt.fields.subType, + Asset: tt.fields.Asset, + } + + got, err := g.GetElasticCommonData() + + assert.NoError(t, err) + assert.Equalf(t, tt.want, got, "GetElasticCommonData()") + }) + } +} diff --git a/resources/fetching/preset/gcp_preset.go b/resources/fetching/preset/gcp_preset.go index 44d40a56ae..ff18213755 100644 --- a/resources/fetching/preset/gcp_preset.go +++ b/resources/fetching/preset/gcp_preset.go @@ -38,5 +38,8 @@ func NewCisGcpFetchers(ctx context.Context, log *logp.Logger, ch chan fetching.R monitoringFetcher := fetchers.NewGcpMonitoringFetcher(ctx, log, ch, inventory) m["gcp_monitoring_fetcher"] = registry.RegisteredFetcher{Fetcher: monitoringFetcher} + serviceUsageFetcher := fetchers.NewGcpServiceUsageFetcher(ctx, log, ch, inventory) + m["gcp_service_usage_fetcher"] = registry.RegisteredFetcher{Fetcher: serviceUsageFetcher} + return m, nil } diff --git a/resources/providers/gcplib/inventory/mock_service_api.go b/resources/providers/gcplib/inventory/mock_service_api.go index 463f83f5fc..d5bec39bfa 100644 --- a/resources/providers/gcplib/inventory/mock_service_api.go +++ b/resources/providers/gcplib/inventory/mock_service_api.go @@ -183,6 +183,59 @@ func (_c *MockServiceAPI_ListMonitoringAssets_Call) RunAndReturn(run func(map[st return _c } +// ListServiceUsageAssets provides a mock function with given fields: +func (_m *MockServiceAPI) ListServiceUsageAssets() ([]*ServiceUsageAsset, error) { + ret := _m.Called() + + var r0 []*ServiceUsageAsset + var r1 error + if rf, ok := ret.Get(0).(func() ([]*ServiceUsageAsset, error)); ok { + return rf() + } + if rf, ok := ret.Get(0).(func() []*ServiceUsageAsset); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]*ServiceUsageAsset) + } + } + + if rf, ok := ret.Get(1).(func() error); ok { + r1 = rf() + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockServiceAPI_ListServiceUsageAssets_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListServiceUsageAssets' +type MockServiceAPI_ListServiceUsageAssets_Call struct { + *mock.Call +} + +// ListServiceUsageAssets is a helper method to define mock.On call +func (_e *MockServiceAPI_Expecter) ListServiceUsageAssets() *MockServiceAPI_ListServiceUsageAssets_Call { + return &MockServiceAPI_ListServiceUsageAssets_Call{Call: _e.mock.On("ListServiceUsageAssets")} +} + +func (_c *MockServiceAPI_ListServiceUsageAssets_Call) Run(run func()) *MockServiceAPI_ListServiceUsageAssets_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *MockServiceAPI_ListServiceUsageAssets_Call) Return(_a0 []*ServiceUsageAsset, _a1 error) *MockServiceAPI_ListServiceUsageAssets_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockServiceAPI_ListServiceUsageAssets_Call) RunAndReturn(run func() ([]*ServiceUsageAsset, error)) *MockServiceAPI_ListServiceUsageAssets_Call { + _c.Call.Return(run) + return _c +} + type mockConstructorTestingTNewMockServiceAPI interface { mock.TestingT Cleanup(func()) diff --git a/resources/providers/gcplib/inventory/provider.go b/resources/providers/gcplib/inventory/provider.go index fd89ebe934..b3ac5e7854 100644 --- a/resources/providers/gcplib/inventory/provider.go +++ b/resources/providers/gcplib/inventory/provider.go @@ -60,11 +60,16 @@ type ResourceManagerWrapper struct { } type MonitoringAsset struct { - Ecs *fetching.EcsGcp `json:"project_id,omitempty"` + Ecs *fetching.EcsGcp LogMetrics []*ExtendedGcpAsset `json:"log_metrics,omitempty"` Alerts []*ExtendedGcpAsset `json:"alerts,omitempty"` } +type ServiceUsageAsset struct { + Ecs *fetching.EcsGcp + Services []*ExtendedGcpAsset `json:"services,omitempty"` +} + type ExtendedGcpAsset struct { *assetpb.Asset Ecs *fetching.EcsGcp @@ -83,6 +88,9 @@ type ServiceAPI interface { // ListMonitoringAssets List all monitoring assets by project id ListMonitoringAssets(map[string][]string) ([]*MonitoringAsset, error) + // ListServiceUsageAssets returns a list of service usage assets grouped by project id + ListServiceUsageAssets() ([]*ServiceUsageAsset, error) + // Close the GCP asset client Close() error } @@ -195,6 +203,17 @@ func (p *Provider) ListMonitoringAssets(monitoringAssetTypes map[string][]string return monitoringAssets, nil } +// ListServiceUsageAssets returns a list of service usage assets grouped by project id +func (p *Provider) ListServiceUsageAssets() ([]*ServiceUsageAsset, error) { + services, err := p.ListAllAssetTypesByName([]string{"serviceusage.googleapis.com/Service"}) + if err != nil { + return nil, err + } + + assets := getServiceUsageAssetsByProject(services, p.log) + return assets, nil +} + func (p *Provider) Close() error { return p.inventory.Close() } @@ -226,6 +245,32 @@ func getMonitoringAssetsByProject(assets []*ExtendedGcpAsset, log *logp.Logger) return monitoringAssets } +// returns monitoring assets grouped by project id +// single project for project scoped accounts +// multiple projects for organization scoped accounts +func getServiceUsageAssetsByProject(assets []*ExtendedGcpAsset, log *logp.Logger) []*ServiceUsageAsset { + assetsByProject := lo.GroupBy(assets, func(asset *ExtendedGcpAsset) string { return asset.Ecs.ProjectId }) + var serviceUsageAssets []*ServiceUsageAsset + for projectId, projectAssets := range assetsByProject { + projectName, organizationId, organizationName, err := getProjectAssetsMetadata(projectAssets) + if err != nil { + log.Error(err) + continue + } + serviceUsageAssets = append(serviceUsageAssets, &ServiceUsageAsset{ + Services: projectAssets, + Ecs: &fetching.EcsGcp{ + Provider: "gcp", + ProjectId: projectId, + ProjectName: projectName, + OrganizationId: organizationId, + OrganizationName: organizationName, + }, + }) + } + return serviceUsageAssets +} + func getAllAssets(log *logp.Logger, it Iterator) []*assetpb.Asset { results := make([]*assetpb.Asset, 0) for { diff --git a/resources/providers/gcplib/inventory/provider_test.go b/resources/providers/gcplib/inventory/provider_test.go index 1fe282a3d6..a878b5cf61 100644 --- a/resources/providers/gcplib/inventory/provider_test.go +++ b/resources/providers/gcplib/inventory/provider_test.go @@ -183,3 +183,80 @@ func (s *ProviderTestSuite) TestListMonitoringAssets() { s.Assert().Equal(value[1].Ecs.OrganizationId, "1") s.Assert().Equal(value[1].Ecs.OrganizationName, "OrganizationName1") } + +func (s *ProviderTestSuite) TestListServiceUsageAssets() { + expected := []*ServiceUsageAsset{ + { + Ecs: &fetching.EcsGcp{ + Provider: "gcp", + ProjectId: "1", + ProjectName: "ProjectName1", + OrganizationId: "1", + OrganizationName: "OrganizationName1", + }, + Services: []*ExtendedGcpAsset{{ + Asset: &assetpb.Asset{Name: "ServiceUsage1", Resource: &assetpb.Resource{}, IamPolicy: nil, Ancestors: []string{"projects/1", "organizations/1"}, AssetType: "serviceusage.googleapis.com/Service"}, + Ecs: &fetching.EcsGcp{ProjectId: "1", ProjectName: "ProjectName1", OrganizationId: "1", OrganizationName: "OrganizationName1"}, + }}, + }, + { + Ecs: &fetching.EcsGcp{ + Provider: "gcp", + ProjectId: "2", + ProjectName: "ProjectName2", + OrganizationId: "1", + OrganizationName: "OrganizationName1", + }, + Services: []*ExtendedGcpAsset{{ + Asset: &assetpb.Asset{Name: "ServiceUsage2", Resource: nil, IamPolicy: &iampb.Policy{}, Ancestors: []string{"projects/2", "organizations/1"}, AssetType: "serviceusage.googleapis.com/Service"}, + Ecs: &fetching.EcsGcp{ProjectId: "2", ProjectName: "ProjectName2", OrganizationId: "1", OrganizationName: "OrganizationName1"}, + }}, + }, + } + + ctx := context.Background() + mockIterator := new(MockIterator) + provider := &Provider{ + log: logp.NewLogger("test"), + inventory: &AssetsInventoryWrapper{ + Close: func() error { return nil }, + ListAssets: func(ctx context.Context, req *assetpb.ListAssetsRequest, opts ...gax.CallOption) Iterator { + return mockIterator + }, + }, + ctx: ctx, + config: auth.GcpFactoryConfig{ + Parent: "projects/1", + ClientOpts: []option.ClientOption{}, + }, + crm: &ResourceManagerWrapper{ + getProjectDisplayName: func(ctx context.Context, parent string) string { + if parent == "projects/1" { + return "ProjectName1" + } + return "ProjectName2" + }, + getOrganizationDisplayName: func(ctx context.Context, parent string) string { + return "OrganizationName1" + }, + }, + crmCache: make(map[string]*fetching.EcsGcp), + } + + // asset's resource + mockIterator.On("Next").Return(&assetpb.Asset{Name: "ServiceUsage1", Resource: &assetpb.Resource{}, Ancestors: []string{"projects/1", "organizations/1"}, AssetType: "serviceusage.googleapis.com/Service"}, nil).Once() + mockIterator.On("Next").Return(&assetpb.Asset{Name: "ServiceUsage2", Resource: nil, Ancestors: []string{"projects/2", "organizations/1"}, AssetType: "serviceusage.googleapis.com/Service"}, nil).Once() + mockIterator.On("Next").Return(&assetpb.Asset{}, iterator.Done).Once() + + // asset's iam policy + mockIterator.On("Next").Return(&assetpb.Asset{Name: "ServiceUsage1", IamPolicy: nil, Ancestors: []string{"projects/1", "organizations/1"}, AssetType: "serviceusage.googleapis.com/Service"}, nil).Once() + mockIterator.On("Next").Return(&assetpb.Asset{Name: "ServiceUsage2", IamPolicy: &iampb.Policy{}, Ancestors: []string{"projects/2", "organizations/1"}, AssetType: "serviceusage.googleapis.com/Service"}, nil).Once() + mockIterator.On("Next").Return(&assetpb.Asset{}, iterator.Done).Once() + + values, err := provider.ListServiceUsageAssets() + s.Assert().NoError(err) + + // 2 assets, 1 for each project + s.Assert().Equal(2, len(values)) + s.ElementsMatch(expected, values) +}