From 83fa2d8d87a69f9224cce81a86cd48fe395dd8da Mon Sep 17 00:00:00 2001 From: Evgeniy Belyi Date: Thu, 21 Sep 2023 11:05:28 +0300 Subject: [PATCH] Implement Azure fetchers and asset provider (#1310) * Implement Azure fetchers and asset provider * Fixing subscription to env * up * CR comments --------- Co-authored-by: Orestis Floros --- go.mod | 5 +- go.sum | 6 +- resources/fetching/fetcher.go | 4 + .../fetching/fetchers/azure/assets_fetcher.go | 103 ++++++++++ .../fetchers/azure/assets_fetcher_test.go | 121 ++++++++++++ resources/fetching/preset/azure_factory.go | 37 ++++ .../providers/azurelib/inventory/asset.go | 23 +++ .../mock_provider_initializer_api.go | 113 +++++++++++ .../azurelib/inventory/mock_service_api.go | 103 ++++++++++ .../providers/azurelib/inventory/provider.go | 170 ++++++++++++++++ .../azurelib/inventory/provider_test.go | 184 ++++++++++++++++++ 11 files changed, 865 insertions(+), 4 deletions(-) create mode 100644 resources/fetching/fetchers/azure/assets_fetcher.go create mode 100644 resources/fetching/fetchers/azure/assets_fetcher_test.go create mode 100644 resources/fetching/preset/azure_factory.go create mode 100644 resources/providers/azurelib/inventory/asset.go create mode 100644 resources/providers/azurelib/inventory/mock_provider_initializer_api.go create mode 100644 resources/providers/azurelib/inventory/mock_service_api.go create mode 100644 resources/providers/azurelib/inventory/provider.go create mode 100644 resources/providers/azurelib/inventory/provider_test.go diff --git a/go.mod b/go.mod index 6f5f3c4177..a0ee6b4292 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( cloud.google.com/go/asset v1.14.1 cloud.google.com/go/iam v1.1.2 github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1 + github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resourcegraph/armresourcegraph v0.8.1 github.com/aquasecurity/go-dep-parser v0.0.0-20230605080024-b71d9356a6c6 github.com/aquasecurity/trivy v0.42.1 github.com/aquasecurity/trivy-db v0.0.0-20230515061101-378ab9ed302c @@ -94,7 +95,7 @@ require ( github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1 // indirect github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20221215162035-5330a85ea652 // indirect github.com/Azure/azure-sdk-for-go v68.0.0+incompatible // indirect - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0-beta.3 github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 // indirect github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect @@ -454,7 +455,7 @@ require ( google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect google.golang.org/grpc v1.55.0 // indirect - google.golang.org/protobuf v1.30.0 // indirect + google.golang.org/protobuf v1.30.0 gopkg.in/cheggaaa/pb.v1 v1.0.28 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/go.sum b/go.sum index be078b1983..69ce7ed2a5 100644 --- a/go.sum +++ b/go.sum @@ -221,12 +221,14 @@ github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9mo github.com/Azure/azure-sdk-for-go v56.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v68.0.0+incompatible h1:fcYLmCpyNYRnvJbPerq7U0hS+6+I79yEDJBqVNcqUzU= github.com/Azure/azure-sdk-for-go v68.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.1 h1:/iHxaJhsFr0+xVFfbMr5vxz848jyiWuIEDhYq3y5odY= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.7.1/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0-beta.3 h1:XV/HZqgyUQQAc1/UwYXK/p9PyPuDrprwSXcKARy183U= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.8.0-beta.3/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1 h1:LNHhpdK7hzUcx/k1LIcuh5k7k1LGIWLQfCjaneSj7Fc= github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.1/go.mod h1:uE9zaUfEQT/nbQjVi2IblCG9iaLtZsuYZ8ne+PuQ02M= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resourcegraph/armresourcegraph v0.8.1 h1:nGiU2ovpbtkcC3x+g/wNHV4S9TOIYe2/yOVAj3wiGHI= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resourcegraph/armresourcegraph v0.8.1/go.mod h1:T3ZgvD1aRKu12mEA0fU3PPvI7V0Nh0wzIdK0QMBhf0Y= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210608223527-2377c96fe795/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= diff --git a/resources/fetching/fetcher.go b/resources/fetching/fetcher.go index 71b8272563..285d85bcc8 100644 --- a/resources/fetching/fetcher.go +++ b/resources/fetching/fetcher.go @@ -69,6 +69,10 @@ const ( KeyManagement = "key-management" ProjectManagement = "project-management" DataProcessing = "data-processing" + + // Azure resource types + AzureVMType = "azure-vm" + AzureStorageAccountType = "azure-storage-account" ) // Fetcher represents a data fetcher. diff --git a/resources/fetching/fetchers/azure/assets_fetcher.go b/resources/fetching/fetchers/azure/assets_fetcher.go new file mode 100644 index 0000000000..6bd40b6d4f --- /dev/null +++ b/resources/fetching/fetchers/azure/assets_fetcher.go @@ -0,0 +1,103 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package fetchers + +import ( + "context" + + "github.com/elastic/elastic-agent-libs/logp" + "golang.org/x/exp/maps" + + "github.com/elastic/cloudbeat/resources/fetching" + "github.com/elastic/cloudbeat/resources/providers/azurelib/inventory" +) + +type AzureAssetsFetcher struct { + log *logp.Logger + resourceCh chan fetching.ResourceInfo + provider inventory.ServiceAPI +} + +type AzureResource struct { + Type string + SubType string + Asset inventory.AzureAsset `json:"asset,omitempty"` +} + +var AzureResourceTypes = map[string]string{ + inventory.VirtualMachineAssetType: fetching.AzureVMType, + inventory.StorageAccountAssetType: fetching.AzureStorageAccountType, +} + +func NewAzureAssetsFetcher(log *logp.Logger, ch chan fetching.ResourceInfo, provider inventory.ServiceAPI) *AzureAssetsFetcher { + return &AzureAssetsFetcher{ + log: log, + resourceCh: ch, + provider: provider, + } +} + +func (f *AzureAssetsFetcher) Fetch(ctx context.Context, cMetadata fetching.CycleMetadata) error { + f.log.Info("Starting AzureAssetsFetcher.Fetch") + // TODO: Maybe we should use a query per type instead of listing all assets in a single query + // This might be relevant if we'd like to fetch assets in parallel in order to evaluate a rule that uses multiple resources + assets, err := f.provider.ListAllAssetTypesByName(maps.Keys(AzureResourceTypes)) + if err != nil { + return err + } + + for _, asset := range assets { + select { + case <-ctx.Done(): + f.log.Infof("AzureAssetsFetcher.Fetch context err: %s", ctx.Err().Error()) + return nil + case f.resourceCh <- fetching.ResourceInfo{ + CycleMetadata: cMetadata, + Resource: &AzureResource{ + Type: AzureResourceTypes[asset.Type], + SubType: getAzureSubType(asset.Type), + Asset: asset, + }, + }: + } + } + + return nil +} + +func getAzureSubType(assetType string) string { + return "" +} + +func (f *AzureAssetsFetcher) Stop() {} + +func (r *AzureResource) GetData() any { + return r.Asset +} + +func (r *AzureResource) GetMetadata() (fetching.ResourceMetadata, error) { + return fetching.ResourceMetadata{ + ID: r.Asset.Id, + Type: r.Type, + SubType: r.SubType, + Name: r.Asset.Name, + Region: r.Asset.Location, + }, nil +} + +func (r *AzureResource) GetElasticCommonData() (map[string]any, error) { return nil, nil } diff --git a/resources/fetching/fetchers/azure/assets_fetcher_test.go b/resources/fetching/fetchers/azure/assets_fetcher_test.go new file mode 100644 index 0000000000..bdef81e625 --- /dev/null +++ b/resources/fetching/fetchers/azure/assets_fetcher_test.go @@ -0,0 +1,121 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package fetchers + +import ( + "context" + "testing" + + "github.com/samber/lo" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" + + "github.com/elastic/cloudbeat/resources/fetching" + "github.com/elastic/cloudbeat/resources/providers/azurelib/inventory" + "github.com/elastic/cloudbeat/resources/utils/testhelper" +) + +type AzureAssetsFetcherTestSuite struct { + suite.Suite + + resourceCh chan fetching.ResourceInfo +} + +func TestAzureAssetsFetcherTestSuite(t *testing.T) { + s := new(AzureAssetsFetcherTestSuite) + + suite.Run(t, s) +} + +func (s *AzureAssetsFetcherTestSuite) SetupTest() { + s.resourceCh = make(chan fetching.ResourceInfo, 50) +} + +func (s *AzureAssetsFetcherTestSuite) TearDownTest() { + close(s.resourceCh) +} + +func (s *AzureAssetsFetcherTestSuite) TestFetcher_Fetch() { + ctx := context.Background() + mockInventoryService := &inventory.MockServiceAPI{} + fetcher := AzureAssetsFetcher{ + log: testhelper.NewLogger(s.T()), + resourceCh: s.resourceCh, + provider: mockInventoryService, + } + + mockAssets := []inventory.AzureAsset{ + { + Id: "id1", + Name: "name1", + Location: "location1", + Properties: map[string]interface{}{"key1": "value1"}, + ResourceGroup: "rg1", + SubscriptionId: "subId1", + TenantId: "tenantId1", + Type: inventory.VirtualMachineAssetType, + }, + { + Id: "id2", + Name: "name2", + Location: "location2", + Properties: map[string]interface{}{"key2": "value2"}, + ResourceGroup: "rg2", + SubscriptionId: "subId2", + TenantId: "tenantId2", + Type: inventory.StorageAccountAssetType, + }, + } + + mockInventoryService.On("ListAllAssetTypesByName", mock.MatchedBy(func(assets []string) bool { + return true + })).Return( + mockAssets, nil, + ) + + err := fetcher.Fetch(ctx, fetching.CycleMetadata{}) + s.NoError(err) + results := testhelper.CollectResources(s.resourceCh) + + s.Equal(len(AzureResourceTypes), len(results)) + + lo.ForEach(results, func(r fetching.ResourceInfo, index int) { + data := r.GetData() + s.NotNil(data) + resource := data.(inventory.AzureAsset) + s.NotEmpty(resource) + s.Equal(mockAssets[index].Id, resource.Id) + s.Equal(mockAssets[index].Name, resource.Name) + s.Equal(mockAssets[index].Location, resource.Location) + s.Equal(mockAssets[index].Properties, resource.Properties) + s.Equal(mockAssets[index].ResourceGroup, resource.ResourceGroup) + s.Equal(mockAssets[index].SubscriptionId, resource.SubscriptionId) + s.Equal(mockAssets[index].TenantId, resource.TenantId) + s.Equal(mockAssets[index].Type, resource.Type) + meta, err := r.GetMetadata() + s.NoError(err) + s.NotNil(meta) + s.NoError(err) + s.NotEmpty(meta) + s.Equal(mockAssets[index].Id, meta.ID) + s.Equal(AzureResourceTypes[mockAssets[index].Type], meta.Type) + s.Equal("", meta.SubType) + s.Equal(mockAssets[index].Name, meta.Name) + s.Equal(mockAssets[index].Location, meta.Region) + }) +} diff --git a/resources/fetching/preset/azure_factory.go b/resources/fetching/preset/azure_factory.go new file mode 100644 index 0000000000..6ef7ebd4c4 --- /dev/null +++ b/resources/fetching/preset/azure_factory.go @@ -0,0 +1,37 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package preset + +import ( + "github.com/elastic/elastic-agent-libs/logp" + + "github.com/elastic/cloudbeat/resources/fetching" + fetchers "github.com/elastic/cloudbeat/resources/fetching/fetchers/azure" + "github.com/elastic/cloudbeat/resources/fetching/registry" + "github.com/elastic/cloudbeat/resources/providers/azurelib/inventory" +) + +func NewCisAzureFactory(log *logp.Logger, ch chan fetching.ResourceInfo, inventory inventory.ServiceAPI) (registry.FetchersMap, error) { + log.Infof("Initializing Azure fetchers") + m := make(registry.FetchersMap) + + assetsFetcher := fetchers.NewAzureAssetsFetcher(log, ch, inventory) + m["azure_cloud_assets_fetcher"] = registry.RegisteredFetcher{Fetcher: assetsFetcher} + + return m, nil +} diff --git a/resources/providers/azurelib/inventory/asset.go b/resources/providers/azurelib/inventory/asset.go new file mode 100644 index 0000000000..dc1069520b --- /dev/null +++ b/resources/providers/azurelib/inventory/asset.go @@ -0,0 +1,23 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package inventory + +const ( + VirtualMachineAssetType = "microsoft.compute/virtualmachines" + StorageAccountAssetType = "microsoft.storage/storageaccounts" +) diff --git a/resources/providers/azurelib/inventory/mock_provider_initializer_api.go b/resources/providers/azurelib/inventory/mock_provider_initializer_api.go new file mode 100644 index 0000000000..420cccf5bc --- /dev/null +++ b/resources/providers/azurelib/inventory/mock_provider_initializer_api.go @@ -0,0 +1,113 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by mockery v2.33.3. DO NOT EDIT. + +package inventory + +import ( + context "context" + + auth "github.com/elastic/cloudbeat/resources/providers/azurelib/auth" + + logp "github.com/elastic/elastic-agent-libs/logp" + + mock "github.com/stretchr/testify/mock" +) + +// MockProviderInitializerAPI is an autogenerated mock type for the ProviderInitializerAPI type +type MockProviderInitializerAPI struct { + mock.Mock +} + +type MockProviderInitializerAPI_Expecter struct { + mock *mock.Mock +} + +func (_m *MockProviderInitializerAPI) EXPECT() *MockProviderInitializerAPI_Expecter { + return &MockProviderInitializerAPI_Expecter{mock: &_m.Mock} +} + +// Init provides a mock function with given fields: ctx, log, azureConfig +func (_m *MockProviderInitializerAPI) Init(ctx context.Context, log *logp.Logger, azureConfig auth.AzureFactoryConfig) (ServiceAPI, error) { + ret := _m.Called(ctx, log, azureConfig) + + var r0 ServiceAPI + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *logp.Logger, auth.AzureFactoryConfig) (ServiceAPI, error)); ok { + return rf(ctx, log, azureConfig) + } + if rf, ok := ret.Get(0).(func(context.Context, *logp.Logger, auth.AzureFactoryConfig) ServiceAPI); ok { + r0 = rf(ctx, log, azureConfig) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(ServiceAPI) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *logp.Logger, auth.AzureFactoryConfig) error); ok { + r1 = rf(ctx, log, azureConfig) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockProviderInitializerAPI_Init_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Init' +type MockProviderInitializerAPI_Init_Call struct { + *mock.Call +} + +// Init is a helper method to define mock.On call +// - ctx context.Context +// - log *logp.Logger +// - azureConfig auth.AzureFactoryConfig +func (_e *MockProviderInitializerAPI_Expecter) Init(ctx interface{}, log interface{}, azureConfig interface{}) *MockProviderInitializerAPI_Init_Call { + return &MockProviderInitializerAPI_Init_Call{Call: _e.mock.On("Init", ctx, log, azureConfig)} +} + +func (_c *MockProviderInitializerAPI_Init_Call) Run(run func(ctx context.Context, log *logp.Logger, azureConfig auth.AzureFactoryConfig)) *MockProviderInitializerAPI_Init_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*logp.Logger), args[2].(auth.AzureFactoryConfig)) + }) + return _c +} + +func (_c *MockProviderInitializerAPI_Init_Call) Return(_a0 ServiceAPI, _a1 error) *MockProviderInitializerAPI_Init_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockProviderInitializerAPI_Init_Call) RunAndReturn(run func(context.Context, *logp.Logger, auth.AzureFactoryConfig) (ServiceAPI, error)) *MockProviderInitializerAPI_Init_Call { + _c.Call.Return(run) + return _c +} + +// NewMockProviderInitializerAPI creates a new instance of MockProviderInitializerAPI. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockProviderInitializerAPI(t interface { + mock.TestingT + Cleanup(func()) +}) *MockProviderInitializerAPI { + mock := &MockProviderInitializerAPI{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/resources/providers/azurelib/inventory/mock_service_api.go b/resources/providers/azurelib/inventory/mock_service_api.go new file mode 100644 index 0000000000..f07bd69992 --- /dev/null +++ b/resources/providers/azurelib/inventory/mock_service_api.go @@ -0,0 +1,103 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Code generated by mockery v2.33.3. DO NOT EDIT. + +package inventory + +import mock "github.com/stretchr/testify/mock" + +// MockServiceAPI is an autogenerated mock type for the ServiceAPI type +type MockServiceAPI struct { + mock.Mock +} + +type MockServiceAPI_Expecter struct { + mock *mock.Mock +} + +func (_m *MockServiceAPI) EXPECT() *MockServiceAPI_Expecter { + return &MockServiceAPI_Expecter{mock: &_m.Mock} +} + +// ListAllAssetTypesByName provides a mock function with given fields: assets +func (_m *MockServiceAPI) ListAllAssetTypesByName(assets []string) ([]AzureAsset, error) { + ret := _m.Called(assets) + + var r0 []AzureAsset + var r1 error + if rf, ok := ret.Get(0).(func([]string) ([]AzureAsset, error)); ok { + return rf(assets) + } + if rf, ok := ret.Get(0).(func([]string) []AzureAsset); ok { + r0 = rf(assets) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).([]AzureAsset) + } + } + + if rf, ok := ret.Get(1).(func([]string) error); ok { + r1 = rf(assets) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockServiceAPI_ListAllAssetTypesByName_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ListAllAssetTypesByName' +type MockServiceAPI_ListAllAssetTypesByName_Call struct { + *mock.Call +} + +// ListAllAssetTypesByName is a helper method to define mock.On call +// - assets []string +func (_e *MockServiceAPI_Expecter) ListAllAssetTypesByName(assets interface{}) *MockServiceAPI_ListAllAssetTypesByName_Call { + return &MockServiceAPI_ListAllAssetTypesByName_Call{Call: _e.mock.On("ListAllAssetTypesByName", assets)} +} + +func (_c *MockServiceAPI_ListAllAssetTypesByName_Call) Run(run func(assets []string)) *MockServiceAPI_ListAllAssetTypesByName_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].([]string)) + }) + return _c +} + +func (_c *MockServiceAPI_ListAllAssetTypesByName_Call) Return(_a0 []AzureAsset, _a1 error) *MockServiceAPI_ListAllAssetTypesByName_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockServiceAPI_ListAllAssetTypesByName_Call) RunAndReturn(run func([]string) ([]AzureAsset, error)) *MockServiceAPI_ListAllAssetTypesByName_Call { + _c.Call.Return(run) + return _c +} + +// NewMockServiceAPI creates a new instance of MockServiceAPI. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockServiceAPI(t interface { + mock.TestingT + Cleanup(func()) +}) *MockServiceAPI { + mock := &MockServiceAPI{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/resources/providers/azurelib/inventory/provider.go b/resources/providers/azurelib/inventory/provider.go new file mode 100644 index 0000000000..6c4188fcb6 --- /dev/null +++ b/resources/providers/azurelib/inventory/provider.go @@ -0,0 +1,170 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package inventory + +import ( + "bytes" + "context" + "os" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resourcegraph/armresourcegraph" + "github.com/elastic/elastic-agent-libs/logp" + + "github.com/elastic/cloudbeat/resources/providers/azurelib/auth" +) + +type Provider struct { + log *logp.Logger + client *AzureClientWrapper + ctx context.Context + Config auth.AzureFactoryConfig +} + +type ProviderInitializer struct{} + +type AzureClientWrapper struct { + AssetQuery func(ctx context.Context, query armresourcegraph.QueryRequest, options *armresourcegraph.ClientResourcesOptions) (armresourcegraph.ClientResourcesResponse, error) +} + +type AzureAsset struct { + Id string `json:"id,omitempty"` + Name string `json:"name,omitempty"` + Location string `json:"location,omitempty"` + Properties map[string]any `json:"properties,omitempty"` + ResourceGroup string `json:"resource_group,omitempty"` + SubscriptionId string `json:"subscription_id,omitempty"` + TenantId string `json:"tenant_id,omitempty"` + Type string `json:"type,omitempty"` +} + +type ServiceAPI interface { + // ListAllAssetTypesByName List all content types of the given assets types + ListAllAssetTypesByName(assets []string) ([]AzureAsset, error) +} + +type ProviderInitializerAPI interface { + // Init initializes the Azure asset client + Init(ctx context.Context, log *logp.Logger, azureConfig auth.AzureFactoryConfig) (ServiceAPI, error) +} + +func (p *ProviderInitializer) Init(ctx context.Context, log *logp.Logger, azureConfig auth.AzureFactoryConfig) (ServiceAPI, error) { + clientFactory, err := armresourcegraph.NewClientFactory(azureConfig.Credentials, nil) + if err != nil { + return nil, err + } + + client := clientFactory.NewClient() + + // We wrap the client so we can mock it in tests + wrapper := &AzureClientWrapper{ + AssetQuery: func(ctx context.Context, query armresourcegraph.QueryRequest, options *armresourcegraph.ClientResourcesOptions) (armresourcegraph.ClientResourcesResponse, error) { + return client.Resources(ctx, query, options) + }, + } + + return &Provider{ + Config: azureConfig, + client: wrapper, + log: log, + ctx: ctx, + }, nil +} + +func (p *Provider) ListAllAssetTypesByName(assets []string) ([]AzureAsset, error) { + p.log.Infof("Listing Azure assets: %v", assets) + var resourceAssets []AzureAsset + + query := armresourcegraph.QueryRequest{ + Query: to.Ptr(generateQuery(assets)), + Options: &armresourcegraph.QueryRequestOptions{ + ResultFormat: to.Ptr(armresourcegraph.ResultFormatObjectArray), + }, + Subscriptions: []*string{ + // TODO: Populate from config or query (not sensitive but still don't want to commit) + to.Ptr(os.Getenv("AZURE_SUBSCRIPTION_ID"))}, + } + + resourceAssets, err := p.runPaginatedQuery(query) + if err != nil { + return nil, err + } + + return resourceAssets, nil +} + +func getAssetFromData(data map[string]any) AzureAsset { + properties, _ := data["properties"].(map[string]any) + + return AzureAsset{ + Id: getString(data, "id"), + Name: getString(data, "name"), + Location: getString(data, "location"), + Properties: properties, + ResourceGroup: getString(data, "resourceGroup"), + SubscriptionId: getString(data, "subscriptionId"), + TenantId: getString(data, "tenantId"), + Type: getString(data, "type"), + } +} + +func getString(data map[string]any, key string) string { + value, _ := data[key].(string) + return value +} + +func generateQuery(assets []string) string { + var query bytes.Buffer + query.WriteString("Resources") + for index, asset := range assets { + if index == 0 { + query.WriteString(" | where type == '") + } else { + query.WriteString(" or type == '") + } + query.WriteString(asset) + query.WriteString("'") + } + return query.String() +} + +func (p *Provider) runPaginatedQuery(query armresourcegraph.QueryRequest) ([]AzureAsset, error) { + var resourceAssets []AzureAsset + + for { + response, err := p.client.AssetQuery(p.ctx, query, nil) + if err != nil { + return nil, err + } + + for _, asset := range response.Data.([]interface{}) { + structuredAsset := getAssetFromData(asset.(map[string]any)) + resourceAssets = append(resourceAssets, structuredAsset) + } + + if *response.ResultTruncated == *to.Ptr(armresourcegraph.ResultTruncatedTrue) && + response.SkipToken != nil && + *response.SkipToken != "" { + query.Options.SkipToken = response.SkipToken + } else { + break + } + } + + return resourceAssets, nil +} diff --git a/resources/providers/azurelib/inventory/provider_test.go b/resources/providers/azurelib/inventory/provider_test.go new file mode 100644 index 0000000000..d00a29fcee --- /dev/null +++ b/resources/providers/azurelib/inventory/provider_test.go @@ -0,0 +1,184 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package inventory + +import ( + "context" + "fmt" + "testing" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/resourcegraph/armresourcegraph" + "github.com/elastic/elastic-agent-libs/logp" + "github.com/samber/lo" + "github.com/stretchr/testify/suite" + + "github.com/elastic/cloudbeat/resources/providers/azurelib/auth" +) + +type ProviderTestSuite struct { + suite.Suite + ctx context.Context + logger *logp.Logger + mockedClient *AzureClientWrapper +} + +var nonTruncatedResponse = armresourcegraph.QueryResponse{ + Count: to.Ptr(int64(1)), + Data: []any{ + map[string]any{ + "id": "3", + "name": "3", + "location": "3", + "properties": map[string]any{"test": "test"}, + "resourceGroup": "3", + "subscriptionId": "3", + "tenantId": "3", + "type": "3", + }, + }, + ResultTruncated: to.Ptr(armresourcegraph.ResultTruncatedFalse), +} + +var truncatedResponse = armresourcegraph.QueryResponse{ + Count: to.Ptr(int64(2)), + Data: []any{ + map[string]any{ + "id": "1", + "name": "1", + "location": "1", + "properties": map[string]any{"test": "test"}, + "resourceGroup": "1", + "subscriptionId": "1", + "tenantId": "1", + "type": "1", + }, + map[string]any{ + "id": "2", + "name": "2", + "location": "2", + "properties": map[string]any{"test": "test"}, + "resourceGroup": "2", + "subscriptionId": "2", + "tenantId": "2", + "type": "2", + }, + }, + ResultTruncated: to.Ptr(armresourcegraph.ResultTruncatedTrue), + SkipToken: to.Ptr("token"), +} + +func TestInventoryProviderTestSuite(t *testing.T) { + s := new(ProviderTestSuite) + + suite.Run(t, s) +} + +func (s *ProviderTestSuite) SetupTest() { + s.ctx = context.Background() + s.logger = logp.NewLogger("test") + s.mockedClient = &AzureClientWrapper{ + AssetQuery: func(ctx context.Context, query armresourcegraph.QueryRequest, options *armresourcegraph.ClientResourcesOptions) (armresourcegraph.ClientResourcesResponse, error) { + if query.Options.SkipToken != nil && *query.Options.SkipToken != "" { + return armresourcegraph.ClientResourcesResponse{ + QueryResponse: nonTruncatedResponse, + }, nil + } else { + return armresourcegraph.ClientResourcesResponse{ + QueryResponse: truncatedResponse, + }, nil + } + }, + } +} + +func (s *ProviderTestSuite) TestGetString() { + tests := []struct { + name string + data map[string]any + key string + want string + }{ + { + name: "nil map", + data: nil, + key: "key", + want: "", + }, + { + name: "key does not exist", + data: map[string]any{"key": "value"}, + key: "other-key", + want: "", + }, + { + name: "wrong type", + data: map[string]any{"key": 1}, + key: "key", + want: "", + }, + { + name: "correct value", + data: map[string]any{"key": "value", "other-key": 1}, + key: "key", + want: "value", + }, + } + for _, tt := range tests { + s.Assert().Equal(tt.want, getString(tt.data, tt.key), "getString(%v, %s) = %s", tt.data, tt.key, tt.want) + } +} + +func (s *ProviderTestSuite) TestProviderInit() { + initMock := new(MockProviderInitializerAPI) + azureConfig := auth.AzureFactoryConfig{ + Credentials: &azidentity.DefaultAzureCredential{}, + } + + initMock.On("Init", s.ctx, s.logger, azureConfig).Return(&Provider{}, nil).Once() + provider, err := initMock.Init(s.ctx, s.logger, azureConfig) + s.Assert().NoError(err) + s.Assert().NotNil(provider) +} + +func (s *ProviderTestSuite) TestListAllAssetTypesByName() { + provider := &Provider{ + log: s.logger, + client: s.mockedClient, + ctx: s.ctx, + Config: auth.AzureFactoryConfig{ + Credentials: &azidentity.DefaultAzureCredential{}, + }, + } + + values, err := provider.ListAllAssetTypesByName([]string{"test"}) + s.Assert().NoError(err) + s.Assert().Equal(int(*nonTruncatedResponse.Count+*truncatedResponse.Count), len(values)) + lo.ForEach(values, func(r AzureAsset, index int) { + strIndex := fmt.Sprintf("%d", index+1) + s.Assert().Equal(r.Id, strIndex) + s.Assert().Equal(r.Name, strIndex) + s.Assert().Equal(r.Location, strIndex) + s.Assert().Equal(r.ResourceGroup, strIndex) + s.Assert().Equal(r.SubscriptionId, strIndex) + s.Assert().Equal(r.TenantId, strIndex) + s.Assert().Equal(r.Type, strIndex) + s.Assert().Equal(r.Properties, map[string]any{"test": "test"}) + }) +}