From d597d3dba845210d05efcb196859a20a242da7be Mon Sep 17 00:00:00 2001 From: gracewilcox <43627800+gracewilcox@users.noreply.github.com> Date: Mon, 17 Jul 2023 13:09:12 -0700 Subject: [PATCH] [monitor] azingest (#21155) --- eng/config.json | 4 + sdk/monitor/azingest/CHANGELOG.md | 4 + sdk/monitor/azingest/LICENSE.txt | 21 ++ sdk/monitor/azingest/README.md | 119 +++++++++++ sdk/monitor/azingest/assets.json | 6 + sdk/monitor/azingest/autorest.md | 60 ++++++ sdk/monitor/azingest/build.go | 12 ++ sdk/monitor/azingest/ci.yml | 30 +++ sdk/monitor/azingest/client.go | 78 +++++++ sdk/monitor/azingest/client_test.go | 92 +++++++++ sdk/monitor/azingest/custom_client.go | 34 +++ sdk/monitor/azingest/example_test.go | 68 ++++++ sdk/monitor/azingest/go.mod | 27 +++ sdk/monitor/azingest/go.sum | 44 ++++ sdk/monitor/azingest/models.go | 16 ++ sdk/monitor/azingest/response_types.go | 15 ++ sdk/monitor/azingest/test-resources.bicep | 240 ++++++++++++++++++++++ sdk/monitor/azingest/utils_test.go | 114 ++++++++++ sdk/monitor/azingest/version.go | 12 ++ 19 files changed, 996 insertions(+) create mode 100644 sdk/monitor/azingest/CHANGELOG.md create mode 100644 sdk/monitor/azingest/LICENSE.txt create mode 100644 sdk/monitor/azingest/README.md create mode 100644 sdk/monitor/azingest/assets.json create mode 100644 sdk/monitor/azingest/autorest.md create mode 100644 sdk/monitor/azingest/build.go create mode 100644 sdk/monitor/azingest/ci.yml create mode 100644 sdk/monitor/azingest/client.go create mode 100644 sdk/monitor/azingest/client_test.go create mode 100644 sdk/monitor/azingest/custom_client.go create mode 100644 sdk/monitor/azingest/example_test.go create mode 100644 sdk/monitor/azingest/go.mod create mode 100644 sdk/monitor/azingest/go.sum create mode 100644 sdk/monitor/azingest/models.go create mode 100644 sdk/monitor/azingest/response_types.go create mode 100644 sdk/monitor/azingest/test-resources.bicep create mode 100644 sdk/monitor/azingest/utils_test.go create mode 100644 sdk/monitor/azingest/version.go diff --git a/eng/config.json b/eng/config.json index 9c2af2feb890..1f7c6ec9a2f2 100644 --- a/eng/config.json +++ b/eng/config.json @@ -97,6 +97,10 @@ "Name": "monitor/azquery", "CoverageGoal": 0.85 }, + { + "Name": "monitor/azingest", + "CoverageGoal": 0.75 + }, { "Name": "security/keyvault/azadmin", "CoverageGoal": 0.80 diff --git a/sdk/monitor/azingest/CHANGELOG.md b/sdk/monitor/azingest/CHANGELOG.md new file mode 100644 index 000000000000..73e1618e38be --- /dev/null +++ b/sdk/monitor/azingest/CHANGELOG.md @@ -0,0 +1,4 @@ +# Release History + +## 0.1.0 (2023-07-17) +* This is the initial release of the `azingest` library \ No newline at end of file diff --git a/sdk/monitor/azingest/LICENSE.txt b/sdk/monitor/azingest/LICENSE.txt new file mode 100644 index 000000000000..4b3ba9df30d6 --- /dev/null +++ b/sdk/monitor/azingest/LICENSE.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) Microsoft Corporation. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE \ No newline at end of file diff --git a/sdk/monitor/azingest/README.md b/sdk/monitor/azingest/README.md new file mode 100644 index 000000000000..ccd5ce6f11de --- /dev/null +++ b/sdk/monitor/azingest/README.md @@ -0,0 +1,119 @@ +# Azure Monitor Ingestion client module for Go + +The Azure Monitor Ingestion client module is used to send custom logs to [Azure Monitor][azure_monitor_overview] using the [Logs Ingestion API][ingestion_overview]. + +This library allows you to send data from virtually any source to supported built-in tables or to custom tables that you create in Log Analytics workspaces. You can even extend the schema of built-in tables with custom columns. + +Source code | Package (pkg.go.dev) | [Product documentation][azure_monitor_overview] | Samples + +## Getting started + +### Prerequisites + +* A supported Go version (the Azure SDK supports the two most recent Go releases) +* An [Azure subscription][azure_subscription] +* An [Azure Log Analytics workspace][azure_monitor_create_using_portal] +* A [Data Collection Endpoint][data_collection_endpoint] +* A [Data Collection Rule][data_collection_rule] + +### Install the package + +Install the `azingest` and `azidentity` modules with `go get`: + +```bash +go get github.com/Azure/azure-sdk-for-go/sdk/monitor/azingest +go get github.com/Azure/azure-sdk-for-go/sdk/azidentity +``` + +The [azidentity][azure_identity] module is used for Azure Active Directory authentication while creating the client. + +### Authentication + +An authenticated client object is required to upload logs. The examples demonstrate using [azidentity.NewDefaultAzureCredential][default_cred_ref] to authenticate; however, the client accepts any [azidentity][azure_identity] credential. See the [azidentity][azure_identity] documentation for more information about other credential types. + +#### Create a client + +Example client + +## Key concepts + +### Data Collection Endpoint + +Data Collection Endpoints (DCEs) allow you to uniquely configure ingestion settings for Azure Monitor. [This article][data_collection_endpoint] provides an overview of data collection endpoints including their contents and structure and how you can create and work with them. + +### Data Collection Rule + +Data collection rules (DCR) define data collected by Azure Monitor and specify how and where that data should be sent or stored. The REST API call must specify a DCR to use. A single DCE can support multiple DCRs, so you can specify a different DCR for different sources and target tables. + +The DCR must understand the structure of the input data and the structure of the target table. If the two don't match, it can use a transformation to convert the source data to match the target table. You may also use the transform to filter source data and perform any other calculations or conversions. + +For more information, see [Data collection rules in Azure Monitor][data_collection_rule], and see [this article][data_collection_rule_structure] for details about a DCR's structure. For information on how to retrieve a DCR ID, see [this tutorial][data_collection_rule_tutorial]. + +### Log Analytics workspace tables + +Custom logs can send data to any custom table that you create and to certain built-in tables in your Log Analytics workspace. The target table must exist before you can send data to it. The following built-in tables are currently supported: + +- [CommonSecurityLog](https://learn.microsoft.com/azure/azure-monitor/reference/tables/commonsecuritylog) +- [SecurityEvents](https://learn.microsoft.com/azure/azure-monitor/reference/tables/securityevent) +- [Syslog](https://learn.microsoft.com/azure/azure-monitor/reference/tables/syslog) +- [WindowsEvents](https://learn.microsoft.com/azure/azure-monitor/reference/tables/windowsevent) + +### Logs retrieval + +The logs that were uploaded using this module can be queried using the [azquery][azure_monitor_query] module (Azure Monitor Query). + +## Examples + +Get started with our examples. + +## Troubleshooting + +### Error Handling + +All methods which send HTTP requests return `*azcore.ResponseError` when these requests fail. `ResponseError` has error details and the raw response from Monitor Query. + +### Logging + +This module uses the logging implementation in `azcore`. To turn on logging for all Azure SDK modules, set `AZURE_SDK_GO_LOGGING` to `all`. By default, the logger writes to stderr. Use the `azcore/log` package to control log output. For example, logging only HTTP request and response events, and printing them to stdout: + +```go +import azlog "github.com/Azure/azure-sdk-for-go/sdk/azcore/log" + +// Print log events to stdout +azlog.SetListener(func(cls azlog.Event, msg string) { + fmt.Println(msg) +}) + +// Includes only requests and responses in credential logs +azlog.SetEvents(azlog.EventRequest, azlog.EventResponse) +``` + +## Next steps + +To learn more about Azure Monitor, see the [Azure Monitor service documentation][azure_monitor_overview]. + +## Contributing + +This project welcomes contributions and suggestions. Most contributions require you to agree to a Contributor License Agreement (CLA) declaring that you have the right to, and actually do, grant us the rights to use your contribution. For details, visit [cla.microsoft.com][cla]. + +When you submit a pull request, a CLA-bot will automatically determine whether you need to provide a CLA and decorate the PR appropriately (e.g., label, comment). Simply follow the instructions provided by the bot. You will only need to do this once across all repositories using our CLA. + +This project has adopted the [Microsoft Open Source Code of Conduct][code_of_conduct]. For more information see the [Code of Conduct FAQ][coc_faq] or contact [opencode@microsoft.com][coc_contact] with any additional questions or comments. + + +[azure_identity]: https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/azidentity +[azure_monitor_create_using_portal]: https://learn.microsoft.com/azure/azure-monitor/logs/quick-create-workspace +[azure_monitor_overview]: https://learn.microsoft.com/azure/azure-monitor/ +[azure_monitor_query]: https://pkg.go.dev/github.com/Azure/azure-sdk-for-go/sdk/monitor/azquery +[azure_subscription]: https://azure.microsoft.com/free/ +[data_collection_endpoint]: https://learn.microsoft.com/azure/azure-monitor/essentials/data-collection-endpoint-overview +[data_collection_rule]: https://learn.microsoft.com/azure/azure-monitor/essentials/data-collection-rule-overview +[data_collection_rule_structure]: https://learn.microsoft.com/azure/azure-monitor/essentials/data-collection-rule-structure +[data_collection_rule_tutorial]: https://learn.microsoft.com/azure/azure-monitor/logs/tutorial-logs-ingestion-portal#collect-information-from-the-dcr +[default_cred_ref]: https://github.com/Azure/azure-sdk-for-go/tree/main/sdk/azidentity#defaultazurecredential +[ingestion_overview]: https://learn.microsoft.com/azure/azure-monitor/logs/logs-ingestion-api-overview + +[cla]: https://cla.microsoft.com +[code_of_conduct]: https://opensource.microsoft.com/codeofconduct/ +[coc_faq]: https://opensource.microsoft.com/codeofconduct/faq/ +[coc_contact]: mailto:opencode@microsoft.com \ No newline at end of file diff --git a/sdk/monitor/azingest/assets.json b/sdk/monitor/azingest/assets.json new file mode 100644 index 000000000000..89085e79c0ff --- /dev/null +++ b/sdk/monitor/azingest/assets.json @@ -0,0 +1,6 @@ +{ + "AssetsRepo": "Azure/azure-sdk-assets", + "AssetsRepoPrefixPath": "go", + "TagPrefix": "go/monitor/azingest", + "Tag": "go/monitor/azingest_1c9fcb94ed" +} diff --git a/sdk/monitor/azingest/autorest.md b/sdk/monitor/azingest/autorest.md new file mode 100644 index 000000000000..9277407c63f4 --- /dev/null +++ b/sdk/monitor/azingest/autorest.md @@ -0,0 +1,60 @@ +## Go + +```yaml +title: MonitorIngestionClient +description: Azure Monitor Ingestion Go Client +generated-metadata: false + +clear-output-folder: false +export-clients: true +go: true +input-file: https://github.com/Azure/azure-rest-api-specs/blob/f07297ce913bfc911470a86436e73c9aceec0587/specification/monitor/data-plane/ingestion/stable/2023-01-01/DataCollectionRules.json +license-header: MICROSOFT_MIT_NO_VERSION +module: github.com/Azure/azure-sdk-for-go/sdk/monitor/azingest +openapi-type: "data-plane" +output-folder: ../azingest +override-client-name: Client +security: "AADToken" +use: "@autorest/go@4.0.0-preview.46" +version: "^3.0.0" +rawjson-as-bytes: true + +directive: + # delete unused model + - remove-model: PendingCertificateSigningRequestResult + + # remove x-ms-client-request-id, it's added in a pipeline policy + - where-operation: Upload + remove-parameter: + in: header + name: x-ms-client-request-id + + # rename parameter from "body" to "logs", "stream" to "streamName" + - from: swagger-document + where: $.paths..parameters..[?(@.name=='body')] + transform: $["x-ms-client-name"] = "logs" + - from: swagger-document + where: $.paths..parameters..[?(@.name=='stream')] + transform: $["x-ms-client-name"] = "streamName" + + # delete unused error models + - from: models.go + where: $ + transform: return $.replace(/(?:\/\/.*\s)+type (?:ErrorResponse|ErrorDetail|ErrorAdditionalInfo).+\{(?:\s.+\s)+\}\s/g, ""); + + # delete client name prefix from method options and response types + - from: + - client.go + - models.go + - response_types.go + where: $ + transform: return $.replace(/Client(\w+)((?:Options|Response))/g, "$1$2"); + + # update doc comments + - from: swagger-document + where: $.paths..parameters..[?(@.name=='Content-Encoding')] + transform: $["description"] = "If the bytes of the \"logs\" parameter are already gzipped, set ContentEncoding to \"gzip\"" + - from: swagger-document + where: $.paths./dataCollectionRules/{ruleId}/streams/{stream}.post + transform: $["description"] = "Ingestion API used to directly ingest data using Data Collection Rules. Maximum size of of API call is 1 MB." +``` \ No newline at end of file diff --git a/sdk/monitor/azingest/build.go b/sdk/monitor/azingest/build.go new file mode 100644 index 000000000000..4731bcd837a5 --- /dev/null +++ b/sdk/monitor/azingest/build.go @@ -0,0 +1,12 @@ +//go:build go1.18 +// +build go1.18 + +//go:generate autorest ./autorest.md +//go:generate rm ./models_serde.go +//go:generate rm ./constants.go +//go:generate gofmt -w . + +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +package azingest diff --git a/sdk/monitor/azingest/ci.yml b/sdk/monitor/azingest/ci.yml new file mode 100644 index 000000000000..74d82d89824f --- /dev/null +++ b/sdk/monitor/azingest/ci.yml @@ -0,0 +1,30 @@ +# NOTE: Please refer to https://aka.ms/azsdk/engsys/ci-yaml before editing this file. +trigger: + branches: + include: + - main + - feature/* + - hotfix/* + - release/* + paths: + include: + - sdk/monitor/azingest + +pr: + branches: + include: + - main + - feature/* + - hotfix/* + - release/* + paths: + include: + - sdk/monitor/azingest + + +stages: +- template: /eng/pipelines/templates/jobs/archetype-sdk-client.yml + parameters: + ServiceDirectory: 'monitor/azingest' + RunLiveTests: true + SupportedClouds: 'Public,UsGov,China' \ No newline at end of file diff --git a/sdk/monitor/azingest/client.go b/sdk/monitor/azingest/client.go new file mode 100644 index 000000000000..8663f0d1970b --- /dev/null +++ b/sdk/monitor/azingest/client.go @@ -0,0 +1,78 @@ +//go:build go1.18 +// +build go1.18 + +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. +// Code generated by Microsoft (R) AutoRest Code Generator. +// Changes may cause incorrect behavior and will be lost if the code is regenerated. +// DO NOT EDIT. + +package azingest + +import ( + "bytes" + "context" + "errors" + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/streaming" + "net/http" + "net/url" + "strings" +) + +// Client contains the methods for the Client group. +// Don't use this type directly, use a constructor function instead. +type Client struct { + internal *azcore.Client + endpoint string +} + +// Upload - Ingestion API used to directly ingest data using Data Collection Rules. Maximum size of of API call is 1 MB. +// If the operation fails it returns an *azcore.ResponseError type. +// +// Generated from API version 2023-01-01 +// - ruleID - The immutable Id of the Data Collection Rule resource. +// - streamName - The streamDeclaration name as defined in the Data Collection Rule. +// - logs - An array of objects matching the schema defined by the provided stream. +// - options - UploadOptions contains the optional parameters for the Client.Upload method. +func (client *Client) Upload(ctx context.Context, ruleID string, streamName string, logs []byte, options *UploadOptions) (UploadResponse, error) { + req, err := client.uploadCreateRequest(ctx, ruleID, streamName, logs, options) + if err != nil { + return UploadResponse{}, err + } + resp, err := client.internal.Pipeline().Do(req) + if err != nil { + return UploadResponse{}, err + } + if !runtime.HasStatusCode(resp, http.StatusNoContent) { + return UploadResponse{}, runtime.NewResponseError(resp) + } + return UploadResponse{}, nil +} + +// uploadCreateRequest creates the Upload request. +func (client *Client) uploadCreateRequest(ctx context.Context, ruleID string, streamName string, logs []byte, options *UploadOptions) (*policy.Request, error) { + urlPath := "/dataCollectionRules/{ruleId}/streams/{stream}" + if ruleID == "" { + return nil, errors.New("parameter ruleID cannot be empty") + } + urlPath = strings.ReplaceAll(urlPath, "{ruleId}", url.PathEscape(ruleID)) + if streamName == "" { + return nil, errors.New("parameter streamName cannot be empty") + } + urlPath = strings.ReplaceAll(urlPath, "{stream}", url.PathEscape(streamName)) + req, err := runtime.NewRequest(ctx, http.MethodPost, runtime.JoinPaths(client.endpoint, urlPath)) + if err != nil { + return nil, err + } + reqQP := req.Raw().URL.Query() + reqQP.Set("api-version", "2023-01-01") + req.Raw().URL.RawQuery = reqQP.Encode() + if options != nil && options.ContentEncoding != nil { + req.Raw().Header["Content-Encoding"] = []string{*options.ContentEncoding} + } + req.Raw().Header["Accept"] = []string{"application/json"} + return req, req.SetBody(streaming.NopCloser(bytes.NewReader(logs)), "application/json") +} diff --git a/sdk/monitor/azingest/client_test.go b/sdk/monitor/azingest/client_test.go new file mode 100644 index 000000000000..2bcf3792714b --- /dev/null +++ b/sdk/monitor/azingest/client_test.go @@ -0,0 +1,92 @@ +//go:build go1.18 +// +build go1.18 + +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +package azingest_test + +import ( + "bytes" + "compress/gzip" + "context" + "encoding/json" + "strconv" + "testing" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/monitor/azingest" + "github.com/stretchr/testify/require" +) + +type ComputerInfo struct { + Time time.Time + Computer string + AdditionalContext AdditionalContext +} + +type AdditionalContext struct { + TestContextKey int + CounterName string +} + +func generateLogs() []byte { + var data []ComputerInfo + for i := 0; i < 10; i++ { + data = append(data, ComputerInfo{ + Time: time.Date(2023, 01, 01, 0, 0, 0, 0, time.UTC), + Computer: "Computer" + strconv.Itoa(i), + AdditionalContext: AdditionalContext{TestContextKey: i, CounterName: "AppMetric2"}, + }) + } + data2, err := json.Marshal(data) + if err != nil { + panic(err) + } + return data2 +} + +func TestUpload(t *testing.T) { + client := startTest(t) + + logs := generateLogs() + + res, err := client.Upload(context.Background(), ruleID, streamName, logs, nil) + require.NoError(t, err) + require.Empty(t, res) +} + +func TestUploadWithGzip(t *testing.T) { + client := startTest(t) + + logs := generateLogs() + + // gzip data + var buf bytes.Buffer + zw := gzip.NewWriter(&buf) + _, err := zw.Write(logs) + require.NoError(t, err) + err = zw.Close() + require.NoError(t, err) + + res, err := client.Upload(context.Background(), ruleID, streamName, buf.Bytes(), &azingest.UploadOptions{ContentEncoding: to.Ptr("gzip")}) + require.NoError(t, err) + require.Empty(t, res) +} + +func TestUploadWithError(t *testing.T) { + client := startTest(t) + + logs := generateLogs() + + res, err := client.Upload(context.Background(), ruleID, "incorrect stream", logs, nil) + require.Error(t, err) + require.Empty(t, res) + + var httpErr *azcore.ResponseError + require.ErrorAs(t, err, &httpErr) + require.Equal(t, httpErr.ErrorCode, "InvalidStream") + require.Equal(t, httpErr.StatusCode, 400) +} diff --git a/sdk/monitor/azingest/custom_client.go b/sdk/monitor/azingest/custom_client.go new file mode 100644 index 000000000000..cd959f74a335 --- /dev/null +++ b/sdk/monitor/azingest/custom_client.go @@ -0,0 +1,34 @@ +//go:build go1.18 +// +build go1.18 + +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +package azingest + +// this file contains handwritten additions to the generated code + +import ( + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/runtime" +) + +// ClientOptions contains optional settings for Client. +type ClientOptions struct { + azcore.ClientOptions +} + +// NewClient creates a client to upload logs to Azure Monitor Ingestion. +func NewClient(endpoint string, credential azcore.TokenCredential, options *ClientOptions) (*Client, error) { + if options == nil { + options = &ClientOptions{} + } + + authPolicy := runtime.NewBearerTokenPolicy(credential, []string{"https://monitor.azure.com/" + "/.default"}, nil) + azcoreClient, err := azcore.NewClient("azingest.Client", version, runtime.PipelineOptions{PerRetry: []policy.Policy{authPolicy}}, &options.ClientOptions) + if err != nil { + return nil, err + } + return &Client{internal: azcoreClient, endpoint: endpoint}, nil +} diff --git a/sdk/monitor/azingest/example_test.go b/sdk/monitor/azingest/example_test.go new file mode 100644 index 000000000000..bc3f90552465 --- /dev/null +++ b/sdk/monitor/azingest/example_test.go @@ -0,0 +1,68 @@ +//go:build go1.18 +// +build go1.18 + +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +package azingest_test + +import ( + "context" + "encoding/json" + "os" + "strconv" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/monitor/azingest" +) + +var client azingest.Client + +type Computer struct { + Time time.Time + Computer string + AdditionalContext string +} + +func ExampleNewClient() { + endpoint = os.Getenv("DATA_COLLECTION_ENDPOINT") + cred, err := azidentity.NewDefaultAzureCredential(nil) + if err != nil { + //TODO: handle error + } + + client, err := azingest.NewClient(endpoint, cred, nil) + if err != nil { + //TODO: handle error + } + _ = client +} + +func ExampleClient_Upload() { + // set necessary data collection rule variables + ruleID := os.Getenv("DATA_COLLECTION_RULE_IMMUTABLE_ID") + streamName := os.Getenv("DATA_COLLECTION_RULE_STREAM_NAME") + + // generating logs + // logs should match the schema defined by the provided stream + var data []Computer + for i := 0; i < 10; i++ { + data = append(data, Computer{ + Time: time.Now().UTC(), + Computer: "Computer" + strconv.Itoa(i), + AdditionalContext: "context", + }) + } + // Marshal data into []byte + logs, err := json.Marshal(data) + if err != nil { + panic(err) + } + + // upload logs + _, err = client.Upload(context.TODO(), ruleID, streamName, logs, nil) + if err != nil { + //TODO: handle error + } +} diff --git a/sdk/monitor/azingest/go.mod b/sdk/monitor/azingest/go.mod new file mode 100644 index 000000000000..cb9f29fb2376 --- /dev/null +++ b/sdk/monitor/azingest/go.mod @@ -0,0 +1,27 @@ +module github.com/Azure/azure-sdk-for-go/sdk/monitor/azingest + +go 1.18 + +require ( + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.1 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 + github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 + github.com/stretchr/testify v1.7.0 +) + +require ( + github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dnaeon/go-vcr v1.2.0 // indirect + github.com/golang-jwt/jwt/v4 v4.5.0 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + golang.org/x/crypto v0.11.0 // indirect + golang.org/x/net v0.12.0 // indirect + golang.org/x/sys v0.10.0 // indirect + golang.org/x/text v0.11.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/sdk/monitor/azingest/go.sum b/sdk/monitor/azingest/go.sum new file mode 100644 index 000000000000..78ec1d5d94d6 --- /dev/null +++ b/sdk/monitor/azingest/go.sum @@ -0,0 +1,44 @@ +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.1 h1:SEy2xmstIphdPwNBUi7uhvjyjhVKISfwjfOJmuy7kg4= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.6.1/go.mod h1:bjGvMhVMb+EEm3VRNQawDMUyMMjo+S5ewNjflkep/0Q= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0 h1:vcYCAze6p19qBW7MhZybIsqD8sMV8js0NyQM8JDnVtg= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.3.0/go.mod h1:OQeznEEkTZ9OrhHJoDD8ZDq51FHgXjqtP9z6bEwBq9U= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0 h1:sXr+ck84g/ZlZUOZiNELInmMgOsuGwdjjVkEIde0OtY= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.3.0/go.mod h1:okt5dMMTOFjX/aovMlrjvvXoPMBVSPzk9185BT0+eZM= +github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 h1:OBhqkivkhkMqLPymWEppkm7vgPQY2XsHoEkaMQ0AdZY= +github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0/go.mod h1:kgDmCTgBzIEPFElEF+FK0SdjAor06dRq2Go927dnQ6o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dnaeon/go-vcr v1.2.0 h1:zHCHvJYTMh1N7xnV7zf1m1GPBF9Ad0Jk/whtQ1663qI= +github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= +github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= +github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= +golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= +golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/sdk/monitor/azingest/models.go b/sdk/monitor/azingest/models.go new file mode 100644 index 000000000000..ac890e7b772c --- /dev/null +++ b/sdk/monitor/azingest/models.go @@ -0,0 +1,16 @@ +//go:build go1.18 +// +build go1.18 + +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. +// Code generated by Microsoft (R) AutoRest Code Generator. +// Changes may cause incorrect behavior and will be lost if the code is regenerated. +// DO NOT EDIT. + +package azingest + +// UploadOptions contains the optional parameters for the Client.Upload method. +type UploadOptions struct { + // If the bytes of the "logs" parameter are already gzipped, set ContentEncoding to "gzip" + ContentEncoding *string +} diff --git a/sdk/monitor/azingest/response_types.go b/sdk/monitor/azingest/response_types.go new file mode 100644 index 000000000000..fe68124f4bcc --- /dev/null +++ b/sdk/monitor/azingest/response_types.go @@ -0,0 +1,15 @@ +//go:build go1.18 +// +build go1.18 + +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. +// Code generated by Microsoft (R) AutoRest Code Generator. +// Changes may cause incorrect behavior and will be lost if the code is regenerated. +// DO NOT EDIT. + +package azingest + +// UploadResponse contains the response from method Client.Upload. +type UploadResponse struct { + // placeholder for future response values +} diff --git a/sdk/monitor/azingest/test-resources.bicep b/sdk/monitor/azingest/test-resources.bicep new file mode 100644 index 000000000000..45b985709dde --- /dev/null +++ b/sdk/monitor/azingest/test-resources.bicep @@ -0,0 +1,240 @@ +@description('The base resource name.') +param baseName string = resourceGroup().name + +@description('Which Azure Region to deploy the resource to. Defaults to the resource group location.') +param location string = resourceGroup().location + +@description('The principal to assign the role to. This is application object id.') +param testApplicationOid string + +@description('Specifies the name of the Data Collection Endpoint to create.') +param dataCollectionEndpointName string = 'az-dce' + +@description('Specifies the name of the Data Collection Rule to create.') +param dataCollectionRuleName string = 'az-dcr' + +// Currently variables can't be used as keys, so the stream name is also hardcoded in the 'streamDeclarations' of the +// 'dataCollectionRules' resource below. +var streamName = 'Custom-MyTableRawData' +var tableName = 'MyTable_CL' + +resource id 'Microsoft.Authorization/roleAssignments@2018-09-01-preview' = { + name: guid(resourceGroup().id) + properties: { + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '5ae67dd6-50cb-40e7-96ff-dc2bfa4b606b') + principalId: testApplicationOid + } +} + +resource applicationInsightsComponent 'Microsoft.Insights/components@2020-02-02-preview' = { + name: '${baseName}-appinsights-python' + location: location + kind: 'other' + properties: { + Application_Type: 'other' + WorkspaceResourceId: primaryWorkspace.id + } +} + +resource dcrRoleAssignment 'Microsoft.Authorization/roleAssignments@2018-09-01-preview' = { + name: guid(resourceGroup().id, dataCollectionRule.name, dataCollectionRule.id) + properties: { + roleDefinitionId: subscriptionResourceId('Microsoft.Authorization/roleDefinitions', '3913510d-42f4-4e42-8a64-420c390055eb') + principalId: testApplicationOid + } +} + +resource dataCollectionEndpoint 'Microsoft.Insights/dataCollectionEndpoints@2021-04-01' = { + name: dataCollectionEndpointName + location: location + properties: { + networkAcls: { + publicNetworkAccess: 'Enabled' + } + } +} + +resource primaryWorkspace 'Microsoft.OperationalInsights/workspaces@2021-12-01-preview' = { + name: '${baseName}-ws' + location: location + properties: { + sku: { + name: 'PerGB2018' + } + retentionInDays: 30 + features: { + enableLogAccessUsingOnlyResourcePermissions: true + } + workspaceCapping: { + dailyQuotaGb: -1 + } + publicNetworkAccessForIngestion: 'Enabled' + publicNetworkAccessForQuery: 'Enabled' + } + + resource workspaceTable 'tables' = { + name: tableName + properties: { + totalRetentionInDays: 30 + plan: 'Analytics' + schema: { + name: tableName + columns: [ + { + name: 'TimeGenerated' + type: 'datetime' + description: 'The time at which the data was generated' + } + { + name: 'AdditionalContext' + type: 'dynamic' + description: 'Additional message properties' + } + { + name: 'ExtendedColumn' + type: 'string' + description: 'An additional column extended at ingestion time' + } + ] + } + retentionInDays: 30 + } + } +} + +resource secondaryWorkspace 'Microsoft.OperationalInsights/workspaces@2021-12-01-preview' = { + name: '${baseName}-ws2' + location: location + properties: { + sku: { + name: 'PerGB2018' + } + retentionInDays: 30 + features: { + enableLogAccessUsingOnlyResourcePermissions: true + } + workspaceCapping: { + dailyQuotaGb: -1 + } + publicNetworkAccessForIngestion: 'Enabled' + publicNetworkAccessForQuery: 'Enabled' + } + + resource workspaceTable 'tables' = { + name: tableName + properties: { + totalRetentionInDays: 30 + plan: 'Analytics' + schema: { + name: tableName + columns: [ + { + name: 'TimeGenerated' + type: 'datetime' + description: 'The time at which the data was generated' + } + { + name: 'AdditionalContext' + type: 'dynamic' + description: 'Additional message properties' + } + { + name: 'ExtendedColumn' + type: 'string' + description: 'An additional column extended at ingestion time' + } + ] + } + retentionInDays: 30 + } + } +} + +resource dataCollectionRule 'Microsoft.Insights/dataCollectionRules@2021-09-01-preview' = { + name: dataCollectionRuleName + location: location + properties: { + dataCollectionEndpointId: dataCollectionEndpoint.id + streamDeclarations: { + 'Custom-MyTableRawData': { + columns: [ + { + name: 'Time' + type: 'datetime' + } + { + name: 'Computer' + type: 'string' + } + { + name: 'AdditionalContext' + type: 'string' + } + ] + } + 'Custom-MyTableRawData2': { + columns: [ + { + name: 'Time' + type: 'datetime' + } + { + name: 'Computer' + type: 'string' + } + { + name: 'AdditionalContext' + type: 'string' + } + ] + } + } + destinations: { + logAnalytics: [ + { + workspaceResourceId: primaryWorkspace.id + name: primaryWorkspace.name + } + { + workspaceResourceId: secondaryWorkspace.id + name: secondaryWorkspace.name + } + ] + } + dataFlows: [ + { + streams: [ + streamName + ] + destinations: [ + primaryWorkspace.name + ] + transformKql: 'source | extend jsonContext = parse_json(AdditionalContext) | project TimeGenerated = Time, Computer, AdditionalContext = jsonContext, ExtendedColumn=tostring(jsonContext.CounterName)' + outputStream: 'Custom-${tableName}' + } + { + streams: [ + '${streamName}2' + ] + destinations: [ + secondaryWorkspace.name + ] + transformKql: 'source | extend jsonContext = parse_json(AdditionalContext) | project TimeGenerated = Time, Computer, AdditionalContext = jsonContext, ExtendedColumn=tostring(jsonContext.CounterName)' + outputStream: 'Custom-${tableName}' + } + ] + } + dependsOn: [ + primaryWorkspace::workspaceTable + secondaryWorkspace::workspaceTable + ] +} + +output APPLICATIONINSIGHTS_CONNECTION_STRING string = applicationInsightsComponent.properties.ConnectionString +output METRICS_RESOURCE_ID string = applicationInsightsComponent.id +output AZURE_MONITOR_WORKSPACE_ID string = primaryWorkspace.properties.customerId +output AZURE_MONITOR_SECONDARY_WORKSPACE_ID string = secondaryWorkspace.properties.customerId +output AZURE_MONITOR_DCE string = dataCollectionEndpoint.properties.logsIngestion.endpoint +output AZURE_MONITOR_DCR_ID string = dataCollectionRule.properties.immutableId +output AZURE_MONITOR_STREAM_NAME string = streamName +output AZURE_MONITOR_TABLE_NAME string = tableName diff --git a/sdk/monitor/azingest/utils_test.go b/sdk/monitor/azingest/utils_test.go new file mode 100644 index 000000000000..b4e715fbfad4 --- /dev/null +++ b/sdk/monitor/azingest/utils_test.go @@ -0,0 +1,114 @@ +//go:build go1.18 +// +build go1.18 + +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +package azingest_test + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/policy" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/internal/recording" + "github.com/Azure/azure-sdk-for-go/sdk/monitor/azingest" + "github.com/stretchr/testify/require" +) + +const fakeEndpoint = "https://test.eastus-1.ingest.monitor.azure.com" +const fakeRuleID = "Custom-TestTable_CL" +const fakeStreamName = "dcr-testing" + +var ( + credential azcore.TokenCredential + endpoint string + ruleID string + streamName string +) + +func TestMain(m *testing.M) { + err := recording.ResetProxy(nil) + if err != nil { + panic(err) + } + if recording.GetRecordMode() == recording.PlaybackMode { + credential = &FakeCredential{} + } else { + tenantID := lookupEnvVar("AZINGEST_TENANT_ID") + clientID := lookupEnvVar("AZINGEST_CLIENT_ID") + secret := lookupEnvVar("AZINGEST_CLIENT_SECRET") + credential, err = azidentity.NewClientSecretCredential(tenantID, clientID, secret, nil) + if err != nil { + panic(err) + } + } + endpoint = getEnvVar("AZURE_MONITOR_DCE", fakeEndpoint) + ruleID = getEnvVar("AZURE_MONITOR_DCR_ID", fakeRuleID) + streamName = getEnvVar("AZURE_MONITOR_STREAM_NAME", fakeStreamName) + + code := m.Run() + os.Exit(code) +} + +func startRecording(t *testing.T) { + err := recording.Start(t, "sdk/monitor/azingest/testdata", nil) + require.NoError(t, err) + t.Cleanup(func() { + err := recording.Stop(t, nil) + require.NoError(t, err) + }) +} + +func startTest(t *testing.T) *azingest.Client { + startRecording(t) + transport, err := recording.NewRecordingHTTPClient(t, nil) + require.NoError(t, err) + opts := &azingest.ClientOptions{ClientOptions: azcore.ClientOptions{Transport: transport}} + + client, err := azingest.NewClient(endpoint, credential, opts) + if err != nil { + panic(err) + } + return client +} + +func getEnvVar(lookupValue string, fakeValue string) string { + // get value + envVar := fakeValue + if recording.GetRecordMode() == recording.LiveMode || recording.GetRecordMode() == recording.RecordingMode { + envVar = os.Getenv(lookupValue) + if envVar == "" { + panic("no value for " + lookupValue) + } + } + + // sanitize value + if recording.GetRecordMode() == recording.RecordingMode { + err := recording.AddGeneralRegexSanitizer(fakeValue, envVar, nil) + if err != nil { + panic(err) + } + } + + return envVar +} + +func lookupEnvVar(s string) string { + ret, ok := os.LookupEnv(s) + if !ok { + panic(fmt.Sprintf("Could not find env var: '%s'", s)) + } + return ret +} + +type FakeCredential struct{} + +func (f *FakeCredential) GetToken(ctx context.Context, options policy.TokenRequestOptions) (azcore.AccessToken, error) { + return azcore.AccessToken{Token: "faketoken", ExpiresOn: time.Now().Add(time.Hour).UTC()}, nil +} diff --git a/sdk/monitor/azingest/version.go b/sdk/monitor/azingest/version.go new file mode 100644 index 000000000000..3882775c1623 --- /dev/null +++ b/sdk/monitor/azingest/version.go @@ -0,0 +1,12 @@ +//go:build go1.18 +// +build go1.18 + +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root for license information. + +package azingest + +const ( + moduleName = "azingest" + version = "v0.1.0" +)