diff --git a/sdk/monitor/monitor-opentelemetry-exporter/CHANGELOG.md b/sdk/monitor/monitor-opentelemetry-exporter/CHANGELOG.md index 9982db05d3bb..12306f60917d 100644 --- a/sdk/monitor/monitor-opentelemetry-exporter/CHANGELOG.md +++ b/sdk/monitor/monitor-opentelemetry-exporter/CHANGELOG.md @@ -8,6 +8,7 @@ - Added retriable behavior for 502, 503 and 504 status codes. - Export Metric attributes and Histogram Min/Max values. - Added new config options disableOfflineStorage, storageDirectory and exposed ApplicationInsightsClientOptionalParams for HTTP client extra configuration. +- Added Network Statsbeat Metrics. ### Breaking Changes diff --git a/sdk/monitor/monitor-opentelemetry-exporter/review/monitor-opentelemetry-exporter.api.md b/sdk/monitor/monitor-opentelemetry-exporter/review/monitor-opentelemetry-exporter.api.md index 089f03339e6c..a31743733a0d 100644 --- a/sdk/monitor/monitor-opentelemetry-exporter/review/monitor-opentelemetry-exporter.api.md +++ b/sdk/monitor/monitor-opentelemetry-exporter/review/monitor-opentelemetry-exporter.api.md @@ -35,7 +35,7 @@ export class ApplicationInsightsSampler implements Sampler { // @public export abstract class AzureMonitorBaseExporter { - constructor(options?: AzureMonitorExporterOptions); + constructor(options?: AzureMonitorExporterOptions, isStatsbeatExporter?: boolean); protected _exportEnvelopes(envelopes: TelemetryItem[]): Promise; protected _instrumentationKey: string; protected _shutdown(): Promise; @@ -59,6 +59,14 @@ export class AzureMonitorMetricExporter extends AzureMonitorBaseExporter impleme shutdown(): Promise; } +// @internal +export class _AzureMonitorStatsbeatExporter extends AzureMonitorBaseExporter implements PushMetricExporter { + constructor(options: AzureMonitorExporterOptions); + export(metrics: ResourceMetrics, resultCallback: (result: ExportResult) => void): Promise; + forceFlush(): Promise; + shutdown(): Promise; +} + // @public export class AzureMonitorTraceExporter extends AzureMonitorBaseExporter implements SpanExporter { constructor(options?: AzureMonitorExporterOptions); diff --git a/sdk/monitor/monitor-opentelemetry-exporter/samples-dev/metricsSample.ts b/sdk/monitor/monitor-opentelemetry-exporter/samples-dev/metricsSample.ts index deefef9f86f0..aa64cd29c606 100644 --- a/sdk/monitor/monitor-opentelemetry-exporter/samples-dev/metricsSample.ts +++ b/sdk/monitor/monitor-opentelemetry-exporter/samples-dev/metricsSample.ts @@ -3,7 +3,7 @@ /** * This example shows how to use - * [@opentelemetry/sdk-metrics-base](https://github.com/open-telemetry/opentelemetry-js/tree/main/experimental/packages/opentelemetry-sdk-metrics-base) + * [@opentelemetry/sdk-metrics](https://github.com/open-telemetry/opentelemetry-js/tree/main/experimental/packages/opentelemetry-sdk-metrics-base) * to generate Metrics in a simple Node.js application and export them to Azure Monitor. * * @summary Basic use of Metrics in Node.js application. diff --git a/sdk/monitor/monitor-opentelemetry-exporter/src/export/base.ts b/sdk/monitor/monitor-opentelemetry-exporter/src/export/base.ts index 3fa56252db92..b837aaa03c92 100644 --- a/sdk/monitor/monitor-opentelemetry-exporter/src/export/base.ts +++ b/sdk/monitor/monitor-opentelemetry-exporter/src/export/base.ts @@ -11,6 +11,8 @@ import { PersistentStorage, Sender } from "../types"; import { isRetriable, BreezeResponse } from "../utils/breezeUtils"; import { DEFAULT_BREEZE_ENDPOINT, ENV_CONNECTION_STRING } from "../Declarations/Constants"; import { TelemetryItem as Envelope } from "../generated"; +import { StatsbeatMetrics } from "./statsbeat/statsbeatMetrics"; +import { MAX_STATSBEAT_FAILURES } from "./statsbeat/types"; const DEFAULT_BATCH_SEND_RETRY_INTERVAL_MS = 60_000; /** @@ -20,13 +22,15 @@ export abstract class AzureMonitorBaseExporter { /** * Instrumentation key to be used for exported envelopes */ - protected _instrumentationKey: string; + protected _instrumentationKey: string = ""; + private _endpointUrl: string = ""; private readonly _persister: PersistentStorage; private readonly _sender: Sender; private _numConsecutiveRedirects: number; private _retryTimer: NodeJS.Timer | null; - private _endpointUrl: string; - + private _statsbeatMetrics: StatsbeatMetrics | undefined; + private _isStatsbeatExporter: boolean; + private _statsbeatFailureCount: number = 0; private _batchSendRetryIntervalMs: number = DEFAULT_BATCH_SEND_RETRY_INTERVAL_MS; /** * Exporter internal configuration @@ -37,12 +41,14 @@ export abstract class AzureMonitorBaseExporter { * Initializes a new instance of the AzureMonitorBaseExporter class. * @param AzureMonitorExporterOptions - Exporter configuration. */ - constructor(options: AzureMonitorExporterOptions = {}) { + constructor(options: AzureMonitorExporterOptions = {}, isStatsbeatExporter?: boolean) { this._options = options; this._numConsecutiveRedirects = 0; this._instrumentationKey = ""; this._endpointUrl = DEFAULT_BREEZE_ENDPOINT; const connectionString = this._options.connectionString || process.env[ENV_CONNECTION_STRING]; + this._isStatsbeatExporter = isStatsbeatExporter ? isStatsbeatExporter : false; + if (connectionString) { const parsedConnectionString = ConnectionStringParser.parse(connectionString); this._instrumentationKey = @@ -57,9 +63,13 @@ export abstract class AzureMonitorBaseExporter { diag.error(message); throw new Error(message); } - this._sender = new HttpSender(this._endpointUrl, this._options); this._persister = new FileSystemPersist(this._instrumentationKey, this._options); + + if (!this._isStatsbeatExporter) { + // Initialize statsbeatMetrics + this._statsbeatMetrics = new StatsbeatMetrics(this._instrumentationKey, this._endpointUrl); + } this._retryTimer = null; diag.debug("AzureMonitorExporter was successfully setup"); } @@ -93,12 +103,18 @@ export abstract class AzureMonitorBaseExporter { */ protected async _exportEnvelopes(envelopes: Envelope[]): Promise { diag.info(`Exporting ${envelopes.length} envelope(s)`); + if (envelopes.length < 1) { return { code: ExportResultCode.SUCCESS }; } + try { + const startTime = new Date().getTime(); const { result, statusCode } = await this._sender.send(envelopes); + const endTime = new Date().getTime(); + const duration = endTime - startTime; this._numConsecutiveRedirects = 0; + if (statusCode === 200) { // Success -- @todo: start retry timer if (!this._retryTimer) { @@ -108,9 +124,14 @@ export abstract class AzureMonitorBaseExporter { }, this._batchSendRetryIntervalMs); this._retryTimer.unref(); } + // If we are not exportings statsbeat and statsbeat is not disabled -- count success + this._statsbeatMetrics?.countSuccess(duration); return { code: ExportResultCode.SUCCESS }; } else if (statusCode && isRetriable(statusCode)) { // Failed -- persist failed data + if (statusCode === 429 || statusCode === 439) { + this._statsbeatMetrics?.countThrottle(statusCode); + } if (result) { diag.info(result); const breezeResponse = JSON.parse(result) as BreezeResponse; @@ -123,19 +144,29 @@ export abstract class AzureMonitorBaseExporter { }); } if (filteredEnvelopes.length > 0) { + this._statsbeatMetrics?.countRetry(statusCode); // calls resultCallback(ExportResult) based on result of persister.push return await this._persist(filteredEnvelopes); } // Failed -- not retriable + this._statsbeatMetrics?.countFailure(duration, statusCode); return { code: ExportResultCode.FAILED, }; } else { // calls resultCallback(ExportResult) based on result of persister.push + this._statsbeatMetrics?.countRetry(statusCode); return await this._persist(envelopes); } } else { // Failed -- not retriable + if (this._statsbeatMetrics) { + if (statusCode) { + this._statsbeatMetrics.countFailure(duration, statusCode); + } + } else { + this._incrementStatsbeatFailure(); + } return { code: ExportResultCode.FAILED, }; @@ -161,19 +192,25 @@ export abstract class AzureMonitorBaseExporter { } } } else { - return { code: ExportResultCode.FAILED, error: new Error("Circular redirect") }; + let redirectError = new Error("Circular redirect"); + this._statsbeatMetrics?.countException(redirectError); + return { code: ExportResultCode.FAILED, error: redirectError }; } } else if (restError.statusCode && isRetriable(restError.statusCode)) { + this._statsbeatMetrics?.countRetry(restError.statusCode); return await this._persist(envelopes); } if (this._isNetworkError(restError)) { + if (restError.statusCode) { + this._statsbeatMetrics?.countRetry(restError.statusCode); + } diag.error( "Retrying due to transient client side error. Error message:", restError.message ); return await this._persist(envelopes); } - + this._statsbeatMetrics?.countException(restError); diag.error( "Envelopes could not be exported and are not retriable. Error message:", restError.message @@ -182,6 +219,17 @@ export abstract class AzureMonitorBaseExporter { } } + // Disable collection of statsbeat metrics after max failures + private _incrementStatsbeatFailure() { + this._statsbeatFailureCount++; + if (this._statsbeatFailureCount > MAX_STATSBEAT_FAILURES) { + this._isStatsbeatExporter = false; + this._statsbeatMetrics?.shutdown(); + this._statsbeatMetrics = undefined; + this._statsbeatFailureCount = 0; + } + } + private async _sendFirstPersistedFile(): Promise { try { const envelopes = (await this._persister.shift()) as Envelope[] | null; diff --git a/sdk/monitor/monitor-opentelemetry-exporter/src/export/metric.ts b/sdk/monitor/monitor-opentelemetry-exporter/src/export/metric.ts index 1b81c5908377..47c5d383842c 100644 --- a/sdk/monitor/monitor-opentelemetry-exporter/src/export/metric.ts +++ b/sdk/monitor/monitor-opentelemetry-exporter/src/export/metric.ts @@ -9,9 +9,9 @@ import { } from "@opentelemetry/sdk-metrics"; import { ExportResult, ExportResultCode, suppressTracing } from "@opentelemetry/core"; import { AzureMonitorBaseExporter } from "./base"; -import { AzureMonitorExporterOptions } from "../config"; import { TelemetryItem as Envelope } from "../generated"; import { resourceMetricsToEnvelope } from "../utils/metricUtils"; +import { AzureMonitorExporterOptions } from "../config"; /** * Azure Monitor OpenTelemetry Metric Exporter. @@ -33,6 +33,7 @@ export class AzureMonitorMetricExporter * Initializes a new instance of the AzureMonitorMetricExporter class. * @param AzureExporterConfig - Exporter configuration. */ + constructor(options: AzureMonitorExporterOptions = {}) { super(options); this._aggregationTemporality = AggregationTemporality.CUMULATIVE; diff --git a/sdk/monitor/monitor-opentelemetry-exporter/src/export/statsbeat.ts b/sdk/monitor/monitor-opentelemetry-exporter/src/export/statsbeat.ts new file mode 100644 index 000000000000..eb3edf14f814 --- /dev/null +++ b/sdk/monitor/monitor-opentelemetry-exporter/src/export/statsbeat.ts @@ -0,0 +1,69 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. +import { context } from "@opentelemetry/api"; +import { PushMetricExporter, ResourceMetrics } from "@opentelemetry/sdk-metrics"; +import { ExportResult, ExportResultCode, suppressTracing } from "@opentelemetry/core"; +import { AzureMonitorExporterOptions } from "../config"; +import { TelemetryItem as Envelope } from "../generated"; +import { resourceMetricsToEnvelope } from "../utils/metricUtils"; +import { AzureMonitorBaseExporter } from "./base"; + +/** + * @internal + * Azure Monitor Statsbeat Exporter + */ +export class _AzureMonitorStatsbeatExporter + extends AzureMonitorBaseExporter + implements PushMetricExporter +{ + /** + * Flag to determine if the Exporter is shutdown. + */ + private _isShutdown = false; + + /** + * Initializes a new instance of the AzureMonitorStatsbeatExporter class. + * @param options - Exporter configuration + */ + constructor(options: AzureMonitorExporterOptions) { + super(options, true); + } + + /** + * Export Statsbeat metrics. + */ + async export( + metrics: ResourceMetrics, + resultCallback: (result: ExportResult) => void + ): Promise { + if (this._isShutdown) { + setTimeout(() => resultCallback({ code: ExportResultCode.FAILED }), 0); + return; + } + + let envelopes: Envelope[] = resourceMetricsToEnvelope( + metrics, + this._instrumentationKey, + true // isStatsbeat flag passed to create a Statsbeat envelope. + ); + // Supress tracing until OpenTelemetry Metrics SDK support it + context.with(suppressTracing(context.active()), async () => { + resultCallback(await this._exportEnvelopes(envelopes)); + }); + } + + /** + * Shutdown AzureMonitorStatsbeatExporter. + */ + public async shutdown(): Promise { + this._isShutdown = true; + return this._shutdown(); + } + + /** + * Force flush. + */ + public async forceFlush(): Promise { + return Promise.resolve(); + } +} diff --git a/sdk/monitor/monitor-opentelemetry-exporter/src/export/statsbeat/statsbeatMetrics.ts b/sdk/monitor/monitor-opentelemetry-exporter/src/export/statsbeat/statsbeatMetrics.ts new file mode 100644 index 000000000000..7bec84520e0f --- /dev/null +++ b/sdk/monitor/monitor-opentelemetry-exporter/src/export/statsbeat/statsbeatMetrics.ts @@ -0,0 +1,416 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import { + createDefaultHttpClient, + createPipelineRequest, + HttpMethods, +} from "@azure/core-rest-pipeline"; +import { diag } from "@opentelemetry/api"; +import { + BatchObservableResult, + ObservableGauge, + ObservableResult, +} from "@opentelemetry/api-metrics"; +import { Meter } from "@opentelemetry/api-metrics/build/src/types/Meter"; +import { + MeterProvider, + PeriodicExportingMetricReader, + PeriodicExportingMetricReaderOptions, +} from "@opentelemetry/sdk-metrics"; +import { AzureMonitorExporterOptions, _AzureMonitorStatsbeatExporter } from "../../index"; +import * as ai from "../../utils/constants/applicationinsights"; +import { + StatsbeatCounter, + StatsbeatResourceProvider, + STATSBEAT_LANGUAGE, + NetworkStatsbeat, + AIMS_URI, + AIMS_API_VERSION, + AIMS_FORMAT, + EU_CONNECTION_STRING, + EU_ENDPOINTS, + NON_EU_CONNECTION_STRING, + CommonStatsbeatProperties, + NetworkStatsbeatProperties, +} from "./types"; + +const os = require("os"); + +export class StatsbeatMetrics { + private _commonProperties: CommonStatsbeatProperties; + private _networkProperties: NetworkStatsbeatProperties; + private _meter: Meter; + private _isInitialized: boolean = false; + private _networkStatsbeatCollection: Array = []; + private _meterProvider: MeterProvider; + private _azureExporter: _AzureMonitorStatsbeatExporter; + private _metricReader: PeriodicExportingMetricReader; + private _statsCollectionShortInterval: number = 900000; // 15 minutes + + // Custom dimensions + private _resourceProvider: string = StatsbeatResourceProvider.unknown; + private _os: string = os.type(); + private _cikey: string; + private _runtimeVersion: string; + private _language: string; + private _version: string; + private _attach: string = "sdk"; + + // Observable Gauges + private _successCountGauge: ObservableGauge; + private _failureCountGauge: ObservableGauge; + private _retryCountGauge: ObservableGauge; + private _throttleCountGauge: ObservableGauge; + private _exceptionCountGauge: ObservableGauge; + private _averageDurationGauge: ObservableGauge; + + // Network attributes + private _connectionString: string; + private _endpointUrl: string; + private _host: string; + + constructor(instrumentationKey: string, endpointUrl: string) { + this._connectionString = this._getConnectionString(endpointUrl); + this._meterProvider = new MeterProvider(); + + const exporterConfig: AzureMonitorExporterOptions = { + connectionString: this._connectionString, + }; + + this._azureExporter = new _AzureMonitorStatsbeatExporter(exporterConfig); + + const metricReaderOptions: PeriodicExportingMetricReaderOptions = { + exporter: this._azureExporter, + exportIntervalMillis: this._statsCollectionShortInterval, // 15 minutes + }; + + // Exports Network Statsbeat every 15 minutes + this._metricReader = new PeriodicExportingMetricReader(metricReaderOptions); + this._meterProvider.addMetricReader(this._metricReader); + this._meter = this._meterProvider.getMeter("Azure Monitor NetworkStatsbeat"); + + this._endpointUrl = endpointUrl; + this._runtimeVersion = process.version; + this._language = STATSBEAT_LANGUAGE; + this._version = ai.packageVersion; + this._host = this._getShortHost(endpointUrl); + this._cikey = instrumentationKey; + + this._successCountGauge = this._meter.createObservableGauge(StatsbeatCounter.SUCCESS_COUNT); + this._failureCountGauge = this._meter.createObservableGauge(StatsbeatCounter.FAILURE_COUNT); + this._retryCountGauge = this._meter.createObservableGauge(StatsbeatCounter.RETRY_COUNT); + this._throttleCountGauge = this._meter.createObservableGauge(StatsbeatCounter.THROTTLE_COUNT); + this._exceptionCountGauge = this._meter.createObservableGauge(StatsbeatCounter.EXCEPTION_COUNT); + this._averageDurationGauge = this._meter.createObservableGauge( + StatsbeatCounter.AVERAGE_DURATION + ); + + this._commonProperties = { + os: this._os, + rp: this._resourceProvider, + cikey: this._cikey, + runtimeVersion: this._runtimeVersion, + language: this._language, + version: this._version, + attach: this._attach, + }; + + this._networkProperties = { + endpoint: this._endpointUrl, + host: this._host, + }; + + this._isInitialized = true; + this._initialize(); + } + + private async _getResourceProvider(): Promise { + // Check resource provider + this._resourceProvider = StatsbeatResourceProvider.unknown; + if (process.env.WEBSITE_SITE_NAME) { + // Web apps + this._resourceProvider = StatsbeatResourceProvider.appsvc; + } else if (process.env.FUNCTIONS_WORKER_RUNTIME) { + // Function apps + this._resourceProvider = StatsbeatResourceProvider.functions; + } else if (await this.getAzureComputeMetadata()) { + this._resourceProvider = StatsbeatResourceProvider.vm; + } else { + this._resourceProvider = StatsbeatResourceProvider.unknown; + } + } + + public async getAzureComputeMetadata(): Promise { + const httpClient = createDefaultHttpClient(); + const method: HttpMethods = "GET"; + + const options = { + url: `${AIMS_URI}?${AIMS_API_VERSION}&${AIMS_FORMAT}`, + timeout: 5000, // 5 seconds + method: method, + allowInsecureConnection: true, + }; + const request = createPipelineRequest(options); + + await httpClient + .sendRequest(request) + .then((res: any) => { + if (res.status === 200) { + return true; + } else { + return false; + } + }) + .catch(() => { + return false; + }); + return false; + } + + public isInitialized() { + return this._isInitialized; + } + + public shutdown() { + this._meterProvider.shutdown(); + } + + private async _initialize() { + try { + await this._getResourceProvider(); + + // Add observable callbacks + this._successCountGauge.addCallback(this._successCallback.bind(this)); + this._meter.addBatchObservableCallback(this._failureCallback.bind(this), [ + this._failureCountGauge, + ]); + this._meter.addBatchObservableCallback(this._retryCallback.bind(this), [ + this._retryCountGauge, + ]); + this._meter.addBatchObservableCallback(this._throttleCallback.bind(this), [ + this._throttleCountGauge, + ]); + this._meter.addBatchObservableCallback(this._exceptionCallback.bind(this), [ + this._exceptionCountGauge, + ]); + this._averageDurationGauge.addCallback(this._durationCallback.bind(this)); + } catch (error) { + diag.debug("Call to get the resource provider failed."); + } + } + + // Observable gauge callbacks + private _successCallback(observableResult: ObservableResult) { + let counter: NetworkStatsbeat = this._getNetworkStatsbeatCounter(this._endpointUrl, this._host); + let attributes = { ...this._commonProperties, ...this._networkProperties }; + observableResult.observe(counter.totalSuccesfulRequestCount, attributes); + counter.totalSuccesfulRequestCount = 0; + } + + private _failureCallback(observableResult: BatchObservableResult) { + let counter: NetworkStatsbeat = this._getNetworkStatsbeatCounter(this._endpointUrl, this._host); + + /* + Takes the failureCountGauge, value (of the counter), and attributes + create a unqiue counter based on statusCode as well + append statusCode to attributes so the newly created attributes are unique. + */ + let attributes = { ...this._networkProperties, ...this._commonProperties, statusCode: 0 }; + + // For each { statusCode -> count } mapping, call observe, passing the count and attributes that include the statusCode + for (let i = 0; i < counter.totalFailedRequestCount.length; i++) { + attributes.statusCode = counter.totalFailedRequestCount[i].statusCode; + observableResult.observe( + this._failureCountGauge, + counter.totalFailedRequestCount[i].count, + attributes + ); + counter.totalFailedRequestCount[i].count = 0; + } + } + + private _retryCallback(observableResult: BatchObservableResult) { + let counter: NetworkStatsbeat = this._getNetworkStatsbeatCounter(this._endpointUrl, this._host); + let attributes = { ...this._networkProperties, ...this._commonProperties, statusCode: 0 }; + + for (let i = 0; i < counter.retryCount.length; i++) { + attributes.statusCode = counter.retryCount[i].statusCode; + observableResult.observe(this._retryCountGauge, counter.retryCount[i].count, attributes); + counter.retryCount[i].count = 0; + } + } + + private _throttleCallback(observableResult: BatchObservableResult) { + let counter: NetworkStatsbeat = this._getNetworkStatsbeatCounter(this._endpointUrl, this._host); + let attributes = { ...this._networkProperties, ...this._commonProperties, statusCode: 0 }; + + for (let i = 0; i < counter.throttleCount.length; i++) { + attributes.statusCode = counter.throttleCount[i].statusCode; + observableResult.observe( + this._throttleCountGauge, + counter.throttleCount[i].count, + attributes + ); + counter.throttleCount[i].count = 0; + } + } + + private _exceptionCallback(observableResult: BatchObservableResult) { + let counter: NetworkStatsbeat = this._getNetworkStatsbeatCounter(this._endpointUrl, this._host); + let attributes = { ...this._networkProperties, ...this._commonProperties, exceptionType: "" }; + + for (let i = 0; i < counter.exceptionCount.length; i++) { + attributes.exceptionType = counter.exceptionCount[i].exceptionType; + observableResult.observe( + this._exceptionCountGauge, + counter.exceptionCount[i].count, + attributes + ); + counter.exceptionCount[i].count = 0; + } + } + + private _durationCallback(observableResult: ObservableResult) { + let counter: NetworkStatsbeat = this._getNetworkStatsbeatCounter(this._endpointUrl, this._host); + let attributes = { ...this._networkProperties, ...this._commonProperties }; + observableResult.observe(counter.averageRequestExecutionTime, attributes); + counter.averageRequestExecutionTime = 0; + } + + // Public methods to increase counters + public countSuccess(duration: number) { + if (!this._isInitialized) { + return; + } + let counter: NetworkStatsbeat = this._getNetworkStatsbeatCounter(this._endpointUrl, this._host); + counter.totalRequestCount++; + counter.totalSuccesfulRequestCount++; + counter.intervalRequestExecutionTime += duration; + } + + public countFailure(duration: number, statusCode: number) { + if (!this._isInitialized) { + return; + } + let counter: NetworkStatsbeat = this._getNetworkStatsbeatCounter(this._endpointUrl, this._host); + let currentStatusCounter = counter.totalFailedRequestCount.find( + (statusCounter) => statusCode === statusCounter.statusCode + ); + + if (currentStatusCounter) { + currentStatusCounter.count++; + } else { + counter.totalFailedRequestCount.push({ statusCode: statusCode, count: 1 }); + } + + counter.totalRequestCount++; + counter.intervalRequestExecutionTime += duration; + } + + public countRetry(statusCode: number) { + if (!this._isInitialized) { + return; + } + let counter: NetworkStatsbeat = this._getNetworkStatsbeatCounter(this._endpointUrl, this._host); + let currentStatusCounter = counter.retryCount.find( + (statusCounter) => statusCode === statusCounter.statusCode + ); + + if (currentStatusCounter) { + currentStatusCounter.count++; + } else { + counter.retryCount.push({ statusCode: statusCode, count: 1 }); + } + } + + public countThrottle(statusCode: number) { + if (!this._isInitialized) { + return; + } + let counter: NetworkStatsbeat = this._getNetworkStatsbeatCounter(this._endpointUrl, this._host); + let currentStatusCounter = counter.throttleCount.find( + (statusCounter) => statusCode === statusCounter.statusCode + ); + + if (currentStatusCounter) { + currentStatusCounter.count++; + } else { + counter.throttleCount.push({ statusCode: statusCode, count: 1 }); + } + } + + public countException(exceptionType: Error) { + if (!this._isInitialized) { + return; + } + let counter: NetworkStatsbeat = this._getNetworkStatsbeatCounter(this._endpointUrl, this._host); + let currentErrorCounter = counter.exceptionCount.find( + (exceptionCounter) => exceptionType.name === exceptionCounter.exceptionType + ); + if (currentErrorCounter) { + currentErrorCounter.count++; + } else { + counter.exceptionCount.push({ exceptionType: exceptionType.name, count: 1 }); + } + } + + public countAverageDuration() { + for (let i = 0; i < this._networkStatsbeatCollection.length; i++) { + let currentCounter = this._networkStatsbeatCollection[i]; + currentCounter.time = Number(new Date()); + let intervalRequests = + currentCounter.totalRequestCount - currentCounter.lastRequestCount || 0; + currentCounter.averageRequestExecutionTime = + (currentCounter.intervalRequestExecutionTime - + currentCounter.lastIntervalRequestExecutionTime) / + intervalRequests || 0; + currentCounter.lastIntervalRequestExecutionTime = currentCounter.intervalRequestExecutionTime; // reset + + currentCounter.lastRequestCount = currentCounter.totalRequestCount; + currentCounter.lastTime = currentCounter.time; + } + } + + // Gets a networkStatsbeat counter if one exists for the given endpoint + private _getNetworkStatsbeatCounter(endpoint: string, host: string): NetworkStatsbeat { + // Check if the counter is available + for (let i = 0; i < this._networkStatsbeatCollection.length; i++) { + // Same object + if ( + endpoint === this._networkStatsbeatCollection[i].endpoint && + host === this._networkStatsbeatCollection[i].host + ) { + return this._networkStatsbeatCollection[i]; + } + } + // Create a new counter if not found + let newCounter = new NetworkStatsbeat(endpoint, host); + this._networkStatsbeatCollection.push(newCounter); + return newCounter; + } + + private _getShortHost(originalHost: string) { + let shortHost = originalHost; + try { + let hostRegex = new RegExp(/^https?:\/\/(?:www\.)?([^\/.-]+)/); + let res = hostRegex.exec(originalHost); + if (res != null && res.length > 1) { + shortHost = res[1]; + } + } catch (error) { + diag.debug("Failed to get the short host name."); + } + return shortHost; + } + + private _getConnectionString(endpointUrl: string) { + let currentEndpoint = endpointUrl; + for (let i = 0; i < EU_ENDPOINTS.length; i++) { + if (currentEndpoint.includes(EU_ENDPOINTS[i])) { + return EU_CONNECTION_STRING; + } + } + return NON_EU_CONNECTION_STRING; + } +} diff --git a/sdk/monitor/monitor-opentelemetry-exporter/src/export/statsbeat/types.ts b/sdk/monitor/monitor-opentelemetry-exporter/src/export/statsbeat/types.ts new file mode 100644 index 000000000000..431802314c4c --- /dev/null +++ b/sdk/monitor/monitor-opentelemetry-exporter/src/export/statsbeat/types.ts @@ -0,0 +1,107 @@ +export class NetworkStatsbeat { + public time: number | undefined; + + public lastTime: number; + + public endpoint: string; + + public host: string; + + public totalRequestCount: number; + + public lastRequestCount: number; + + public totalSuccesfulRequestCount: number; + + public totalFailedRequestCount: { statusCode: number; count: number }[]; + + public retryCount: { statusCode: number; count: number }[]; + + public exceptionCount: { exceptionType: string; count: number }[]; + + public throttleCount: { statusCode: number; count: number }[]; + + public intervalRequestExecutionTime: number; + + public lastIntervalRequestExecutionTime: number; + + public averageRequestExecutionTime: number; + + constructor(endpoint: string, host: string) { + this.endpoint = endpoint; + this.host = host; + this.totalRequestCount = 0; + this.totalSuccesfulRequestCount = 0; + this.totalFailedRequestCount = []; + this.retryCount = []; + this.exceptionCount = []; + this.throttleCount = []; + this.intervalRequestExecutionTime = 0; + this.lastIntervalRequestExecutionTime = 0; + this.lastTime = +new Date(); + this.lastRequestCount = 0; + this.averageRequestExecutionTime = 0; + } +} + +export const STATSBEAT_LANGUAGE = "node"; + +export const MAX_STATSBEAT_FAILURES = 3; + +export const StatsbeatResourceProvider = { + appsvc: "appsvc", + functions: "functions", + vm: "vm", + unknown: "unknown", +}; + +export enum StatsbeatCounter { + SUCCESS_COUNT = "Request Success Count", + FAILURE_COUNT = "Request Failure Count", + RETRY_COUNT = "Retry Count", + THROTTLE_COUNT = "Throttle Count", + EXCEPTION_COUNT = "Exception Count", + AVERAGE_DURATION = "Request Duration", +} + +export const AIMS_URI = "http://169.254.169.254/metadata/instance/compute"; +export const AIMS_API_VERSION = "api-version=2017-12-01"; +export const AIMS_FORMAT = "format=json"; +export const NON_EU_CONNECTION_STRING = + "InstrumentationKey=c4a29126-a7cb-47e5-b348-11414998b11e;IngestionEndpoint=https://westus-0.in.applicationinsights.azure.com"; +export const EU_CONNECTION_STRING = + "InstrumentationKey=7dc56bab-3c0c-4e9f-9ebb-d1acadee8d0f;IngestionEndpoint=https://westeurope-5.in.applicationinsights.azure.com"; +export const EU_ENDPOINTS = [ + "westeurope", + "northeurope", + "francecentral", + "francesouth", + "germanywestcentral", + "norwayeast", + "norwaywest", + "swedencentral", + "switzerlandnorth", + "switzerlandwest", +]; + +export interface IVirtualMachineInfo { + isVM?: boolean; + id?: string; + subscriptionId?: string; + osType?: string; +} + +export interface CommonStatsbeatProperties { + os: string; + rp: string; + cikey: string; + runtimeVersion: string; + language: string; + version: string; + attach: string; +} + +export interface NetworkStatsbeatProperties { + endpoint: string; + host: string; +} diff --git a/sdk/monitor/monitor-opentelemetry-exporter/src/index.ts b/sdk/monitor/monitor-opentelemetry-exporter/src/index.ts index c28dcc6f6c05..4a0d12d6724c 100644 --- a/sdk/monitor/monitor-opentelemetry-exporter/src/index.ts +++ b/sdk/monitor/monitor-opentelemetry-exporter/src/index.ts @@ -5,6 +5,7 @@ export { ApplicationInsightsSampler } from "./sampling"; export { AzureMonitorBaseExporter } from "./export/base"; export { AzureMonitorTraceExporter } from "./export/trace"; export { AzureMonitorMetricExporter } from "./export/metric"; +export { _AzureMonitorStatsbeatExporter } from "./export/statsbeat"; export { AzureMonitorExporterOptions } from "./config"; export { ServiceApiVersion } from "./Declarations/Constants"; export { diff --git a/sdk/monitor/monitor-opentelemetry-exporter/src/utils/breezeUtils.ts b/sdk/monitor/monitor-opentelemetry-exporter/src/utils/breezeUtils.ts index 988f46303bfe..d6e4d7dfb8d9 100644 --- a/sdk/monitor/monitor-opentelemetry-exporter/src/utils/breezeUtils.ts +++ b/sdk/monitor/monitor-opentelemetry-exporter/src/utils/breezeUtils.ts @@ -32,6 +32,7 @@ export function isRetriable(statusCode: number): boolean { statusCode === 403 || // Forbidden statusCode === 408 || // Timeout statusCode === 429 || // Too many requests + statusCode === 439 || // Daily quota exceeded (legacy) statusCode === 500 || // Server Error statusCode === 502 || // Bad Gateway statusCode === 503 || // Server Unavailable diff --git a/sdk/monitor/monitor-opentelemetry-exporter/src/utils/metricUtils.ts b/sdk/monitor/monitor-opentelemetry-exporter/src/utils/metricUtils.ts index fd38cf43709b..207ee6129bd2 100644 --- a/sdk/monitor/monitor-opentelemetry-exporter/src/utils/metricUtils.ts +++ b/sdk/monitor/monitor-opentelemetry-exporter/src/utils/metricUtils.ts @@ -33,11 +33,22 @@ function createPropertiesFromMetricAttributes(attributes?: MetricAttributes): { * Metric to Azure envelope parsing. * @internal */ -export function resourceMetricsToEnvelope(metrics: ResourceMetrics, ikey: string): Envelope[] { +export function resourceMetricsToEnvelope( + metrics: ResourceMetrics, + ikey: string, + isStatsbeat?: boolean +): Envelope[] { let envelopes: Envelope[] = []; const time = new Date(); const instrumentationKey = ikey; const tags = createTagsFromResource(metrics.resource); + let envelopeName: string; + + if (isStatsbeat) { + envelopeName = "Microsoft.ApplicationInsights.Statsbeat"; + } else { + envelopeName = "Microsoft.ApplicationInsights.Metric"; + } metrics.scopeMetrics.forEach((scopeMetric) => { scopeMetric.metrics.forEach((metric) => { @@ -76,7 +87,7 @@ export function resourceMetricsToEnvelope(metrics: ResourceMetrics, ikey: string } baseData.metrics.push(metricDataPoint); let envelope: Envelope = { - name: "Microsoft.ApplicationInsights.Metric", + name: envelopeName, time: time, sampleRate: 100, // Metrics are never sampled instrumentationKey: instrumentationKey, diff --git a/sdk/monitor/monitor-opentelemetry-exporter/test/internal/base.exporter.test.ts b/sdk/monitor/monitor-opentelemetry-exporter/test/internal/base.exporter.test.ts index 5c2016f30237..f634bc83fd76 100644 --- a/sdk/monitor/monitor-opentelemetry-exporter/test/internal/base.exporter.test.ts +++ b/sdk/monitor/monitor-opentelemetry-exporter/test/internal/base.exporter.test.ts @@ -3,7 +3,7 @@ import * as assert from "assert"; import { ExportResult, ExportResultCode } from "@opentelemetry/core"; -import { AzureMonitorBaseExporter } from "../../src/export/base"; +import { AzureMonitorBaseExporter } from "../../src/index"; import { DEFAULT_BREEZE_ENDPOINT } from "../../src/Declarations/Constants"; import { failedBreezeResponse, diff --git a/sdk/monitor/monitor-opentelemetry-exporter/test/internal/statsbeat.test.ts b/sdk/monitor/monitor-opentelemetry-exporter/test/internal/statsbeat.test.ts new file mode 100644 index 000000000000..4829ae049a27 --- /dev/null +++ b/sdk/monitor/monitor-opentelemetry-exporter/test/internal/statsbeat.test.ts @@ -0,0 +1,307 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT license. + +import * as assert from "assert"; +import { ExportResult, ExportResultCode } from "@opentelemetry/core"; +import { failedBreezeResponse, successfulBreezeResponse } from "../utils/breezeTestUtils"; +import { AzureMonitorBaseExporter } from "../../src/index"; +import { DEFAULT_BREEZE_ENDPOINT } from "../../src/Declarations/Constants"; +import { TelemetryItem as Envelope } from "../../src/generated"; +import nock from "nock"; +import { StatsbeatMetrics } from "../../src/export/statsbeat/statsbeatMetrics"; +// @ts-ignore Need to ignore this while we do not import types +import sinon from "sinon"; + +describe("#AzureMonitorStatsbeatExporter", () => { + class TestExporter extends AzureMonitorBaseExporter { + private thisAsAny: any; + constructor() { + super({ + connectionString: `instrumentationkey=foo-ikey`, + }); + this.thisAsAny = this; + } + + getTelemetryProcesors() { + return this.thisAsAny._telemetryProcessors; + } + + async exportEnvelopesPrivate(payload: Envelope[]): Promise { + return this.thisAsAny._exportEnvelopes(payload); + } + } + + describe("Export/Statsbeat", () => { + let scope: nock.Interceptor; + const envelope = { + name: "Name", + time: new Date(), + }; + + before(() => { + scope = nock(DEFAULT_BREEZE_ENDPOINT).post("/v2.1/track"); + }); + + after(() => { + nock.cleanAll(); + }); + + describe("Initialization and connection string functions", () => { + it("should pass the options to the exporter", () => { + const exporter = new TestExporter(); + assert.ok(exporter["_options"]); + }); + it("should initialize statsbeat by default", async () => { + const exporter = new TestExporter(); + const response = successfulBreezeResponse(1); + scope.reply(200, JSON.stringify(response)); + + const result = await exporter.exportEnvelopesPrivate([envelope]); + assert.strictEqual(result.code, ExportResultCode.SUCCESS); + assert.ok(exporter["_statsbeatMetrics"]); + assert.strictEqual(exporter["_statsbeatMetrics"]?.isInitialized(), true); + }); + + it("should use non EU connection string", () => { + const statsbeat = new StatsbeatMetrics( + "InstrumentationKey=1aa11111-bbbb-1ccc-8ddd-eeeeffff3333;", + "IngestionEndpoint=https://westus-0.in.applicationinsights.azure.com" + ); + assert.strictEqual( + statsbeat["_host"], + "IngestionEndpoint=https://westus-0.in.applicationinsights.azure.com" + ); + }); + + it("should use EU connection string", () => { + const statsbeat = new StatsbeatMetrics( + "InstrumentationKey=1aa11111-bbbb-1ccc-8ddd-eeeeffff3333;", + "IngestionEndpoint=https://westeurope-5.in.applicationinsights.azure.com" + ); + assert.strictEqual( + statsbeat["_host"], + "IngestionEndpoint=https://westeurope-5.in.applicationinsights.azure.com" + ); + }); + + it("_getShortHost", () => { + const statsbeat = new StatsbeatMetrics( + "InstrumentationKey=1aa11111-bbbb-1ccc-8ddd-eeeeffff3333;", + "IngestionEndpoint=https://westeurope-5.in.applicationinsights.azure.com" + ); + assert.strictEqual( + statsbeat["_getShortHost"]("http://westus02-1.in.applicationinsights.azure.com"), + "westus02" + ); + assert.strictEqual( + statsbeat["_getShortHost"]("https://westus02-1.in.applicationinsights.azure.com"), + "westus02" + ); + assert.strictEqual( + statsbeat["_getShortHost"]("https://dc.services.visualstudio.com"), + "dc" + ); + assert.strictEqual(statsbeat["_getShortHost"]("https://www.test.com"), "test"); + }); + }); + + describe("Resource provider function", () => { + let sandbox: any; + + before(() => { + sandbox = sinon.createSandbox(); + }); + + afterEach(() => { + sandbox.restore(); + }); + + const statsbeat = new StatsbeatMetrics( + "InstrumentationKey=1aa11111-bbbb-1ccc-8ddd-eeeeffff3333;", + "IngestionEndpoint=https://westeurope-5.in.applicationinsights.azure.com" + ); + + it("it should determine if the rp is unknown", (done) => { + statsbeat["_getResourceProvider"]() + .then(() => { + assert.strictEqual(statsbeat["_resourceProvider"], "unknown"); + done(); + }) + .catch((error) => { + done(error); + }); + }); + + it("it should determine if the rp is an app service", (done) => { + let newEnv = <{ [id: string]: string }>{}; + newEnv["WEBSITE_SITE_NAME"] = "Test Website"; + newEnv["WEBSITE_HOME_STAMPNAME"] = "test_home"; + let originalEnv = process.env; + process.env = newEnv; + statsbeat["_getResourceProvider"]() + .then(() => { + process.env = originalEnv; + assert.strictEqual(statsbeat["_resourceProvider"], "appsvc"); + done(); + }) + .catch((error) => { + done(error); + }); + }); + + it("should determine if the rp is an Azure Function", (done) => { + let newEnv = <{ [id: string]: string }>{}; + newEnv["FUNCTIONS_WORKER_RUNTIME"] = "test"; + newEnv["WEBSITE_HOSTNAME"] = "test_host"; + let originalEnv = process.env; + process.env = newEnv; + statsbeat["_getResourceProvider"]() + .then(() => { + process.env = originalEnv; + assert.strictEqual(statsbeat["_resourceProvider"], "functions"); + done(); + }) + .catch((error) => { + done(error); + }); + }); + + it("should determine if the rp is an Azure VM", (done) => { + const getAzureComputeStub = sandbox.stub(statsbeat, "getAzureComputeMetadata"); + getAzureComputeStub.returns(Promise.resolve(true)); + + let newEnv = <{ [id: string]: string }>{}; + let originalEnv = process.env; + process.env = newEnv; + + statsbeat["_getResourceProvider"]() + .then(() => { + process.env = originalEnv; + assert.strictEqual(statsbeat["_resourceProvider"], "vm"); + done(); + }) + .catch((error) => { + done(error); + }); + }); + }); + + describe("Track statsbeats", () => { + it("should add correct network properites to the custom metric", (done) => { + const statsbeat = new StatsbeatMetrics( + "InstrumentationKey=1aa11111-bbbb-1ccc-8ddd-eeeeffff3333;", + "IngestionEndpoint=https://westeurope-5.in.applicationinsights.azure.com" + ); + statsbeat["_statsCollectionShortInterval"]; + statsbeat.countSuccess(100); + let metric = statsbeat["_networkStatsbeatCollection"][0]; + assert.strictEqual(metric.intervalRequestExecutionTime, 100); + + // Ensure network statsbeat attributes are populated + assert.strictEqual(statsbeat["_attach"], "sdk"); + assert.strictEqual( + statsbeat["_cikey"], + "InstrumentationKey=1aa11111-bbbb-1ccc-8ddd-eeeeffff3333;" + ); + assert.strictEqual(statsbeat["_language"], "node"); + assert.strictEqual(statsbeat["_resourceProvider"], "unknown"); + assert.strictEqual( + statsbeat["_endpointUrl"], + "IngestionEndpoint=https://westeurope-5.in.applicationinsights.azure.com" + ); + assert.ok(statsbeat["_os"]); + assert.ok(statsbeat["_runtimeVersion"]); + assert.ok(statsbeat["_version"]); + + done(); + }); + + it("should track duration", () => { + const statsbeat = new StatsbeatMetrics( + "InstrumentationKey=1aa11111-bbbb-1ccc-8ddd-eeeeffff3333;", + "IngestionEndpoint=https://westeurope-5.in.applicationinsights.azure.com" + ); + statsbeat["_statsCollectionShortInterval"] = 0; + statsbeat.countSuccess(100); + statsbeat.countSuccess(200); + statsbeat.countFailure(100, 400); + statsbeat.countFailure(500, 400); + statsbeat.countAverageDuration(); + + let metric = statsbeat["_networkStatsbeatCollection"][0]; + assert.strictEqual(metric.averageRequestExecutionTime, 225); + }); + + it("should track statsbeat counts", () => { + const statsbeat = new StatsbeatMetrics( + "InstrumentationKey=1aa11111-bbbb-1ccc-8ddd-eeeeffff3333;", + "IngestionEndpoint=https://westeurope-5.in.applicationinsights.azure.com" + ); + statsbeat["_statsCollectionShortInterval"] = 0; + statsbeat.countSuccess(100); + statsbeat.countSuccess(100); + statsbeat.countSuccess(100); + statsbeat.countSuccess(100); + statsbeat.countFailure(200, 500); + statsbeat.countFailure(100, 500); + statsbeat.countFailure(200, 501); + statsbeat.countFailure(200, 502); + statsbeat.countRetry(206); + statsbeat.countRetry(206); + statsbeat.countRetry(204); + statsbeat.countThrottle(402); + statsbeat.countThrottle(439); + statsbeat.countException({ name: "Statsbeat", message: "Statsbeat Exception" }); + statsbeat.countException({ name: "Statsbeat2", message: "Second Statsbeat Exception" }); + statsbeat.countAverageDuration(); + + let metric = statsbeat["_networkStatsbeatCollection"][0]; + + assert.ok(metric, "Statsbeat metrics not properly initialized"); + assert.strictEqual(metric.totalRequestCount, 8); + + assert.strictEqual(metric.totalSuccesfulRequestCount, 4); + + assert.strictEqual(metric.totalFailedRequestCount.length, 3); + assert.strictEqual( + metric.totalFailedRequestCount.find((failedRequest) => failedRequest.statusCode === 500) + ?.count, + 2 + ); + assert.strictEqual( + metric.totalFailedRequestCount.find((failedRequest) => failedRequest.statusCode === 501) + ?.count, + 1 + ); + + assert.strictEqual(metric.retryCount.length, 2); + assert.strictEqual( + metric.retryCount.find((retryRequest) => retryRequest.statusCode === 206)?.count, + 2 + ); + + assert.strictEqual( + metric.exceptionCount.find( + (exceptionRequest) => exceptionRequest.exceptionType === "Statsbeat" + )?.count, + 1 + ); + assert.strictEqual( + metric.throttleCount.find((throttledRequest) => throttledRequest.statusCode === 439) + ?.count, + 1 + ); + }); + + it("should turn off statsbeat after max failures", async () => { + const exporter = new TestExporter(); + const response = failedBreezeResponse(1, 200); + scope.reply(200, JSON.stringify(response)); + exporter["_statsbeatFailureCount"] = 4; + + const result = await exporter["_exportEnvelopes"]([envelope]); + assert.strictEqual(result.code, ExportResultCode.SUCCESS); + }); + }); + }); +});