From 59294114b3b0532f17065ebe822bff43094d4e1f Mon Sep 17 00:00:00 2001 From: Hector Hernandez Guzman Date: Tue, 15 Dec 2020 13:40:01 -0800 Subject: [PATCH 1/4] OT Exporter retry when there are network issues --- .../opentelemetry-exporter-azure-monitor/src/export/trace.ts | 4 ++-- .../test/unit/export/trace.test.ts | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sdk/monitor/opentelemetry-exporter-azure-monitor/src/export/trace.ts b/sdk/monitor/opentelemetry-exporter-azure-monitor/src/export/trace.ts index 270b37d6022c..11c7e0a7f75c 100644 --- a/sdk/monitor/opentelemetry-exporter-azure-monitor/src/export/trace.ts +++ b/sdk/monitor/opentelemetry-exporter-azure-monitor/src/export/trace.ts @@ -107,10 +107,10 @@ export class AzureMonitorTraceExporter implements SpanExporter { } catch (senderErr) { // Request failed -- always retry this._logger.error( - "Envelopes could not be exported and are not retriable. Error message:", + "Retrying due to transient client side error. Error message:", senderErr.message ); - return ExportResult.FAILED_NOT_RETRYABLE; + return ExportResult.FAILED_RETRYABLE; } } diff --git a/sdk/monitor/opentelemetry-exporter-azure-monitor/test/unit/export/trace.test.ts b/sdk/monitor/opentelemetry-exporter-azure-monitor/test/unit/export/trace.test.ts index c312db8b9667..d1ae63bbc81d 100644 --- a/sdk/monitor/opentelemetry-exporter-azure-monitor/test/unit/export/trace.test.ts +++ b/sdk/monitor/opentelemetry-exporter-azure-monitor/test/unit/export/trace.test.ts @@ -103,12 +103,12 @@ describe("#AzureMonitorBaseExporter", () => { assert.strictEqual(persistedEnvelopes, null); }); - it("should not persist when an error is caught", async () => { + it("should persist when an error is caught", async () => { const exporter = new TestExporter(); scope.reply(1, ""); // httpSender will throw const result = await exporter.exportEnvelopesPrivate([envelope]); - assert.strictEqual(result, ExportResult.FAILED_NOT_RETRYABLE); + assert.strictEqual(result, ExportResult.FAILED_RETRYABLE); const persistedEnvelopes = await exporter["_persister"].shift(); assert.strictEqual(persistedEnvelopes, null); From fc3fac82bd5759999aea998559b12649cd1caffc Mon Sep 17 00:00:00 2001 From: Hector Hernandez Guzman Date: Tue, 5 Jan 2021 11:43:48 -0800 Subject: [PATCH 2/4] Adding network error check --- .../CHANGELOG.md | 2 + .../src/export/trace.ts | 38 ++++++++++++++++--- .../test/unit/export/trace.test.ts | 10 +++-- 3 files changed, 40 insertions(+), 10 deletions(-) diff --git a/sdk/monitor/opentelemetry-exporter-azure-monitor/CHANGELOG.md b/sdk/monitor/opentelemetry-exporter-azure-monitor/CHANGELOG.md index a6a89d242e3f..5e4ef205587e 100644 --- a/sdk/monitor/opentelemetry-exporter-azure-monitor/CHANGELOG.md +++ b/sdk/monitor/opentelemetry-exporter-azure-monitor/CHANGELOG.md @@ -2,6 +2,8 @@ ## 1.0.0-beta.1 (Unreleased) +- OT Exporter retry when there are network issues +- OpenTelemetry Exporter using Resources API to get service properties - Rename package to `@azure/opentelemetry-exporter-azure-monitor` - [BREAKING] Deprecate all configuration options except for `connectionString` - [BREAKING] Removed support for `TelemetryProcessor` diff --git a/sdk/monitor/opentelemetry-exporter-azure-monitor/src/export/trace.ts b/sdk/monitor/opentelemetry-exporter-azure-monitor/src/export/trace.ts index 11c7e0a7f75c..64c64e049e3f 100644 --- a/sdk/monitor/opentelemetry-exporter-azure-monitor/src/export/trace.ts +++ b/sdk/monitor/opentelemetry-exporter-azure-monitor/src/export/trace.ts @@ -4,6 +4,7 @@ import { Logger } from "@opentelemetry/api"; import { ConsoleLogger, LogLevel, ExportResult } from "@opentelemetry/core"; import { ReadableSpan, SpanExporter } from "@opentelemetry/tracing"; +import { RestError } from "@azure/core-http"; import { ConnectionStringParser } from "../utils/connectionStringParser"; import { HttpSender, FileSystemPersist } from "../platform"; import { @@ -105,12 +106,20 @@ export class AzureMonitorTraceExporter implements SpanExporter { return ExportResult.FAILED_NOT_RETRYABLE; } } catch (senderErr) { - // Request failed -- always retry - this._logger.error( - "Retrying due to transient client side error. Error message:", - senderErr.message - ); - return ExportResult.FAILED_RETRYABLE; + if (this._isNetworkError(senderErr)) { + this._logger.error( + "Retrying due to transient client side error. Error message:", + senderErr.message + ); + return ExportResult.FAILED_RETRYABLE; + } + else { + this._logger.error( + "Envelopes could not be exported and are not retriable. Error message:", + senderErr.message + ); + return ExportResult.FAILED_NOT_RETRYABLE; + } } } @@ -140,4 +149,21 @@ export class AzureMonitorTraceExporter implements SpanExporter { this._logger.warn(`Failed to fetch persisted file`, err); } } + + private _isNetworkError(error: Error): boolean { + if (error instanceof RestError) { + if ( + error && + error.code && + (error.code === "ETIMEDOUT" || + error.code === "ESOCKETTIMEDOUT" || + error.code === "ECONNREFUSED" || + error.code === "ECONNRESET" || + error.code === "ENOENT") + ) { + return true; + } + } + return false; + } } diff --git a/sdk/monitor/opentelemetry-exporter-azure-monitor/test/unit/export/trace.test.ts b/sdk/monitor/opentelemetry-exporter-azure-monitor/test/unit/export/trace.test.ts index d1ae63bbc81d..e391ded343af 100644 --- a/sdk/monitor/opentelemetry-exporter-azure-monitor/test/unit/export/trace.test.ts +++ b/sdk/monitor/opentelemetry-exporter-azure-monitor/test/unit/export/trace.test.ts @@ -52,7 +52,9 @@ describe("#AzureMonitorBaseExporter", () => { describe("Sender/Persister Controller", () => { describe("#exportEnvelopes()", () => { - const scope = nock(DEFAULT_BREEZE_ENDPOINT).post("/v2/track"); + const scope = nock(DEFAULT_BREEZE_ENDPOINT) + .log(console.log) + .post("/v2/track"); const envelope = { name: "Name", time: new Date() @@ -103,12 +105,12 @@ describe("#AzureMonitorBaseExporter", () => { assert.strictEqual(persistedEnvelopes, null); }); - it("should persist when an error is caught", async () => { + it("should not persist when an error is caught", async () => { const exporter = new TestExporter(); - scope.reply(1, ""); // httpSender will throw + scope.replyWithError("Error"); const result = await exporter.exportEnvelopesPrivate([envelope]); - assert.strictEqual(result, ExportResult.FAILED_RETRYABLE); + assert.strictEqual(result, ExportResult.FAILED_NOT_RETRYABLE); const persistedEnvelopes = await exporter["_persister"].shift(); assert.strictEqual(persistedEnvelopes, null); From a27c946f33da4f266f69073155f2b6ae83d4e630 Mon Sep 17 00:00:00 2001 From: Hector Hernandez Guzman Date: Tue, 5 Jan 2021 15:14:29 -0800 Subject: [PATCH 3/4] Fixing issue with tests --- .../test/unit/export/trace.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/monitor/opentelemetry-exporter-azure-monitor/test/unit/export/trace.test.ts b/sdk/monitor/opentelemetry-exporter-azure-monitor/test/unit/export/trace.test.ts index e391ded343af..37ec8832d05e 100644 --- a/sdk/monitor/opentelemetry-exporter-azure-monitor/test/unit/export/trace.test.ts +++ b/sdk/monitor/opentelemetry-exporter-azure-monitor/test/unit/export/trace.test.ts @@ -107,7 +107,7 @@ describe("#AzureMonitorBaseExporter", () => { it("should not persist when an error is caught", async () => { const exporter = new TestExporter(); - scope.replyWithError("Error"); + scope.reply(1, ""); // httpSender will throw const result = await exporter.exportEnvelopesPrivate([envelope]); assert.strictEqual(result, ExportResult.FAILED_NOT_RETRYABLE); From 182c7b34ad1912d77ed0eb025961a5b5ae101a8a Mon Sep 17 00:00:00 2001 From: Hector Hernandez Guzman Date: Thu, 7 Jan 2021 11:07:35 -0800 Subject: [PATCH 4/4] reverting unnecessary change --- .../test/unit/export/trace.test.ts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/sdk/monitor/opentelemetry-exporter-azure-monitor/test/unit/export/trace.test.ts b/sdk/monitor/opentelemetry-exporter-azure-monitor/test/unit/export/trace.test.ts index 37ec8832d05e..c312db8b9667 100644 --- a/sdk/monitor/opentelemetry-exporter-azure-monitor/test/unit/export/trace.test.ts +++ b/sdk/monitor/opentelemetry-exporter-azure-monitor/test/unit/export/trace.test.ts @@ -52,9 +52,7 @@ describe("#AzureMonitorBaseExporter", () => { describe("Sender/Persister Controller", () => { describe("#exportEnvelopes()", () => { - const scope = nock(DEFAULT_BREEZE_ENDPOINT) - .log(console.log) - .post("/v2/track"); + const scope = nock(DEFAULT_BREEZE_ENDPOINT).post("/v2/track"); const envelope = { name: "Name", time: new Date()