use our own app insights client to record test results

package.json

@@ -2299,6 +2299,7 @@
         "@vscode/test-electron": "^1.6.1",
         "@vscode/test-web": "^0.0.24",
         "acorn": "^6.4.1",
+        "applicationinsights": "^2.3.1",
         "babel-polyfill": "^6.26.0",
         "bufferutil": "^4.0.6",
         "chai": "^4.3.0",

src/telemetry/index.ts

@@ -155,7 +155,7 @@ export function sendTelemetryEvent<P extends IEventNamePropertyMapping, E extend
     ex?: Error,
     sendOriginalEventWithErrors?: boolean
 ) {
-    if (!isTelemetrySupported() || (isTestExecution() && eventName !== Telemetry.RunTest)) {
+    if (!isTelemetrySupported() || isTestExecution()) {
         return;
     }
     // If stuff is already queued, then queue the rest.

src/test/testHooks.ts

@@ -1,20 +1,47 @@
 import { Context } from 'mocha';
-import { Telemetry } from '../platform/common/constants';
-import { sendTelemetryEvent } from '../telemetry';
+import { AppinsightsKey, JVSC_EXTENSION_ID, Telemetry } from '../platform/common/constants';
 import { IS_CI_SERVER } from './ciConstants.node';
+import { extensions } from 'vscode';
+import { sleep } from './core';
+import { CiTelemetryReporter } from './utils/ciTelemetry/ciTelemetryReporter.node';
+
+let telemetryReporter: CiTelemetryReporter | undefined;
 
 export const rootHooks = {
+    beforeAll() {
+        if (!IS_CI_SERVER) {
+            return;
+        }
+
+        const extensionVersion = extensions.getExtension(JVSC_EXTENSION_ID)?.packageJSON.version;
+        telemetryReporter = new CiTelemetryReporter(JVSC_EXTENSION_ID, extensionVersion, AppinsightsKey, true);
+    },
     afterEach(this: Context) {
         if (!IS_CI_SERVER) {
             return;
         }
 
         let result = this.currentTest?.isFailed() ? 'failed' : this.currentTest?.isPassed() ? 'passed' : 'skipped';
         if (this.currentTest?.title) {
-            sendTelemetryEvent(Telemetry.RunTest, this.currentTest?.duration, {
-                testName: this.currentTest?.title,
-                testResult: result
-            });
+            const duration = this.currentTest?.duration;
+            const measures = typeof duration === 'number' ? { duration: duration } : duration ? duration : undefined;
+            telemetryReporter?.sendRawTelemetryEvent(
+                Telemetry.RunTest,
+                {
+                    testName: this.currentTest?.title,
+                    testResult: result
+                },
+                measures
+            );
         }
+    },
+    afterAll: async function () {
+        if (!IS_CI_SERVER) {
+            return;
+        }
+
+        await telemetryReporter?.dispose();
+        // allow some time for the telemetry to flush
+        await sleep(2000);
     }
 };

src/test/utils/ciTelemetry/TelemetryAppender.ts

@@ -0,0 +1,98 @@
+/*---------------------------------------------------------
+ * Copyright (C) Microsoft Corporation. All rights reserved.
+ *--------------------------------------------------------*/
+
+import { AppenderData, ITelemetryAppender } from './baseCiTelemetryReporter';
+
+export interface BaseTelemetryClient {
+    logEvent(eventName: string, data?: AppenderData): void;
+    logException(exception: Error, data?: AppenderData): void;
+    flush(): void | Promise<void>;
+}
+
+export class TelemetryAppender implements ITelemetryAppender {
+    private _telemetryClient: BaseTelemetryClient | undefined;
+    private _clientInitialization: Promise<void> | undefined;
+
+    // Queues used to store events until the appender is ready
+    private _eventQueue: Array<{ eventName: string; data: AppenderData | undefined }> = [];
+    private _exceptionQueue: Array<{ exception: Error; data: AppenderData | undefined }> = [];
+
+    // Necessary information to create a telemetry client
+    private _clientFactory: (key: string) => Promise<BaseTelemetryClient>;
+    private _key: string;
+
+    constructor(key: string, clientFactory: (key: string) => Promise<BaseTelemetryClient>) {
+        this._clientFactory = clientFactory;
+        this._key = key;
+        this.instantiateAppender();
+    }
+
+    /**
+     * Sends the event to the passed in telemetry client
+     * @param eventName The named of the event to log
+     * @param data The data contanied in the event
+     */
+    logEvent(eventName: string, data?: AppenderData): void {
+        if (this._telemetryClient) {
+            this._telemetryClient.logEvent(eventName, data);
+        } else {
+            this._eventQueue.push({ eventName, data });
+        }
+    }
+
+    /**
+     * Sends an exception to the passed in telemetry client
+     * @param exception The exception to collect
+     * @param data Data associated with the exception
+     */
+    logException(exception: Error, data?: AppenderData): void {
+        if (this._telemetryClient) {
+            this._telemetryClient.logException(exception, data);
+        } else {
+            this._exceptionQueue.push({ exception, data });
+        }
+    }
+
+    /**
+     * Flushes the buffered telemetry data
+     */
+    async flush(): Promise<void> {
+        if (this._clientInitialization) {
+            await this._clientInitialization;
+            if (this._telemetryClient) {
+                await this._telemetryClient.flush();
+                this._telemetryClient = undefined;
+            }
+        }
+        return;
+    }
+
+    /**
+     * Flushes the queued events that existed before the client was instantiated
+     */
+    private _flushQueues(): void {
+        this._eventQueue.forEach(({ eventName, data }) => this.logEvent(eventName, data));
+        this._eventQueue = [];
+        this._exceptionQueue.forEach(({ exception, data }) => this.logException(exception, data));
+        this._exceptionQueue = [];
+    }
+
+    /**
+     * Instantiates the telemetry client to make the appender "active"
+     */
+    instantiateAppender(): void {
+        if (this._clientInitialization) {
+            return;
+        }
+        // Call the client factory to get the client and then let it know it's instatntiated
+        this._clientInitialization = this._clientFactory(this._key)
+            .then((client) => {
+                this._telemetryClient = client;
+                this._flushQueues();
+            })
+            .catch((err) => {
+                console.error(err);
+            });
+    }
+}

src/test/utils/ciTelemetry/appInsightsClientFactory.node.ts

@@ -0,0 +1,86 @@
+import { TelemetryClient } from 'applicationinsights';
+import { BaseTelemetryClient } from './TelemetryAppender';
+import { AppenderData } from './baseCiTelemetryReporter';
+import * as vscode from 'vscode';
+
+/**
+ * A factory function which creates a telemetry client to be used by an appender to send telemetry in a node application.
+ *
+ * @param key The app insights key
+ * @param replacementOptions Optional list of {@link ReplacementOption replacements} to apply to the telemetry client. This allows
+ * the appender to filter out any sensitive or unnecessary information from the telemetry server.
+ *
+ * @returns A promise which resolves to the telemetry client or rejects upon error
+ */
+export async function appInsightsClientFactory(key: string): Promise<BaseTelemetryClient> {
+    let appInsightsClient: TelemetryClient | undefined;
+    try {
+        process.env['APPLICATION_INSIGHTS_NO_DIAGNOSTIC_CHANNEL'] = '1';
+        const appInsights = await import('applicationinsights');
+        //check if another instance is already initialized
+        if (appInsights.defaultClient) {
+            appInsightsClient = new appInsights.TelemetryClient(key);
+            // no other way to enable offline mode
+            appInsightsClient.channel.setUseDiskRetryCaching(true);
+        } else {
+            appInsights
+                .setup(key)
+                .setAutoCollectRequests(false)
+                .setAutoCollectPerformance(false)
+                .setAutoCollectExceptions(false)
+                .setAutoCollectDependencies(false)
+                .setAutoDependencyCorrelation(false)
+                .setAutoCollectConsole(false)
+                .setAutoCollectHeartbeat(false)
+                .setUseDiskRetryCaching(true)
+                .start();
+            appInsightsClient = appInsights.defaultClient;
+        }
+        if (vscode && vscode.env) {
+            appInsightsClient.context.tags[appInsightsClient.context.keys.userId] = vscode.env.machineId;
+            appInsightsClient.context.tags[appInsightsClient.context.keys.sessionId] = vscode.env.sessionId;
+            appInsightsClient.context.tags[appInsightsClient.context.keys.cloudRole] = vscode.env.appName;
+            appInsightsClient.context.tags[appInsightsClient.context.keys.cloudRoleInstance] = vscode.env.appName;
+        }
+        //check if it's an Asimov key to change the endpoint
+        if (key && key.indexOf('AIF-') === 0) {
+            appInsightsClient.config.endpointUrl = 'https://vortex.data.microsoft.com/collect/v1';
+        }
+    } catch (e) {
+        return Promise.reject('Failed to initialize app insights!\n' + e.message);
+    }
+
+    // Sets the appinsights client into a standardized form
+    const telemetryClient: BaseTelemetryClient = {
+        logEvent: (eventName: string, data?: AppenderData) => {
+            try {
+                appInsightsClient?.trackEvent({
+                    name: eventName,
+                    properties: data?.properties,
+                    measurements: data?.measurements
+                });
+            } catch (e) {
+                throw new Error('Failed to log event to app insights!\n' + e.message);
+            }
+        },
+        logException: (exception: Error, data?: AppenderData) => {
+            try {
+                appInsightsClient?.trackException({
+                    exception,
+                    properties: data?.properties,
+                    measurements: data?.measurements
+                });
+            } catch (e) {
+                throw new Error('Failed to log exception to app insights!\n' + e.message);
+            }
+        },
+        flush: async () => {
+            try {
+                appInsightsClient?.flush();
+            } catch (e) {
+                throw new Error('Failed to flush app insights!\n' + e.message);
+            }
+        }
+    };
+    return telemetryClient;
+}