From 212476281cd09ebdd03a592b1bdcd3dcf99df815 Mon Sep 17 00:00:00 2001 From: pras0131 Date: Fri, 8 Nov 2024 12:12:40 +0100 Subject: [PATCH] Emit chat and inline events to destination (#574) * Add logging support in case of failures from STE call * Emit ChatAddMessage and ChatInteractWithMessage to destination based on client flag * Emit ChatUserModification to destination based on client flag * Inline completion events to destination behind a flag * Add unit test to check no event is send to destination when sendTelemetryEventsToDestination flag is disabled * changing the default value of enableTelemetryEventsToDestination to true * Fix the error function to catch STE call gracefully * Disable unnecessary configuration call and remove unused imports --------- Co-authored-by: Paras Co-authored-by: Viktor Shcherba --- .../chat/chatController.test.ts | 7 +- .../language-server/chat/chatController.ts | 8 +- .../chat/telemetry/chatTelemetryController.ts | 88 +- .../language-server/codeWhispererServer.ts | 50 +- .../src/language-server/telemetry.test.ts | 24 +- .../telemetry/codePercentage.test.ts | 92 ++- .../telemetry/codePercentage.ts | 28 +- .../telemetry/userTriggerDecision.test.ts | 767 ++++++++++++++---- .../language-server/telemetryService.test.ts | 361 +++++++-- .../src/language-server/telemetryService.ts | 186 ++++- 10 files changed, 1166 insertions(+), 445 deletions(-) diff --git a/server/aws-lsp-codewhisperer/src/language-server/chat/chatController.test.ts b/server/aws-lsp-codewhisperer/src/language-server/chat/chatController.test.ts index e608e35e..796fd9c2 100644 --- a/server/aws-lsp-codewhisperer/src/language-server/chat/chatController.test.ts +++ b/server/aws-lsp-codewhisperer/src/language-server/chat/chatController.test.ts @@ -83,6 +83,7 @@ describe('ChatController', () => { let chatController: ChatController let telemetryService: TelemetryService let invokeSendTelemetryEventStub: sinon.SinonStub + let telemetry: Telemetry beforeEach(() => { sendMessageStub = sinon.stub(CodeWhispererStreaming.prototype, 'sendMessage').callsFake(() => { @@ -120,7 +121,11 @@ describe('ChatController', () => { }), } - telemetryService = new TelemetryService(mockCredentialsProvider, 'bearer', {} as Telemetry, logging, {}) + telemetry = { + emitMetric: sinon.stub(), + onClientTelemetry: sinon.stub(), + } + telemetryService = new TelemetryService(mockCredentialsProvider, 'bearer', telemetry, logging, {}) invokeSendTelemetryEventStub = sinon.stub(telemetryService, 'sendTelemetryEvent' as any) chatController = new ChatController(chatSessionManagementService, testFeatures, telemetryService) }) diff --git a/server/aws-lsp-codewhisperer/src/language-server/chat/chatController.ts b/server/aws-lsp-codewhisperer/src/language-server/chat/chatController.ts index 2cc13375..7427ec5c 100644 --- a/server/aws-lsp-codewhisperer/src/language-server/chat/chatController.ts +++ b/server/aws-lsp-codewhisperer/src/language-server/chat/chatController.ts @@ -415,8 +415,12 @@ export class ChatController implements ChatHandlers { if (qConfig) { this.#customizationArn = undefinedIfEmpty(qConfig.customization) this.#log(`Chat configuration updated to use ${this.#customizationArn}`) - const enableTelemetryEventsToDestination = qConfig['enableTelemetryEventsToDestination'] === true - this.#telemetryService.updateEnableTelemetryEventsToDestination(enableTelemetryEventsToDestination) + /* + The flag enableTelemetryEventsToDestination is set to true temporarily. It's value will be determined through destination + configuration post all events migration to STE. It'll be replaced by qConfig['enableTelemetryEventsToDestination'] === true + */ + // const enableTelemetryEventsToDestination = true + // this.#telemetryService.updateEnableTelemetryEventsToDestination(enableTelemetryEventsToDestination) const optOutTelemetryPreference = qConfig['optOutTelemetry'] === true ? 'OPTOUT' : 'OPTIN' this.#telemetryService.updateOptOutPreference(optOutTelemetryPreference) } diff --git a/server/aws-lsp-codewhisperer/src/language-server/chat/telemetry/chatTelemetryController.ts b/server/aws-lsp-codewhisperer/src/language-server/chat/telemetry/chatTelemetryController.ts index 597df5e5..b83bd001 100644 --- a/server/aws-lsp-codewhisperer/src/language-server/chat/telemetry/chatTelemetryController.ts +++ b/server/aws-lsp-codewhisperer/src/language-server/chat/telemetry/chatTelemetryController.ts @@ -121,16 +121,6 @@ export class ChatTelemetryController { } public emitModifyCodeMetric(entry: AcceptedSuggestionChatEntry, percentage: number) { - const data: Omit = { - cwsprChatMessageId: entry.messageId, - cwsprChatModificationPercentage: percentage, - codewhispererCustomizationArn: entry.customizationArn, - } - - this.emitConversationMetric({ - name: ChatTelemetryEventName.ModifyCode, - data, - }) this.#telemetryService.emitChatUserModificationEvent({ conversationId: entry.conversationId, messageId: entry.messageId, @@ -175,48 +165,31 @@ export class ChatTelemetryController { public emitAddMessageMetric(tabId: string, metric: Partial) { const conversationId = this.getConversationId(tabId) - this.#telemetryService.emitChatAddMessage({ - conversationId: conversationId, - messageId: metric.cwsprChatMessageId, - customizationArn: metric.codewhispererCustomizationArn, - userIntent: metric.cwsprChatUserIntent, - hasCodeSnippet: metric.cwsprChatHasCodeSnippet, - programmingLanguage: metric.cwsprChatProgrammingLanguage as CodewhispererLanguage, - activeEditorTotalCharacters: metric.cwsprChatActiveEditorTotalCharacters, - timeToFirstChunkMilliseconds: metric.cwsprTimeToFirstChunk, - timeBetweenChunks: metric.cwsprChatTimeBetweenChunks, - fullResponselatency: metric.cwsprChatFullResponseLatency, - requestLength: metric.cwsprChatRequestLength, - responseLength: metric.cwsprChatResponseLength, - numberOfCodeBlocks: metric.cwsprChatResponseCodeSnippetCount, - }) - this.emitConversationMetric( + this.#telemetryService.emitChatAddMessage( { - name: ChatTelemetryEventName.AddMessage, - data: { - cwsprChatHasCodeSnippet: metric.cwsprChatHasCodeSnippet, - cwsprChatTriggerInteraction: metric.cwsprChatTriggerInteraction, - cwsprChatMessageId: metric.cwsprChatMessageId, - cwsprChatUserIntent: metric.cwsprChatUserIntent, - cwsprChatProgrammingLanguage: metric.cwsprChatProgrammingLanguage, - cwsprChatResponseCodeSnippetCount: metric.cwsprChatResponseCodeSnippetCount, - cwsprChatResponseCode: metric.cwsprChatResponseCode, - cwsprChatSourceLinkCount: metric.cwsprChatSourceLinkCount, - cwsprChatReferencesCount: metric.cwsprChatReferencesCount, - cwsprChatFollowUpCount: metric.cwsprChatFollowUpCount, - cwsprTimeToFirstChunk: metric.cwsprTimeToFirstChunk, - cwsprChatFullResponseLatency: metric.cwsprChatFullResponseLatency, - cwsprChatTimeBetweenChunks: metric.cwsprChatTimeBetweenChunks, - cwsprChatRequestLength: metric.cwsprChatRequestLength, - cwsprChatResponseLength: metric.cwsprChatResponseLength, - cwsprChatConversationType: metric.cwsprChatConversationType, - cwsprChatActiveEditorTotalCharacters: metric.cwsprChatActiveEditorTotalCharacters, - cwsprChatActiveEditorImportCount: metric.cwsprChatActiveEditorImportCount, - codewhispererCustomizationArn: metric.codewhispererCustomizationArn, - // not possible: cwsprChatResponseType: metric.cwsprChatResponseType, - }, + conversationId: conversationId, + messageId: metric.cwsprChatMessageId, + customizationArn: metric.codewhispererCustomizationArn, + userIntent: metric.cwsprChatUserIntent, + hasCodeSnippet: metric.cwsprChatHasCodeSnippet, + programmingLanguage: metric.cwsprChatProgrammingLanguage as CodewhispererLanguage, + activeEditorTotalCharacters: metric.cwsprChatActiveEditorTotalCharacters, + timeToFirstChunkMilliseconds: metric.cwsprTimeToFirstChunk, + timeBetweenChunks: metric.cwsprChatTimeBetweenChunks, + fullResponselatency: metric.cwsprChatFullResponseLatency, + requestLength: metric.cwsprChatRequestLength, + responseLength: metric.cwsprChatResponseLength, + numberOfCodeBlocks: metric.cwsprChatResponseCodeSnippetCount, }, - tabId + { + chatTriggerInteraction: metric.cwsprChatTriggerInteraction, + chatResponseCode: metric.cwsprChatResponseCode, + chatSourceLinkCount: metric.cwsprChatSourceLinkCount, + chatReferencesCount: metric.cwsprChatReferencesCount, + chatFollowUpCount: metric.cwsprChatFollowUpCount, + chatConversationType: metric.cwsprChatConversationType, + chatActiveEditorImportCount: metric.cwsprChatActiveEditorImportCount, + } ) // Store the customization value associated with the message if (metric.cwsprChatMessageId && metric.codewhispererCustomizationArn) { @@ -250,13 +223,6 @@ export class ChatTelemetryController { this.#telemetryService.emitChatInteractWithMessage(metric, { conversationId: this.getConversationId(tabId), }) - this.emitConversationMetric( - { - name: ChatTelemetryEventName.InteractWithMessage, - data: metric, - }, - tabId - ) } public emitMessageResponseError(tabId: string, metric: Partial) { @@ -364,10 +330,6 @@ export class ChatTelemetryController { this.#telemetryService.emitChatInteractWithMessage(voteData, { conversationId: this.getConversationId(params.tabId), }) - this.emitConversationMetric({ - name: ChatTelemetryEventName.InteractWithMessage, - data: voteData, - }) break case ChatUIEventName.InsertToCursorPosition: case ChatUIEventName.CopyToClipboard: @@ -394,10 +356,6 @@ export class ChatTelemetryController { ? params.code?.split('\n').length : undefined, }) - this.emitConversationMetric({ - name: ChatTelemetryEventName.InteractWithMessage, - data: interactData, - }) break case ChatUIEventName.LinkClick: case ChatUIEventName.InfoLinkClick: diff --git a/server/aws-lsp-codewhisperer/src/language-server/codeWhispererServer.ts b/server/aws-lsp-codewhisperer/src/language-server/codeWhispererServer.ts index 0d272ab6..e98856c2 100644 --- a/server/aws-lsp-codewhisperer/src/language-server/codeWhispererServer.ts +++ b/server/aws-lsp-codewhisperer/src/language-server/codeWhispererServer.ts @@ -177,56 +177,18 @@ const emitUserTriggerDecisionTelemetry = ( return } - emitAggregatedUserTriggerDecisionTelemetry(telemetry, telemetryService, session, timeSinceLastUserModification) + emitAggregatedUserTriggerDecisionTelemetry(telemetryService, session, timeSinceLastUserModification) emitUserDecisionTelemetry(telemetry, session) session.reportedUserDecision = true } const emitAggregatedUserTriggerDecisionTelemetry = ( - telemetry: Telemetry, telemetryService: TelemetryService, session: CodeWhispererSession, timeSinceLastUserModification?: number ) => { telemetryService.emitUserTriggerDecision(session, timeSinceLastUserModification) - // TODO: the below emitted event to the Toolkit DWH will be moved to telemetryService as well. - const data: CodeWhispererUserTriggerDecisionEvent = { - codewhispererSessionId: session.codewhispererSessionId || '', - codewhispererFirstRequestId: session.responseContext?.requestId || '', - credentialStartUrl: session.credentialStartUrl, - codewhispererSuggestionState: session.getAggregatedUserTriggerDecision(), - codewhispererCompletionType: - session.suggestions.length > 0 ? getCompletionType(session.suggestions[0]) : undefined, - codewhispererLanguage: session.language, - codewhispererTriggerType: session.triggerType, - codewhispererAutomatedTriggerType: session.autoTriggerType, - codewhispererTriggerCharacter: - session.autoTriggerType === 'SpecialCharacters' ? session.triggerCharacter : undefined, - codewhispererLineNumber: session.startPosition.line, - codewhispererCursorOffset: session.startPosition.character, - codewhispererSuggestionCount: session.suggestions.length, - codewhispererClassifierResult: session.classifierResult, - codewhispererClassifierThreshold: session.classifierThreshold, - codewhispererTotalShownTime: session.totalSessionDisplayTime || 0, - codewhispererTypeaheadLength: session.typeaheadLength || 0, - // Global time between any 2 document changes - codewhispererTimeSinceLastDocumentChange: timeSinceLastUserModification, - codewhispererTimeSinceLastUserDecision: session.previousTriggerDecisionTime - ? session.startTime - session.previousTriggerDecisionTime - : undefined, - codewhispererTimeToFirstRecommendation: session.timeToFirstRecommendation, - codewhispererPreviousSuggestionState: session.previousTriggerDecision, - codewhispererSupplementalContextTimeout: session.supplementalMetadata?.isProcessTimeout, - codewhispererSupplementalContextIsUtg: session.supplementalMetadata?.isUtg, - codewhispererSupplementalContextLength: session.supplementalMetadata?.contentsLength, - codewhispererCustomizationArn: session.customizationArn, - } - - telemetry.emitMetric({ - name: 'codewhisperer_userTriggerDecision', - data, - }) } const emitUserDecisionTelemetry = (telemetry: Telemetry, session: CodeWhispererSession) => { @@ -335,7 +297,7 @@ export const CodewhispererServerFactory = // the context of a single response. let includeSuggestionsWithCodeReferences = false - const codePercentageTracker = new CodePercentageTracker(telemetry, telemetryService) + const codePercentageTracker = new CodePercentageTracker(telemetryService) const codeDiffTracker: CodeDiffTracker = new CodeDiffTracker( workspace, @@ -640,8 +602,12 @@ export const CodewhispererServerFactory = logging.log( `Inline completion configuration updated to use ${codeWhispererService.customizationArn}` ) - const enableTelemetryEventsToDestination = qConfig['enableTelemetryEventsToDestination'] === true - telemetryService.updateEnableTelemetryEventsToDestination(enableTelemetryEventsToDestination) + /* + The flag enableTelemetryEventsToDestination is set to true temporarily. It's value will be determined through destination + configuration post all events migration to STE. It'll be replaced by qConfig['enableTelemetryEventsToDestination'] === true + */ + // const enableTelemetryEventsToDestination = true + // telemetryService.updateEnableTelemetryEventsToDestination(enableTelemetryEventsToDestination) const optOutTelemetryPreference = qConfig['optOutTelemetry'] === true ? 'OPTOUT' : 'OPTIN' telemetryService.updateOptOutPreference(optOutTelemetryPreference) } diff --git a/server/aws-lsp-codewhisperer/src/language-server/telemetry.test.ts b/server/aws-lsp-codewhisperer/src/language-server/telemetry.test.ts index befeea11..1a9488ee 100644 --- a/server/aws-lsp-codewhisperer/src/language-server/telemetry.test.ts +++ b/server/aws-lsp-codewhisperer/src/language-server/telemetry.test.ts @@ -9,6 +9,7 @@ import sinon, { StubbedInstance, stubInterface } from 'ts-sinon' import { TextDocument } from 'vscode-languageserver-textdocument' import { CodewhispererServerFactory } from './codeWhispererServer' import { CodeWhispererServiceBase, ResponseContext, Suggestion } from './codeWhispererService' +import { TelemetryService } from './telemetryService' describe('CodeWhisperer Server', () => { const HELLO_WORLD_IN_CSHARP = ` @@ -36,6 +37,7 @@ class HelloWorld // for examples on how to mock just the SDK client let service: StubbedInstance let clock: sinon.SinonFakeTimers + let telemetryServiceSpy: sinon.SinonSpy beforeEach(async () => { clock = sinon.useFakeTimers() @@ -69,6 +71,7 @@ class HelloWorld }) it('should emit Code Percentage telemetry event every 5 minutes', async () => { + telemetryServiceSpy = sinon.spy(TelemetryService.prototype, 'emitCodeCoverageEvent') await features.simulateTyping(SOME_FILE.uri, SOME_TYPING) const updatedDocument = features.documents[SOME_FILE.uri] @@ -107,16 +110,19 @@ class HelloWorld clock.tick(5000 * 60) - sinon.assert.calledWithExactly(features.telemetry.emitMetric, { - name: 'codewhisperer_codePercentage', - data: { - codewhispererTotalTokens: totalInsertCharacters, - codewhispererLanguage: 'csharp', - codewhispererSuggestedTokens: codeWhispererCharacters, - codewhispererPercentage: codePercentage, - successCount: 1, + sinon.assert.calledWithExactly( + telemetryServiceSpy, + { + languageId: 'csharp', + customizationArn: undefined, + totalCharacterCount: totalInsertCharacters, + acceptedCharacterCount: codeWhispererCharacters, }, - }) + { + percentage: codePercentage, + successCount: 1, + } + ) }) }) }) diff --git a/server/aws-lsp-codewhisperer/src/language-server/telemetry/codePercentage.test.ts b/server/aws-lsp-codewhisperer/src/language-server/telemetry/codePercentage.test.ts index 3fc3ffe7..b7cd2697 100644 --- a/server/aws-lsp-codewhisperer/src/language-server/telemetry/codePercentage.test.ts +++ b/server/aws-lsp-codewhisperer/src/language-server/telemetry/codePercentage.test.ts @@ -1,7 +1,5 @@ -import { Telemetry } from '@aws/language-server-runtimes/server-interface' import sinon, { StubbedInstance, stubInterface } from 'ts-sinon' import { CodePercentageTracker } from './codePercentage' -import assert = require('assert') import { TelemetryService } from '../telemetryService' describe('CodePercentage', () => { @@ -13,15 +11,13 @@ describe('CodePercentage', () => { const SOME_ACCEPTED_CONTENT = 'accepted.\n' let tracker: CodePercentageTracker - let telemetry: StubbedInstance let telemetryService: StubbedInstance let clock: sinon.SinonFakeTimers beforeEach(() => { clock = sinon.useFakeTimers() - telemetry = stubInterface() telemetryService = stubInterface() - tracker = new CodePercentageTracker(telemetry, telemetryService) + tracker = new CodePercentageTracker(telemetryService) }) afterEach(() => { @@ -31,7 +27,6 @@ describe('CodePercentage', () => { it('does not send telemetry without edits', () => { clock.tick(5000 * 60) - sinon.assert.notCalled(telemetry.emitMetric) sinon.assert.notCalled(telemetryService.emitCodeCoverageEvent) }) @@ -44,12 +39,19 @@ describe('CodePercentage', () => { clock.tick(5000 * 60) - sinon.assert.calledWith(telemetryService.emitCodeCoverageEvent, { - languageId: LANGUAGE_ID, - totalCharacterCount: 20, - acceptedCharacterCount: 10, - customizationArn: undefined, - }) + sinon.assert.calledWith( + telemetryService.emitCodeCoverageEvent, + { + languageId: LANGUAGE_ID, + totalCharacterCount: 20, + acceptedCharacterCount: 10, + customizationArn: undefined, + }, + { + percentage: 50, + successCount: 1, + } + ) }) it('emits no metrics without invocations', () => { @@ -57,7 +59,6 @@ describe('CodePercentage', () => { clock.tick(5000 * 60) - sinon.assert.notCalled(telemetry.emitMetric) sinon.assert.notCalled(telemetryService.emitCodeCoverageEvent) }) @@ -77,29 +78,33 @@ describe('CodePercentage', () => { clock.tick(5000 * 60) - sinon.assert.calledWith(telemetryService.emitCodeCoverageEvent, { - languageId: LANGUAGE_ID, - totalCharacterCount: 20, - acceptedCharacterCount: 10, - customizationArn: undefined, - }) - - sinon.assert.calledWith(telemetry.emitMetric, { - name: 'codewhisperer_codePercentage', - data: { - codewhispererTotalTokens: 30, - codewhispererLanguage: OTHER_LANGUAGE_ID, - codewhispererSuggestedTokens: 10, - codewhispererPercentage: 33.33, + sinon.assert.calledWith( + telemetryService.emitCodeCoverageEvent, + { + languageId: LANGUAGE_ID, + totalCharacterCount: 20, + acceptedCharacterCount: 10, + customizationArn: undefined, + }, + { + percentage: 50, successCount: 1, + } + ) + + sinon.assert.calledWith( + telemetryService.emitCodeCoverageEvent, + { + languageId: OTHER_LANGUAGE_ID, + totalCharacterCount: 30, + acceptedCharacterCount: 10, + customizationArn: undefined, }, - }) - sinon.assert.calledWith(telemetryService.emitCodeCoverageEvent, { - languageId: OTHER_LANGUAGE_ID, - totalCharacterCount: 30, - acceptedCharacterCount: 10, - customizationArn: undefined, - }) + { + percentage: 33.33, + successCount: 1, + } + ) }) it('emits metrics with customizationArn value', () => { @@ -112,11 +117,18 @@ describe('CodePercentage', () => { clock.tick(5000 * 60) - sinon.assert.calledWith(telemetryService.emitCodeCoverageEvent, { - languageId: LANGUAGE_ID, - totalCharacterCount: 20, - acceptedCharacterCount: 10, - customizationArn: 'test-arn', - }) + sinon.assert.calledWith( + telemetryService.emitCodeCoverageEvent, + { + languageId: LANGUAGE_ID, + totalCharacterCount: 20, + acceptedCharacterCount: 10, + customizationArn: 'test-arn', + }, + { + percentage: 50, + successCount: 1, + } + ) }) }) diff --git a/server/aws-lsp-codewhisperer/src/language-server/telemetry/codePercentage.ts b/server/aws-lsp-codewhisperer/src/language-server/telemetry/codePercentage.ts index 632de852..be2fee0f 100644 --- a/server/aws-lsp-codewhisperer/src/language-server/telemetry/codePercentage.ts +++ b/server/aws-lsp-codewhisperer/src/language-server/telemetry/codePercentage.ts @@ -1,4 +1,3 @@ -import { Telemetry } from '@aws/language-server-runtimes/server-interface' import { CodeWhispererCodePercentageEvent } from './types' import { TelemetryService } from '../telemetryService' import { CodewhispererLanguage } from '../languageDetection' @@ -19,13 +18,11 @@ type TelemetryBuckets = { export class CodePercentageTracker { private buckets: TelemetryBuckets private intervalId: NodeJS.Timeout - private telemetry: Telemetry private telemetryService: TelemetryService public customizationArn?: string - constructor(telemetry: Telemetry, telemetryService: TelemetryService) { + constructor(telemetryService: TelemetryService) { this.buckets = {} - this.telemetry = telemetry this.telemetryService = telemetryService this.intervalId = this.startListening() @@ -34,17 +31,18 @@ export class CodePercentageTracker { private startListening() { return setInterval(() => { this.getEventDataAndRotate().forEach(event => { - this.telemetry.emitMetric({ - name: CODE_PERCENTAGE_EVENT_NAME, - data: event, - }) - - this.telemetryService.emitCodeCoverageEvent({ - languageId: event.codewhispererLanguage as CodewhispererLanguage, - customizationArn: this.customizationArn, - totalCharacterCount: event.codewhispererTotalTokens, - acceptedCharacterCount: event.codewhispererSuggestedTokens, - }) + this.telemetryService.emitCodeCoverageEvent( + { + languageId: event.codewhispererLanguage as CodewhispererLanguage, + customizationArn: this.customizationArn, + totalCharacterCount: event.codewhispererTotalTokens, + acceptedCharacterCount: event.codewhispererSuggestedTokens, + }, + { + percentage: event.codewhispererPercentage, + successCount: event.successCount, + } + ) }) }, CODE_PERCENTAGE_INTERVAL) } diff --git a/server/aws-lsp-codewhisperer/src/language-server/telemetry/userTriggerDecision.test.ts b/server/aws-lsp-codewhisperer/src/language-server/telemetry/userTriggerDecision.test.ts index 6de1139b..a65cb8a3 100644 --- a/server/aws-lsp-codewhisperer/src/language-server/telemetry/userTriggerDecision.test.ts +++ b/server/aws-lsp-codewhisperer/src/language-server/telemetry/userTriggerDecision.test.ts @@ -11,6 +11,7 @@ import { TextDocument } from 'vscode-languageserver-textdocument' import { CodewhispererServerFactory } from '../codeWhispererServer' import { CodeWhispererServiceBase, ResponseContext, Suggestion } from '../codeWhispererService' import { CodeWhispererSession, SessionManager } from '../session/sessionManager' +import { TelemetryService } from '../telemetryService' describe('Telemetry', () => { const sandbox = sinon.createSandbox() @@ -19,6 +20,7 @@ describe('Telemetry', () => { let sessionManagerSpy: sinon.SinonSpiedInstance let generateSessionIdStub: sinon.SinonStub let clock: sinon.SinonFakeTimers + let telemetryServiceSpy: sinon.SinonSpy beforeEach(() => { const StubSessionIdGenerator = () => { @@ -38,12 +40,14 @@ describe('Telemetry', () => { clock = sinon.useFakeTimers({ now: 1483228800000, }) + telemetryServiceSpy = sinon.spy(TelemetryService.prototype, 'emitUserTriggerDecision') }) afterEach(() => { generateSessionIdStub.restore() clock.restore() sandbox.restore() + telemetryServiceSpy.restore() }) describe('User Trigger Decision telemetry', () => { @@ -193,28 +197,71 @@ describe('Telemetry', () => { const aUserTriggerDecision = (override: object = {}) => { return { - name: 'codewhisperer_userTriggerDecision', - data: { + id: 'some-random-session-uuid-0', + document: { + _uri: 'file:///test.cs', + _languageId: 'csharp', + _version: 1, + _content: + 'class HelloWorld\n' + + '{\n' + + ' static void Main()\n' + + ' {\n' + + ' Console.WriteLine("Hello World!");\n' + + ' }\n' + + '}\n', + _lineOffsets: [0, 17, 19, 42, 48, 91, 97, 99], + }, + startTime: 1483228800000, + closeTime: 1483228802000, + state: 'CLOSED', + codewhispererSessionId: 'cwspr-session-id', + startPosition: { line: 2, character: 21 }, + suggestions: [ + { itemId: 'cwspr-item-id-1', content: '' }, + { itemId: 'cwspr-item-id-2', content: '' }, + { itemId: 'cwspr-item-id-3', content: '' }, + ], + suggestionsStates: new Map([ + ['cwspr-item-id-1', 'Empty'], + ['cwspr-item-id-2', 'Empty'], + ['cwspr-item-id-3', 'Empty'], + ]), + acceptedSuggestionId: undefined, + responseContext: { + requestId: 'cwspr-request-id', codewhispererSessionId: 'cwspr-session-id', - codewhispererFirstRequestId: 'cwspr-request-id', - credentialStartUrl: 'teststarturl', - codewhispererSuggestionState: 'Reject', - codewhispererCompletionType: 'Line', - codewhispererLanguage: 'csharp', - codewhispererTriggerType: 'AutoTrigger', - codewhispererAutomatedTriggerType: 'SpecialCharacters', - codewhispererTriggerCharacter: '(', - codewhispererLineNumber: 2, - codewhispererCursorOffset: 21, - codewhispererSuggestionCount: 3, - codewhispererTotalShownTime: 0, - codewhispererTypeaheadLength: 0, - codewhispererTimeSinceLastDocumentChange: 0, - codewhispererSupplementalContextTimeout: undefined, - codewhispererSupplementalContextIsUtg: undefined, - codewhispererSupplementalContextLength: undefined, - ...override, }, + triggerType: 'AutoTrigger', + autoTriggerType: 'SpecialCharacters', + triggerCharacter: '(', + classifierResult: 0.46733811481459187, + classifierThreshold: 0.43, + language: 'csharp', + requestContext: { + fileContext: { + filename: 'file:///test.cs', + programmingLanguage: { + languageName: 'csharp', + }, + leftFileContent: 'class HelloWorld\n{\n static void Main(', + rightFileContent: ')\n {\n Console.WriteLine("Hello World!");\n }\n}\n', + }, + maxResults: 1, + supplementalContexts: [], + }, + supplementalMetadata: undefined, + timeToFirstRecommendation: 2000, + credentialStartUrl: 'teststarturl', + completionSessionResult: undefined, + firstCompletionDisplayLatency: undefined, + totalSessionDisplayTime: undefined, + typeaheadLength: undefined, + previousTriggerDecision: undefined, + previousTriggerDecisionTime: undefined, + reportedUserDecision: true, + customizationArn: undefined, + ...override, } } @@ -234,10 +281,8 @@ describe('Telemetry', () => { assert.equal(currentSession.state, 'CLOSED') sinon.assert.calledOnceWithExactly(sessionManagerSpy.closeSession, currentSession) - const expectedUserTriggerDecisionMetric = aUserTriggerDecision({ - codewhispererSuggestionState: 'Empty', - }) - sinon.assert.calledWithMatch(features.telemetry.emitMetric, expectedUserTriggerDecisionMetric) + const expectedUserTriggerDecisionMetric = aUserTriggerDecision() + sinon.assert.calledWithMatch(telemetryServiceSpy, expectedUserTriggerDecisionMetric, 0) }) it('should send Empty User Decision when Codewhisperer returned empty list of suggestions', async () => { @@ -252,11 +297,10 @@ describe('Telemetry', () => { sinon.assert.calledOnceWithExactly(sessionManagerSpy.closeSession, currentSession) const expectedUserTriggerDecisionMetric = aUserTriggerDecision({ - codewhispererSuggestionState: 'Empty', - codewhispererCompletionType: undefined, - codewhispererSuggestionCount: 0, + suggestions: [], + suggestionsStates: new Map([]), }) - sinon.assert.calledWithMatch(features.telemetry.emitMetric, expectedUserTriggerDecisionMetric) + sinon.assert.calledWithMatch(telemetryServiceSpy, expectedUserTriggerDecisionMetric, 0) }) it('should send Discard User Decision when all suggestions are filtered out by includeSuggestionsWithCodeReferences setting filter', async () => { @@ -290,9 +334,30 @@ describe('Telemetry', () => { sinon.assert.calledOnceWithExactly(sessionManagerSpy.closeSession, currentSession) const expectedUserTriggerDecisionMetric = aUserTriggerDecision({ - codewhispererSuggestionState: 'Discard', + suggestions: [ + { + itemId: 'cwspr-item-id-1', + content: 'recommendation with reference', + references: [EXPECTED_REFERENCE], + }, + { + itemId: 'cwspr-item-id-2', + content: 'recommendation with reference', + references: [EXPECTED_REFERENCE], + }, + { + itemId: 'cwspr-item-id-3', + content: 'recommendation with reference', + references: [EXPECTED_REFERENCE], + }, + ], + suggestionsStates: new Map([ + ['cwspr-item-id-1', 'Filter'], + ['cwspr-item-id-2', 'Filter'], + ['cwspr-item-id-3', 'Filter'], + ]), }) - sinon.assert.calledWithMatch(features.telemetry.emitMetric, expectedUserTriggerDecisionMetric) + sinon.assert.calledWithMatch(telemetryServiceSpy, expectedUserTriggerDecisionMetric, 0) }) it('should send Discard User Decision when all suggestions are discarded after right context merge', async () => { @@ -311,15 +376,72 @@ describe('Telemetry', () => { sinon.assert.calledOnceWithExactly(sessionManagerSpy.closeSession, currentSession) const expectedUserTriggerDecisionMetric = aUserTriggerDecision({ - codewhispererSuggestionState: 'Discard', - codewhispererCompletionType: 'Block', - codewhispererTriggerType: 'OnDemand', - codewhispererAutomatedTriggerType: undefined, - codewhispererTriggerCharacter: undefined, - codewhispererLineNumber: 0, - codewhispererCursorOffset: 0, + startPosition: { line: 0, character: 0 }, + suggestions: [ + { + itemId: 'cwspr-item-id-1', + content: + 'class HelloWorld\n' + + '{\n' + + ' static void Main()\n' + + ' {\n' + + ' Console.WriteLine("Hello World!");\n' + + ' }\n' + + '}\n', + }, + { + itemId: 'cwspr-item-id-2', + content: + 'class HelloWorld\n' + + '{\n' + + ' static void Main()\n' + + ' {\n' + + ' Console.WriteLine("Hello World!");\n' + + ' }\n' + + '}\n', + }, + { + itemId: 'cwspr-item-id-3', + content: + 'class HelloWorld\n' + + '{\n' + + ' static void Main()\n' + + ' {\n' + + ' Console.WriteLine("Hello World!");\n' + + ' }\n' + + '}\n', + }, + ], + suggestionsStates: new Map([ + ['cwspr-item-id-1', 'Discard'], + ['cwspr-item-id-2', 'Discard'], + ['cwspr-item-id-3', 'Discard'], + ]), + triggerType: 'OnDemand', + autoTriggerType: undefined, + triggerCharacter: '', + classifierResult: -0.8524073111924992, + requestContext: { + fileContext: { + filename: 'file:///test.cs', + programmingLanguage: { + languageName: 'csharp', + }, + leftFileContent: '', + rightFileContent: + 'class HelloWorld\n' + + '{\n' + + ' static void Main()\n' + + ' {\n' + + ' Console.WriteLine("Hello World!");\n' + + ' }\n' + + '}\n', + }, + maxResults: 5, + supplementalContexts: [], + }, }) - sinon.assert.calledWithMatch(features.telemetry.emitMetric, expectedUserTriggerDecisionMetric) + sinon.assert.calledWithMatch(telemetryServiceSpy, expectedUserTriggerDecisionMetric, 0) }) }) @@ -333,15 +455,10 @@ describe('Telemetry', () => { assert(currentSession) assert.equal(currentSession?.state, 'ACTIVE') sinon.assert.notCalled(sessionManagerSpy.closeSession) - sinon.assert.neverCalledWithMatch(features.telemetry.emitMetric, { - name: 'codewhisperer_userTriggerDecision', - }) + sinon.assert.notCalled(telemetryServiceSpy) await features.doLogInlineCompletionSessionResults(DEFAULT_SESSION_RESULT_DATA) - - sinon.assert.calledWithMatch(features.telemetry.emitMetric, { - name: 'codewhisperer_userTriggerDecision', - }) + sinon.assert.called(telemetryServiceSpy) }) it('should emit User Decision event with correct typeaheadLength value when session results are received', async () => { @@ -353,21 +470,13 @@ describe('Telemetry', () => { assert(currentSession) assert.equal(currentSession?.state, 'ACTIVE') sinon.assert.notCalled(sessionManagerSpy.closeSession) - sinon.assert.neverCalledWithMatch(features.telemetry.emitMetric, { - name: 'codewhisperer_userTriggerDecision', - }) + sinon.assert.notCalled(telemetryServiceSpy) await features.doLogInlineCompletionSessionResults({ ...DEFAULT_SESSION_RESULT_DATA, typeaheadLength: 20, }) - - sinon.assert.calledWithMatch(features.telemetry.emitMetric, { - name: 'codewhisperer_userTriggerDecision', - data: { - codewhispererTypeaheadLength: 20, - }, - }) + sinon.assert.called(telemetryServiceSpy) }) it('should not emit User Decision event when session results are received after session was closed', async () => { @@ -382,9 +491,7 @@ describe('Telemetry', () => { assert(firstSession) assert.equal(firstSession.state, 'ACTIVE') sinon.assert.notCalled(sessionManagerSpy.closeSession) - sinon.assert.neverCalledWithMatch(features.telemetry.emitMetric, { - name: 'codewhisperer_userTriggerDecision', - }) + sinon.assert.notCalled(telemetryServiceSpy) // Send second completion request to close first one setServiceResponse(DEFAULT_SUGGESTIONS, { @@ -397,25 +504,34 @@ describe('Telemetry', () => { assert.notEqual(firstSession, sessionManager.getCurrentSession()) sinon.assert.calledWithExactly(sessionManagerSpy.closeSession, firstSession) // Test that session reports it's status when second request is received - sinon.assert.calledWithMatch(features.telemetry.emitMetric, { - name: 'codewhisperer_userTriggerDecision', - data: { + const expectedEvent = aUserTriggerDecision({ + state: 'DISCARD', + codewhispererSessionId: 'cwspr-session-id-1', + suggestions: [ + { itemId: 'cwspr-item-id-1', content: 'recommendation' }, + { itemId: 'cwspr-item-id-2', content: 'recommendation' }, + { itemId: 'cwspr-item-id-3', content: 'recommendation' }, + ], + suggestionsStates: new Map([ + ['cwspr-item-id-1', 'Discard'], + ['cwspr-item-id-2', 'Discard'], + ['cwspr-item-id-3', 'Discard'], + ]), + responseContext: { + requestId: 'cwspr-request-id', codewhispererSessionId: 'cwspr-session-id-1', - codewhispererSuggestionState: 'Discard', }, }) + sinon.assert.calledWithMatch(telemetryServiceSpy, expectedEvent, 0) - features.telemetry.emitMetric.resetHistory() + telemetryServiceSpy.resetHistory() // Send session results for closed first session await features.doLogInlineCompletionSessionResults({ ...DEFAULT_SESSION_RESULT_DATA, sessionId: firstSession.id, }) - - sinon.assert.neverCalledWithMatch(features.telemetry.emitMetric, { - name: 'codewhisperer_userTriggerDecision', - }) + sinon.assert.notCalled(telemetryServiceSpy) }) it('should not emit User Decision event when session results received for session that does not exist', async () => { @@ -424,13 +540,7 @@ describe('Telemetry', () => { ...DEFAULT_SESSION_RESULT_DATA, sessionId: 'cwspr-session-id-never-created', }) - - sinon.assert.neverCalledWithMatch(features.telemetry.emitMetric, { - name: 'codewhisperer_userTriggerDecision', - data: { - codewhispererSessionId: 'cwspr-session-id-never-created', - }, - }) + sinon.assert.notCalled(telemetryServiceSpy) }) it('should emit Accept User Decision event for current active completion session when session results are received with accepted suggestion', async () => { @@ -457,17 +567,30 @@ describe('Telemetry', () => { } await autoTriggerInlineCompletionWithReferences() - sinon.assert.neverCalledWithMatch(features.telemetry.emitMetric, { - name: 'codewhisperer_userTriggerDecision', - }) + sinon.assert.notCalled(telemetryServiceSpy) // Send session results for closed first session await features.doLogInlineCompletionSessionResults(SESSION_RESULT_DATA) const expectedUserTriggerDecisionMetric = aUserTriggerDecision({ - codewhispererSuggestionState: 'Accept', + completionSessionResult: { + 'cwspr-item-id-1': { seen: true, accepted: false, discarded: false }, + 'cwspr-item-id-2': { seen: true, accepted: true, discarded: false }, + 'cwspr-item-id-3': { seen: true, accepted: false, discarded: false }, + }, + suggestions: [ + { itemId: 'cwspr-item-id-1', content: 'recommendation' }, + { itemId: 'cwspr-item-id-2', content: 'recommendation' }, + { itemId: 'cwspr-item-id-3', content: 'recommendation' }, + ], + suggestionsStates: new Map([ + ['cwspr-item-id-1', 'Ignore'], + ['cwspr-item-id-2', 'Accept'], + ['cwspr-item-id-3', 'Ignore'], + ]), + acceptedSuggestionId: 'cwspr-item-id-2', }) - sinon.assert.calledWithMatch(features.telemetry.emitMetric, expectedUserTriggerDecisionMetric) + sinon.assert.calledWithMatch(telemetryServiceSpy, expectedUserTriggerDecisionMetric, 0) }) it('should emit Reject User Decision event for current active completion session when session results are received without accepted suggestion', async () => { @@ -494,17 +617,29 @@ describe('Telemetry', () => { } await autoTriggerInlineCompletionWithReferences() - sinon.assert.neverCalledWithMatch(features.telemetry.emitMetric, { - name: 'codewhisperer_userTriggerDecision', - }) + sinon.assert.notCalled(telemetryServiceSpy) // Send session results for closed first session await features.doLogInlineCompletionSessionResults(SESSION_RESULT_DATA) const expectedUserTriggerDecisionMetric = aUserTriggerDecision({ - codewhispererSuggestionState: 'Reject', + suggestions: [ + { itemId: 'cwspr-item-id-1', content: 'recommendation' }, + { itemId: 'cwspr-item-id-2', content: 'recommendation' }, + { itemId: 'cwspr-item-id-3', content: 'recommendation' }, + ], + suggestionsStates: new Map([ + ['cwspr-item-id-1', 'Reject'], + ['cwspr-item-id-2', 'Discard'], + ['cwspr-item-id-3', 'Discard'], + ]), + completionSessionResult: { + 'cwspr-item-id-1': { seen: true, accepted: false, discarded: false }, + 'cwspr-item-id-2': { seen: false, accepted: false, discarded: true }, + 'cwspr-item-id-3': { seen: false, accepted: false, discarded: true }, + }, }) - sinon.assert.calledWithMatch(features.telemetry.emitMetric, expectedUserTriggerDecisionMetric) + sinon.assert.calledWithMatch(telemetryServiceSpy, expectedUserTriggerDecisionMetric, 0) }) it('should send Discard User Decision when all suggestions have Discard state', async () => { @@ -531,17 +666,29 @@ describe('Telemetry', () => { } await autoTriggerInlineCompletionWithReferences() - sinon.assert.neverCalledWithMatch(features.telemetry.emitMetric, { - name: 'codewhisperer_userTriggerDecision', - }) + sinon.assert.notCalled(telemetryServiceSpy) // Send session results for closed first session await features.doLogInlineCompletionSessionResults(SESSION_RESULT_DATA) const expectedUserTriggerDecisionMetric = aUserTriggerDecision({ - codewhispererSuggestionState: 'Discard', + suggestions: [ + { itemId: 'cwspr-item-id-1', content: 'recommendation' }, + { itemId: 'cwspr-item-id-2', content: 'recommendation' }, + { itemId: 'cwspr-item-id-3', content: 'recommendation' }, + ], + suggestionsStates: new Map([ + ['cwspr-item-id-1', 'Discard'], + ['cwspr-item-id-2', 'Discard'], + ['cwspr-item-id-3', 'Discard'], + ]), + completionSessionResult: { + 'cwspr-item-id-1': { seen: false, accepted: false, discarded: true }, + 'cwspr-item-id-2': { seen: false, accepted: false, discarded: true }, + 'cwspr-item-id-3': { seen: false, accepted: false, discarded: true }, + }, }) - sinon.assert.calledWithMatch(features.telemetry.emitMetric, expectedUserTriggerDecisionMetric) + sinon.assert.calledWithMatch(telemetryServiceSpy, expectedUserTriggerDecisionMetric, 0) }) it('should set codewhispererTimeSinceLastDocumentChange as difference between 2 any document changes', async () => { @@ -568,10 +715,24 @@ describe('Telemetry', () => { await features.doLogInlineCompletionSessionResults(DEFAULT_SESSION_RESULT_DATA) const expectedUserTriggerDecisionMetric = aUserTriggerDecision({ - codewhispererSuggestionState: 'Reject', - codewhispererTimeSinceLastDocumentChange: 5678, + completionSessionResult: { + 'cwspr-item-id-1': { seen: true, accepted: false, discarded: false }, + 'cwspr-item-id-2': { seen: true, accepted: false, discarded: false }, + 'cwspr-item-id-3': { seen: true, accepted: false, discarded: false }, + }, + suggestions: [ + { itemId: 'cwspr-item-id-1', content: 'recommendation' }, + { itemId: 'cwspr-item-id-2', content: 'recommendation' }, + { itemId: 'cwspr-item-id-3', content: 'recommendation' }, + ], + suggestionsStates: new Map([ + ['cwspr-item-id-1', 'Reject'], + ['cwspr-item-id-2', 'Reject'], + ['cwspr-item-id-3', 'Reject'], + ]), + closeTime: clock.now, }) - sinon.assert.calledWithMatch(features.telemetry.emitMetric, expectedUserTriggerDecisionMetric) + sinon.assert.calledWithMatch(telemetryServiceSpy, expectedUserTriggerDecisionMetric, 5678) }) }) @@ -590,16 +751,31 @@ describe('Telemetry', () => { await manualTriggerInlineCompletionWithReferences() const expectedUserTriggerDecisionMetric = aUserTriggerDecision({ + state: 'DISCARD', codewhispererSessionId: 'cwspr-session-id-1', - codewhispererSuggestionState: 'Discard', + suggestions: [ + { itemId: 'cwspr-item-id-1', content: 'recommendation' }, + { itemId: 'cwspr-item-id-2', content: 'recommendation' }, + { itemId: 'cwspr-item-id-3', content: 'recommendation' }, + ], + suggestionsStates: new Map([ + ['cwspr-item-id-1', 'Discard'], + ['cwspr-item-id-2', 'Discard'], + ['cwspr-item-id-3', 'Discard'], + ]), + responseContext: { + requestId: 'cwspr-request-id', + codewhispererSessionId: 'cwspr-session-id-1', + }, }) - sinon.assert.calledWithMatch(features.telemetry.emitMetric, expectedUserTriggerDecisionMetric) - sinon.assert.neverCalledWithMatch(features.telemetry.emitMetric, { - name: 'codewhisperer_userTriggerDecision', - data: { + sinon.assert.calledWithMatch(telemetryServiceSpy, expectedUserTriggerDecisionMetric, 0) + sinon.assert.neverCalledWithMatch( + telemetryServiceSpy, + { codewhispererSessionId: 'cwspr-session-id-2', }, - }) + 0 + ) }) it('should close ACTIVE session and emit Discard user trigger decision event on Auto trigger', async () => { @@ -616,16 +792,31 @@ describe('Telemetry', () => { await autoTriggerInlineCompletionWithReferences() const expectedUserTriggerDecisionMetric = aUserTriggerDecision({ + state: 'DISCARD', codewhispererSessionId: 'cwspr-session-id-1', - codewhispererSuggestionState: 'Discard', + suggestions: [ + { itemId: 'cwspr-item-id-1', content: 'recommendation' }, + { itemId: 'cwspr-item-id-2', content: 'recommendation' }, + { itemId: 'cwspr-item-id-3', content: 'recommendation' }, + ], + suggestionsStates: new Map([ + ['cwspr-item-id-1', 'Discard'], + ['cwspr-item-id-2', 'Discard'], + ['cwspr-item-id-3', 'Discard'], + ]), + responseContext: { + requestId: 'cwspr-request-id', + codewhispererSessionId: 'cwspr-session-id-1', + }, }) - sinon.assert.calledWithMatch(features.telemetry.emitMetric, expectedUserTriggerDecisionMetric) - sinon.assert.neverCalledWithMatch(features.telemetry.emitMetric, { - name: 'codewhisperer_userTriggerDecision', - data: { + sinon.assert.calledWithMatch(telemetryServiceSpy, expectedUserTriggerDecisionMetric, 0) + sinon.assert.neverCalledWithMatch( + telemetryServiceSpy, + { codewhispererSessionId: 'cwspr-session-id-2', }, - }) + 0 + ) }) it('should attach previous session trigger decision', async () => { @@ -648,28 +839,84 @@ describe('Telemetry', () => { }) await autoTriggerInlineCompletionWithReferences() - sinon.assert.calledWithMatch( - features.telemetry.emitMetric, - aUserTriggerDecision({ - codewhispererSessionId: 'cwspr-session-id-1', - codewhispererSuggestionState: 'Discard', - }) - ) - sinon.assert.calledWithMatch( - features.telemetry.emitMetric, + sinon.assert.calledTwice(telemetryServiceSpy) + const firstCallArgs = telemetryServiceSpy.getCall(0).args[0] + const secondCallArgs = telemetryServiceSpy.getCall(1).args[0] + sinon.assert.match(firstCallArgs, { + id: 'some-random-session-uuid-0', + document: { + _uri: 'file:///test.cs', + _languageId: 'csharp', + _version: 1, + _content: + 'class HelloWorld\n{\n static void Main()\n {\n Console.WriteLine("Hello World!");\n }\n}\n', + _lineOffsets: [0, 17, 19, 42, 48, 91, 97, 99], + }, + startTime: 1483228800000, + closeTime: 1483228802000, + state: 'DISCARD', + codewhispererSessionId: 'cwspr-session-id-1', + startPosition: { line: 2, character: 21 }, + suggestions: [ + { itemId: 'cwspr-item-id-1', content: 'recommendation' }, + { itemId: 'cwspr-item-id-2', content: 'recommendation' }, + { itemId: 'cwspr-item-id-3', content: 'recommendation' }, + ], + suggestionsStates: {}, + responseContext: { requestId: 'cwspr-request-id', codewhispererSessionId: 'cwspr-session-id-1' }, + triggerType: 'AutoTrigger', + autoTriggerType: 'SpecialCharacters', + triggerCharacter: '(', + classifierResult: 0.46733811481459187, + classifierThreshold: 0.43, + language: 'csharp', + requestContext: { + fileContext: { + filename: 'file:///test.cs', + programmingLanguage: { languageName: 'csharp' }, + leftFileContent: 'class HelloWorld\n{\n static void Main(', + rightFileContent: ')\n {\n Console.WriteLine("Hello World!");\n }\n}\n', + }, + maxResults: 1, + supplementalContexts: [], + }, + timeToFirstRecommendation: 2000, + credentialStartUrl: 'teststarturl', + reportedUserDecision: true, + }) + sinon.assert.match( + secondCallArgs, aUserTriggerDecision({ + previousTriggerDecision: 'Discard', + previousTriggerDecisionTime: 1483228802000, + responseContext: { + requestId: 'cwspr-request-id', + codewhispererSessionId: 'cwspr-session-id-2', + }, + id: 'some-random-session-uuid-1', + startTime: 1483228802000, + closeTime: 1483228804000, + state: 'DISCARD', codewhispererSessionId: 'cwspr-session-id-2', - codewhispererSuggestionState: 'Discard', - codewhispererPreviousSuggestionState: firstSession?.getAggregatedUserTriggerDecision(), // 'Discard' - codewhispererTimeSinceLastUserDecision: 0, + suggestions: [ + { itemId: 'cwspr-item-id-1', content: 'recommendation' }, + { itemId: 'cwspr-item-id-2', content: 'recommendation' }, + { itemId: 'cwspr-item-id-3', content: 'recommendation' }, + ], + suggestionsStates: new Map([ + ['cwspr-item-id-1', 'Discard'], + ['cwspr-item-id-2', 'Discard'], + ['cwspr-item-id-3', 'Discard'], + ]), }) ) - sinon.assert.neverCalledWithMatch(features.telemetry.emitMetric, { - name: 'codewhisperer_userTriggerDecision', - data: { + sinon.assert.neverCalledWithMatch( + telemetryServiceSpy, + aUserTriggerDecision({ codewhispererSessionId: 'cwspr-session-id-3', - }, - }) + }), + 0 + ) }) it('should set correct values for past trigger result fields', async () => { @@ -694,26 +941,101 @@ describe('Telemetry', () => { await autoTriggerInlineCompletionWithReferences() // For first session previous data does not exist - sinon.assert.calledWithMatch( - features.telemetry.emitMetric, + sinon.assert.calledTwice(telemetryServiceSpy) + const firstCallArgs = telemetryServiceSpy.getCall(0).args[0] + const secondCallArgs = telemetryServiceSpy.getCall(1).args[0] + sinon.assert.match( + firstCallArgs, aUserTriggerDecision({ + previousTriggerDecision: undefined, + previousTriggerDecisionTime: undefined, + responseContext: { + requestId: 'cwspr-request-id', + codewhispererSessionId: 'cwspr-session-id-1', + }, + id: 'some-random-session-uuid-0', + startTime: 1483228800000, + closeTime: 1483228802000, + state: 'CLOSED', codewhispererSessionId: 'cwspr-session-id-1', - codewhispererSuggestionState: 'Reject', - codewhispererTimeSinceLastUserDecision: undefined, - codewhispererPreviousSuggestionState: undefined, + suggestions: [ + { itemId: 'cwspr-item-id-1', content: 'recommendation' }, + { itemId: 'cwspr-item-id-2', content: 'recommendation' }, + { itemId: 'cwspr-item-id-3', content: 'recommendation' }, + ], + suggestionsStates: new Map([ + ['cwspr-item-id-1', 'Reject'], + ['cwspr-item-id-2', 'Reject'], + ['cwspr-item-id-3', 'Reject'], + ]), + completionSessionResult: { + 'cwspr-item-id-1': { seen: true, accepted: false, discarded: false }, + 'cwspr-item-id-2': { seen: true, accepted: false, discarded: false }, + 'cwspr-item-id-3': { seen: true, accepted: false, discarded: false }, + }, }) ) - - // For second session previous data matches - sinon.assert.calledWithMatch( - features.telemetry.emitMetric, - aUserTriggerDecision({ + sinon.assert.match(secondCallArgs, { + id: 'some-random-session-uuid-1', + document: { + _uri: 'file:///test.cs', + _languageId: 'csharp', + _version: 1, + _content: + 'class HelloWorld\n{\n static void Main()\n {\n Console.WriteLine("Hello World!");\n }\n}\n', + _lineOffsets: [0, 17, 19, 42, 48, 91, 97, 99], + }, + startTime: 1483228803234, + closeTime: 1483228805234, + state: 'DISCARD', + codewhispererSessionId: 'cwspr-session-id-2', + startPosition: { + line: 2, + character: 21, + }, + suggestions: [ + { + itemId: 'cwspr-item-id-1', + content: 'recommendation', + }, + { + itemId: 'cwspr-item-id-2', + content: 'recommendation', + }, + { + itemId: 'cwspr-item-id-3', + content: 'recommendation', + }, + ], + suggestionsStates: {}, + responseContext: { + requestId: 'cwspr-request-id', codewhispererSessionId: 'cwspr-session-id-2', - codewhispererSuggestionState: 'Discard', - codewhispererPreviousSuggestionState: firstSession?.getAggregatedUserTriggerDecision(), // 'Reject' - codewhispererTimeSinceLastUserDecision: 1234, - }) - ) + }, + triggerType: 'AutoTrigger', + autoTriggerType: 'SpecialCharacters', + triggerCharacter: '(', + classifierResult: 0.30173811481459184, + classifierThreshold: 0.43, + language: 'csharp', + requestContext: { + fileContext: { + filename: 'file:///test.cs', + programmingLanguage: { + languageName: 'csharp', + }, + leftFileContent: 'class HelloWorld\n{\n static void Main(', + rightFileContent: ')\n {\n Console.WriteLine("Hello World!");\n }\n}\n', + }, + maxResults: 1, + supplementalContexts: [], + }, + timeToFirstRecommendation: 2000, + credentialStartUrl: 'teststarturl', + previousTriggerDecision: 'Reject', + previousTriggerDecisionTime: 1483228802000, + reportedUserDecision: true, + }) }) }) @@ -757,32 +1079,80 @@ describe('Telemetry', () => { // 3 sessions were created, each one closes previous one in REQUESTING state assert.equal(SESSION_IDS_LOG.length, 3) - - sinon.assert.calledWithMatch( - features.telemetry.emitMetric, + sinon.assert.calledTwice(telemetryServiceSpy) + const firstCallArgs = telemetryServiceSpy.getCall(0).args[0] + const secondCallArgs = telemetryServiceSpy.getCall(1).args[0] + sinon.assert.match( + firstCallArgs, aUserTriggerDecision({ codewhispererSessionId: 'cwspr-session-id-0', - codewhispererSuggestionState: 'Discard', - codewhispererTimeToFirstRecommendation: 1260, - }) - ) - sinon.assert.calledWithMatch( - features.telemetry.emitMetric, - aUserTriggerDecision({ - codewhispererSessionId: 'cwspr-session-id-1', - codewhispererSuggestionState: 'Discard', - codewhispererTimeToFirstRecommendation: 1260, - codewhispererPreviousSuggestionState: 'Discard', - }) - ) - sinon.assert.neverCalledWithMatch( - features.telemetry.emitMetric, - aUserTriggerDecision({ - codewhispererSessionId: 'cwspr-session-id-2', - codewhispererSuggestionState: 'Empty', + state: 'DISCARD', + suggestions: [ + { itemId: 'cwspr-item-id-1', content: 'recommendation' }, + { itemId: 'cwspr-item-id-2', content: 'recommendation' }, + { itemId: 'cwspr-item-id-3', content: 'recommendation' }, + ], + suggestionsStates: new Map([ + ['cwspr-item-id-1', 'Discard'], + ['cwspr-item-id-2', 'Discard'], + ['cwspr-item-id-3', 'Discard'], + ]), + responseContext: { + requestId: 'cwspr-request-id', + codewhispererSessionId: 'cwspr-session-id-0', + }, + timeToFirstRecommendation: 1260, + closeTime: 1483228801000, }) ) - + sinon.assert.match(secondCallArgs, { + id: 'some-random-session-uuid-1', + document: { + _uri: 'file:///test.cs', + _languageId: 'csharp', + _version: 1, + _content: + 'class HelloWorld\n{\n static void Main()\n {\n Console.WriteLine("Hello World!");\n }\n}\n', + _lineOffsets: [0, 17, 19, 42, 48, 91, 97, 99], + }, + startTime: 1483228801260, + closeTime: 1483228802260, + state: 'DISCARD', + codewhispererSessionId: 'cwspr-session-id-1', + startPosition: { line: 2, character: 21 }, + suggestions: [ + { itemId: 'cwspr-item-id-1', content: 'recommendation' }, + { itemId: 'cwspr-item-id-2', content: 'recommendation' }, + { itemId: 'cwspr-item-id-3', content: 'recommendation' }, + ], + suggestionsStates: {}, + responseContext: { requestId: 'cwspr-request-id', codewhispererSessionId: 'cwspr-session-id-1' }, + triggerType: 'AutoTrigger', + autoTriggerType: 'SpecialCharacters', + triggerCharacter: '(', + classifierResult: 0.46733811481459187, + classifierThreshold: 0.43, + language: 'csharp', + requestContext: { + fileContext: { + filename: 'file:///test.cs', + programmingLanguage: { languageName: 'csharp' }, + leftFileContent: 'class HelloWorld\n{\n static void Main(', + rightFileContent: ')\n {\n Console.WriteLine("Hello World!");\n }\n}\n', + }, + maxResults: 1, + supplementalContexts: [], + }, + timeToFirstRecommendation: 1260, + credentialStartUrl: 'teststarturl', + previousTriggerDecision: 'Discard', + previousTriggerDecisionTime: 1483228801000, + reportedUserDecision: true, + }) + sinon.assert.neverCalledWithMatch(telemetryServiceSpy, { + codewhispererSessionId: 'cwspr-session-id-2', + }) + telemetryServiceSpy.resetHistory() const activeSession = sessionManager.getActiveSession() assert.equal(activeSession?.id, SESSION_IDS_LOG[2]) @@ -791,14 +1161,36 @@ describe('Telemetry', () => { sessionId: SESSION_IDS_LOG[2], }) sinon.assert.calledWithMatch( - features.telemetry.emitMetric, + telemetryServiceSpy, aUserTriggerDecision({ + id: 'some-random-session-uuid-2', + startTime: 1483228802520, + closeTime: 1483228803770, codewhispererSessionId: 'cwspr-session-id-2', - codewhispererSuggestionState: 'Reject', - codewhispererTimeSinceLastUserDecision: 260, - codewhispererPreviousSuggestionState: 'Discard', - codewhispererTimeToFirstRecommendation: 1250, - }) + suggestions: [ + { itemId: 'cwspr-item-id-1', content: 'recommendation' }, + { itemId: 'cwspr-item-id-2', content: 'recommendation' }, + { itemId: 'cwspr-item-id-3', content: 'recommendation' }, + ], + suggestionsStates: new Map([ + ['cwspr-item-id-1', 'Reject'], + ['cwspr-item-id-2', 'Reject'], + ['cwspr-item-id-3', 'Reject'], + ]), + previousTriggerDecision: 'Discard', + previousTriggerDecisionTime: 1483228802260, + timeToFirstRecommendation: 1250, + completionSessionResult: { + 'cwspr-item-id-1': { seen: true, accepted: false, discarded: false }, + 'cwspr-item-id-2': { seen: true, accepted: false, discarded: false }, + 'cwspr-item-id-3': { seen: true, accepted: false, discarded: false }, + }, + responseContext: { + requestId: 'cwspr-request-id', + codewhispererSessionId: 'cwspr-session-id-2', + }, + }), + 0 ) }) }) @@ -811,22 +1203,39 @@ describe('Telemetry', () => { await autoTriggerInlineCompletionWithReferences() const firstSession = sessionManager.getCurrentSession() - sinon.assert.neverCalledWithMatch(features.telemetry.emitMetric, { - name: 'codewhisperer_userTriggerDecision', - }) + sinon.assert.neverCalledWithMatch(telemetryServiceSpy) // Record session results and close the session await features.doLogInlineCompletionSessionResults(DEFAULT_SESSION_RESULT_DATA) - - sinon.assert.calledWithMatch(features.telemetry.emitMetric, { - name: 'codewhisperer_userTriggerDecision', - data: { + sinon.assert.calledWithMatch( + telemetryServiceSpy, + aUserTriggerDecision({ codewhispererSessionId: 'cwspr-session-id-1', - }, - }) + suggestions: [ + { itemId: 'cwspr-item-id-1', content: 'recommendation' }, + { itemId: 'cwspr-item-id-2', content: 'recommendation' }, + { itemId: 'cwspr-item-id-3', content: 'recommendation' }, + ], + suggestionsStates: new Map([ + ['cwspr-item-id-1', 'Reject'], + ['cwspr-item-id-2', 'Reject'], + ['cwspr-item-id-3', 'Reject'], + ]), + responseContext: { + requestId: 'cwspr-request-id', + codewhispererSessionId: 'cwspr-session-id-1', + }, + completionSessionResult: { + 'cwspr-item-id-1': { seen: true, accepted: false, discarded: false }, + 'cwspr-item-id-2': { seen: true, accepted: false, discarded: false }, + 'cwspr-item-id-3': { seen: true, accepted: false, discarded: false }, + }, + }), + 0 + ) assert.equal(firstSession?.state, 'CLOSED') - features.telemetry.emitMetric.resetHistory() + telemetryServiceSpy.resetHistory() // Triggering new completion request creates new session // and should not emit telemetry for previous session, which was closed earlier @@ -838,13 +1247,13 @@ describe('Telemetry', () => { // Or attempt to record data await features.doLogInlineCompletionSessionResults(DEFAULT_SESSION_RESULT_DATA) - - sinon.assert.neverCalledWithMatch(features.telemetry.emitMetric, { - name: 'codewhisperer_userTriggerDecision', - data: { + sinon.assert.neverCalledWithMatch( + telemetryServiceSpy, + { codewhispererSessionId: 'cwspr-session-id-1', }, - }) + 0 + ) }) }) diff --git a/server/aws-lsp-codewhisperer/src/language-server/telemetryService.test.ts b/server/aws-lsp-codewhisperer/src/language-server/telemetryService.test.ts index 71653b24..262b76b9 100644 --- a/server/aws-lsp-codewhisperer/src/language-server/telemetryService.test.ts +++ b/server/aws-lsp-codewhisperer/src/language-server/telemetryService.test.ts @@ -8,18 +8,11 @@ import { Logging, Telemetry, } from '@aws/language-server-runtimes/server-interface' -import { - UserContext, - OptOutPreference, - UserIntent, - Boolean, - timeBetweenChunks, -} from '../client/token/codewhispererbearertokenclient' +import { UserContext, OptOutPreference } from '../client/token/codewhispererbearertokenclient' import { CodeWhispererSession } from './session/sessionManager' import sinon from 'ts-sinon' import { BUILDER_ID_START_URL } from './constants' import { ChatInteractionType } from './telemetry/types' -import { CodewhispererLanguage } from './languageDetection' class MockCredentialsProvider implements CredentialsProvider { private mockIamCredentials: IamCredentials | undefined @@ -54,6 +47,7 @@ class MockCredentialsProvider implements CredentialsProvider { } describe('TelemetryService', () => { + let telemetry: Telemetry let clock: sinon.SinonFakeTimers let telemetryService: TelemetryService let mockCredentialsProvider: MockCredentialsProvider @@ -88,6 +82,10 @@ describe('TelemetryService', () => { firstCompletionDisplayLatency: 100, timeToFirstRecommendation: 200, getAggregatedUserTriggerDecision: () => 'Accept', + startPosition: { + line: 12, + character: 23, + }, } beforeEach(() => { @@ -95,6 +93,10 @@ describe('TelemetryService', () => { now: 1483228800000, }) mockCredentialsProvider = new MockCredentialsProvider() + telemetry = { + emitMetric: sinon.stub(), + onClientTelemetry: sinon.stub(), + } }) afterEach(() => { @@ -173,7 +175,7 @@ describe('TelemetryService', () => { }) it('should not emit user trigger decision if login is invalid (IAM)', () => { - telemetryService = new TelemetryService(mockCredentialsProvider, 'iam', {} as Telemetry, logging, {}) + telemetryService = new TelemetryService(mockCredentialsProvider, 'iam', telemetry, logging, {}) const invokeSendTelemetryEventStub: sinon.SinonStub = sinon.stub(telemetryService, 'sendTelemetryEvent' as any) telemetryService.emitUserTriggerDecision(mockSession as CodeWhispererSession) @@ -187,7 +189,7 @@ describe('TelemetryService', () => { startUrl: BUILDER_ID_START_URL, }, }) - telemetryService = new TelemetryService(mockCredentialsProvider, 'bearer', {} as Telemetry, logging, {}) + telemetryService = new TelemetryService(mockCredentialsProvider, 'bearer', telemetry, logging, {}) const invokeSendTelemetryEventStub: sinon.SinonStub = sinon.stub(telemetryService, 'sendTelemetryEvent' as any) telemetryService.updateOptOutPreference('OPTOUT') @@ -197,7 +199,7 @@ describe('TelemetryService', () => { }) it('should handle SSO connection type change at runtime', () => { - telemetryService = new TelemetryService(mockCredentialsProvider, 'bearer', {} as Telemetry, logging, {}) + telemetryService = new TelemetryService(mockCredentialsProvider, 'bearer', telemetry, logging, {}) const sendTelemetryEventStub: sinon.SinonStub = sinon .stub(telemetryService, 'sendTelemetryEvent' as any) .returns(Promise.resolve()) @@ -226,7 +228,7 @@ describe('TelemetryService', () => { sinon.assert.notCalled(sendTelemetryEventStub) }) - it('should emit userTriggerDecision event correctly', () => { + it('should emit userTriggerDecision event to STE and to the destination', () => { const expectedUserTriggerDecisionEvent = { telemetryEvent: { userTriggerDecisionEvent: { @@ -253,7 +255,8 @@ describe('TelemetryService', () => { startUrl: 'idc-start-url', }, }) - telemetryService = new TelemetryService(mockCredentialsProvider, 'bearer', {} as Telemetry, logging, {}) + telemetryService = new TelemetryService(mockCredentialsProvider, 'bearer', telemetry, logging, {}) + telemetryService.updateEnableTelemetryEventsToDestination(true) const invokeSendTelemetryEventStub: sinon.SinonStub = sinon .stub(telemetryService, 'sendTelemetryEvent' as any) .returns(Promise.resolve()) @@ -262,6 +265,50 @@ describe('TelemetryService', () => { telemetryService.emitUserTriggerDecision(mockSession as CodeWhispererSession) sinon.assert.calledOnceWithExactly(invokeSendTelemetryEventStub, expectedUserTriggerDecisionEvent) + sinon.assert.calledOnceWithExactly(telemetry.emitMetric as sinon.SinonStub, { + name: 'codewhisperer_userTriggerDecision', + data: { + codewhispererSessionId: 'test-session-id', + codewhispererFirstRequestId: 'test-request-id', + credentialStartUrl: undefined, + codewhispererSuggestionState: 'Accept', + codewhispererCompletionType: 'Block', + codewhispererLanguage: 'tsx', + codewhispererTriggerType: undefined, + codewhispererAutomatedTriggerType: undefined, + codewhispererTriggerCharacter: undefined, + codewhispererLineNumber: 12, + codewhispererCursorOffset: 23, + codewhispererSuggestionCount: 1, + codewhispererClassifierResult: undefined, + codewhispererClassifierThreshold: undefined, + codewhispererTotalShownTime: 0, + codewhispererTypeaheadLength: 0, + codewhispererTimeSinceLastDocumentChange: undefined, + codewhispererTimeSinceLastUserDecision: undefined, + codewhispererTimeToFirstRecommendation: 200, + codewhispererPreviousSuggestionState: undefined, + codewhispererSupplementalContextTimeout: undefined, + codewhispererSupplementalContextIsUtg: undefined, + codewhispererSupplementalContextLength: undefined, + codewhispererCustomizationArn: 'test-arn', + }, + }) + }) + + it('should not emit userTriggerDecision event to destination when enableTelemetryEventsToDestination is disabled', () => { + mockCredentialsProvider.setConnectionMetadata({ + sso: { + startUrl: BUILDER_ID_START_URL, + }, + }) + telemetryService = new TelemetryService(mockCredentialsProvider, 'bearer', telemetry, logging, {}) + telemetryService.updateEnableTelemetryEventsToDestination(false) + telemetryService.updateOptOutPreference('OPTOUT') + telemetryService.emitUserTriggerDecision(mockSession as CodeWhispererSession) + sinon.assert.neverCalledWithMatch(telemetry.emitMetric as sinon.SinonStub, { + name: 'codewhisperer_userTriggerDecision', + }) }) describe('Chat interact with message', () => { @@ -276,7 +323,7 @@ describe('TelemetryService', () => { startUrl: 'idc-start-url', }, }) - telemetryService = new TelemetryService(mockCredentialsProvider, 'bearer', {} as Telemetry, logging, {}) + telemetryService = new TelemetryService(mockCredentialsProvider, 'bearer', telemetry, logging, {}) invokeSendTelemetryEventStub = sinon .stub(telemetryService, 'sendTelemetryEvent' as any) .returns(Promise.resolve()) @@ -286,7 +333,7 @@ describe('TelemetryService', () => { sinon.restore() }) - it('should send InteractWithMessage event with correct parameters', () => { + it('should send InteractWithMessage event with correct parameters and emit metric to destination', () => { const metric = { cwsprChatMessageId: 'message123', codewhispererCustomizationArn: 'arn:123', @@ -296,6 +343,7 @@ describe('TelemetryService', () => { } const conversationId = 'conv123' const acceptedLineCount = 5 + telemetryService.updateEnableTelemetryEventsToDestination(true) telemetryService.emitChatInteractWithMessage(metric, { conversationId, acceptedLineCount, @@ -318,6 +366,45 @@ describe('TelemetryService', () => { } sinon.assert.calledOnceWithExactly(invokeSendTelemetryEventStub, expectedEvent) + sinon.assert.calledOnceWithExactly(telemetry.emitMetric as sinon.SinonStub, { + name: 'amazonq_interactWithMessage', + data: { + cwsprChatMessageId: 'message123', + codewhispererCustomizationArn: 'arn:123', + cwsprChatInteractionType: 'insertAtCursor', + cwsprChatInteractionTarget: 'CODE', + cwsprChatAcceptedCharactersLength: 100, + cwsprChatConversationId: 'conv123', + credentialStartUrl: 'idc-start-url', + }, + }) + }) + + it('should not send InteractWithMessage event to destination when enableTelemetryEventsToDestination flag is disabled', () => { + const metric = { + cwsprChatMessageId: 'message123', + codewhispererCustomizationArn: 'arn:123', + cwsprChatInteractionType: ChatInteractionType.InsertAtCursor, + cwsprChatInteractionTarget: 'CODE', + cwsprChatAcceptedCharactersLength: 100, + } + const conversationId = 'conv123' + const acceptedLineCount = 5 + mockCredentialsProvider.setConnectionMetadata({ + sso: { + startUrl: BUILDER_ID_START_URL, + }, + }) + telemetryService = new TelemetryService(mockCredentialsProvider, 'bearer', {} as Telemetry, logging, {}) + telemetryService.updateEnableTelemetryEventsToDestination(false) + telemetryService.updateOptOutPreference('OPTOUT') + telemetryService.emitChatInteractWithMessage(metric, { + conversationId, + acceptedLineCount, + }) + sinon.assert.neverCalledWithMatch(telemetry.emitMetric as sinon.SinonStub, { + name: 'amazonq_interactWithMessage', + }) }) it('should not send InteractWithMessage when conversationId is undefined', () => { @@ -336,7 +423,7 @@ describe('TelemetryService', () => { }) it('should not send InteractWithMessage when credentialsType is IAM', () => { - telemetryService = new TelemetryService(mockCredentialsProvider, 'iam', {} as Telemetry, logging, {}) + telemetryService = new TelemetryService(mockCredentialsProvider, 'iam', telemetry, logging, {}) invokeSendTelemetryEventStub = sinon.stub(telemetryService, 'sendTelemetryEvent' as any) const metric = { cwsprChatMessageId: 'message123', @@ -359,7 +446,7 @@ describe('TelemetryService', () => { startUrl: BUILDER_ID_START_URL, }, }) - telemetryService = new TelemetryService(mockCredentialsProvider, 'bearer', {} as Telemetry, logging, {}) + telemetryService = new TelemetryService(mockCredentialsProvider, 'bearer', telemetry, logging, {}) invokeSendTelemetryEventStub = sinon.stub(telemetryService, 'sendTelemetryEvent' as any) telemetryService.updateOptOutPreference('OPTOUT') const metric = { @@ -409,7 +496,7 @@ describe('TelemetryService', () => { }) }) - it('should emit CodeCoverageEvent event', () => { + it('should emit CodeCoverageEvent event to STE and to the destination', () => { const expectedEvent = { telemetryEvent: { codeCoverageEvent: { @@ -427,20 +514,64 @@ describe('TelemetryService', () => { startUrl: 'idc-start-url', }, }) - telemetryService = new TelemetryService(mockCredentialsProvider, 'bearer', {} as Telemetry, logging, {}) + telemetryService = new TelemetryService(mockCredentialsProvider, 'bearer', telemetry, logging, {}) const invokeSendTelemetryEventStub: sinon.SinonStub = sinon .stub(telemetryService, 'sendTelemetryEvent' as any) .returns(Promise.resolve()) telemetryService.updateOptOutPreference('OPTIN') + telemetryService.updateEnableTelemetryEventsToDestination(true) - telemetryService.emitCodeCoverageEvent({ - languageId: 'typescript', - customizationArn: 'test-arn', - acceptedCharacterCount: 123, - totalCharacterCount: 456, - }) + telemetryService.emitCodeCoverageEvent( + { + languageId: 'typescript', + customizationArn: 'test-arn', + acceptedCharacterCount: 123, + totalCharacterCount: 456, + }, + { + percentage: 50, + successCount: 1, + } + ) sinon.assert.calledOnceWithExactly(invokeSendTelemetryEventStub, expectedEvent) + sinon.assert.calledOnceWithExactly(telemetry.emitMetric as sinon.SinonStub, { + name: 'codewhisperer_codePercentage', + data: { + codewhispererTotalTokens: 456, + codewhispererLanguage: 'typescript', + codewhispererSuggestedTokens: 123, + codewhispererPercentage: 50, + successCount: 1, + }, + }) + }) + + it('should not emit CodeCoverageEvent event to destination when enableTelemetryEventsToDestination flag is disabled', () => { + mockCredentialsProvider.setConnectionMetadata({ + sso: { + startUrl: BUILDER_ID_START_URL, + }, + }) + telemetryService = new TelemetryService(mockCredentialsProvider, 'bearer', telemetry, logging, {}) + telemetryService.updateOptOutPreference('OPTOUT') + telemetryService.updateEnableTelemetryEventsToDestination(false) + + telemetryService.emitCodeCoverageEvent( + { + languageId: 'typescript', + customizationArn: 'test-arn', + acceptedCharacterCount: 123, + totalCharacterCount: 456, + }, + { + percentage: 50, + successCount: 1, + } + ) + sinon.assert.neverCalledWithMatch(telemetry.emitMetric as sinon.SinonStub, { + name: 'codewhisperer_codePercentage', + }) }) it('should emit userModificationEvent event', () => { @@ -486,13 +617,14 @@ describe('TelemetryService', () => { sinon.assert.calledOnceWithExactly(invokeSendTelemetryEventStub, expectedEvent) }) - it('should emit chatUserModificationEvent event', () => { + it('should emit chatUserModificationEvent event including emitting event to destination', () => { mockCredentialsProvider.setConnectionMetadata({ sso: { startUrl: 'idc-start-url', }, }) - telemetryService = new TelemetryService(mockCredentialsProvider, 'bearer', {} as Telemetry, logging, {}) + telemetryService = new TelemetryService(mockCredentialsProvider, 'bearer', telemetry, logging, {}) + telemetryService.updateEnableTelemetryEventsToDestination(true) const invokeSendTelemetryEventStub: sinon.SinonStub = sinon .stub(telemetryService, 'sendTelemetryEvent' as any) .returns(Promise.resolve()) @@ -517,6 +649,36 @@ describe('TelemetryService', () => { optOutPreference: 'OPTIN', } sinon.assert.calledOnceWithExactly(invokeSendTelemetryEventStub, expectedEvent) + sinon.assert.calledOnceWithExactly(telemetry.emitMetric as sinon.SinonStub, { + name: 'amazonq_modifyCode', + data: { + cwsprChatConversationId: 'test-conversation-id', + cwsprChatMessageId: 'test-message-id', + cwsprChatModificationPercentage: 0.2, + codewhispererCustomizationArn: 'test-arn', + credentialStartUrl: 'idc-start-url', + }, + }) + }) + + it('should not emit chatUserModificationEvent event to destination when enableTelemetryEventsToDestination flag is disabled', () => { + mockCredentialsProvider.setConnectionMetadata({ + sso: { + startUrl: BUILDER_ID_START_URL, + }, + }) + telemetryService = new TelemetryService(mockCredentialsProvider, 'bearer', telemetry, logging, {}) + telemetryService.updateEnableTelemetryEventsToDestination(false) + telemetryService.updateOptOutPreference('OPTOUT') + telemetryService.emitChatUserModificationEvent({ + conversationId: 'test-conversation-id', + messageId: 'test-message-id', + customizationArn: 'test-arn', + modificationPercentage: 0.2, + }) + sinon.assert.neverCalledWithMatch(telemetry.emitMetric as sinon.SinonStub, { + name: 'amazonq_modifyCode', + }) }) describe('Chat add message', () => { @@ -531,7 +693,7 @@ describe('TelemetryService', () => { startUrl: 'idc-start-url', }, }) - telemetryService = new TelemetryService(mockCredentialsProvider, 'bearer', {} as Telemetry, logging, {}) + telemetryService = new TelemetryService(mockCredentialsProvider, 'bearer', telemetry, logging, {}) invokeSendTelemetryEventStub = sinon .stub(telemetryService, 'sendTelemetryEvent' as any) .returns(Promise.resolve()) @@ -541,21 +703,25 @@ describe('TelemetryService', () => { sinon.restore() }) - it('should send ChatAddMessage event with correct parameters', () => { - telemetryService.emitChatAddMessage({ - conversationId: 'conv123', - messageId: 'message123', - customizationArn: 'cust-123', - programmingLanguage: 'jsx', - userIntent: 'SUGGEST_ALTERNATE_IMPLEMENTATION', - hasCodeSnippet: false, - timeToFirstChunkMilliseconds: 100, - activeEditorTotalCharacters: 250, - fullResponselatency: 400, - requestLength: 100, - responseLength: 3000, - numberOfCodeBlocks: 0, - }) + it('should send ChatAddMessage event with correct parameters and emit metric to destination', () => { + telemetryService.updateEnableTelemetryEventsToDestination(true) + telemetryService.emitChatAddMessage( + { + conversationId: 'conv123', + messageId: 'message123', + customizationArn: 'cust-123', + programmingLanguage: 'jsx', + userIntent: 'SUGGEST_ALTERNATE_IMPLEMENTATION', + hasCodeSnippet: false, + timeToFirstChunkMilliseconds: 100, + activeEditorTotalCharacters: 250, + fullResponselatency: 400, + requestLength: 100, + responseLength: 3000, + numberOfCodeBlocks: 0, + }, + {} + ) const expectedEvent = { telemetryEvent: { @@ -581,32 +747,98 @@ describe('TelemetryService', () => { } sinon.assert.calledOnceWithExactly(invokeSendTelemetryEventStub, expectedEvent) + sinon.assert.calledOnceWithExactly(telemetry.emitMetric as sinon.SinonStub, { + name: 'amazonq_addMessage', + data: { + credentialStartUrl: 'idc-start-url', + cwsprChatConversationId: 'conv123', + cwsprChatHasCodeSnippet: false, + cwsprChatTriggerInteraction: undefined, + cwsprChatMessageId: 'message123', + cwsprChatUserIntent: 'SUGGEST_ALTERNATE_IMPLEMENTATION', + cwsprChatProgrammingLanguage: 'jsx', + cwsprChatResponseCodeSnippetCount: 0, + cwsprChatResponseCode: undefined, + cwsprChatSourceLinkCount: undefined, + cwsprChatReferencesCount: undefined, + cwsprChatFollowUpCount: undefined, + cwsprTimeToFirstChunk: 100, + cwsprChatFullResponseLatency: 400, + cwsprChatTimeBetweenChunks: undefined, + cwsprChatRequestLength: 100, + cwsprChatResponseLength: 3000, + cwsprChatConversationType: undefined, + cwsprChatActiveEditorTotalCharacters: 250, + cwsprChatActiveEditorImportCount: undefined, + codewhispererCustomizationArn: 'cust-123', + }, + }) }) - it('should not send ChatAddMessage when conversationId is undefined', () => { - telemetryService.emitChatAddMessage({ - messageId: 'message123', - customizationArn: 'cust-123', + it('should not send ChatAddMessage event to destination when enableTelemetryEventsToDestination flag is disabled', () => { + mockCredentialsProvider.setConnectionMetadata({ + sso: { + startUrl: BUILDER_ID_START_URL, + }, + }) + telemetryService = new TelemetryService(mockCredentialsProvider, 'bearer', {} as Telemetry, logging, {}) + telemetryService.updateOptOutPreference('OPTOUT') + telemetryService.updateEnableTelemetryEventsToDestination(false) + telemetryService.emitChatAddMessage( + { + conversationId: 'conv123', + messageId: 'message123', + customizationArn: 'cust-123', + programmingLanguage: 'jsx', + userIntent: 'SUGGEST_ALTERNATE_IMPLEMENTATION', + hasCodeSnippet: false, + timeToFirstChunkMilliseconds: 100, + activeEditorTotalCharacters: 250, + fullResponselatency: 400, + requestLength: 100, + responseLength: 3000, + numberOfCodeBlocks: 0, + }, + {} + ) + sinon.assert.neverCalledWithMatch(telemetry.emitMetric as sinon.SinonStub, { + name: 'amazonq_addMessage', }) + }) + + it('should not send ChatAddMessage when conversationId is undefined', () => { + telemetryService.emitChatAddMessage( + { + messageId: 'message123', + customizationArn: 'cust-123', + }, + {} + ) sinon.assert.notCalled(invokeSendTelemetryEventStub) }) it('should not send ChatAddMessage when messageId is undefined', () => { - telemetryService.emitChatAddMessage({ - conversationId: 'conv123', - customizationArn: 'cust-123', - }) + telemetryService.emitChatAddMessage( + { + conversationId: 'conv123', + customizationArn: 'cust-123', + }, + {} + ) sinon.assert.notCalled(invokeSendTelemetryEventStub) }) it('should not send ChatAddMessage when credentialsType is IAM', () => { - telemetryService = new TelemetryService(mockCredentialsProvider, 'iam', {} as Telemetry, logging, {}) + telemetryService = new TelemetryService(mockCredentialsProvider, 'iam', telemetry, logging, {}) invokeSendTelemetryEventStub = sinon.stub(telemetryService, 'sendTelemetryEvent' as any) - telemetryService.emitChatAddMessage({ - conversationId: 'conv123', - messageId: 'message123', - customizationArn: 'cust-123', - }) + telemetryService.emitChatAddMessage( + { + conversationId: 'conv123', + messageId: 'message123', + customizationArn: 'cust-123', + }, + {} + ) sinon.assert.notCalled(invokeSendTelemetryEventStub) }) @@ -616,14 +848,17 @@ describe('TelemetryService', () => { startUrl: BUILDER_ID_START_URL, }, }) - telemetryService = new TelemetryService(mockCredentialsProvider, 'bearer', {} as Telemetry, logging, {}) + telemetryService = new TelemetryService(mockCredentialsProvider, 'bearer', telemetry, logging, {}) invokeSendTelemetryEventStub = sinon.stub(telemetryService, 'sendTelemetryEvent' as any) telemetryService.updateOptOutPreference('OPTOUT') - telemetryService.emitChatAddMessage({ - conversationId: 'conv123', - messageId: 'message123', - customizationArn: 'cust-123', - }) + telemetryService.emitChatAddMessage( + { + conversationId: 'conv123', + messageId: 'message123', + customizationArn: 'cust-123', + }, + {} + ) sinon.assert.notCalled(invokeSendTelemetryEventStub) }) }) diff --git a/server/aws-lsp-codewhisperer/src/language-server/telemetryService.ts b/server/aws-lsp-codewhisperer/src/language-server/telemetryService.ts index e0659cc5..6506226e 100644 --- a/server/aws-lsp-codewhisperer/src/language-server/telemetryService.ts +++ b/server/aws-lsp-codewhisperer/src/language-server/telemetryService.ts @@ -21,13 +21,20 @@ import { UserIntent, } from '../client/token/codewhispererbearertokenclient' import { getCompletionType, getSsoConnectionType, isAwsError } from './utils' -import { ChatInteractionType, InteractWithMessageEvent } from './telemetry/types' +import { + ChatConversationType, + ChatInteractionType, + ChatTelemetryEventName, + CodeWhispererUserTriggerDecisionEvent, + InteractWithMessageEvent, +} from './telemetry/types' import { CodewhispererLanguage, getRuntimeLanguage } from './languageDetection' +import { CONVERSATION_ID_METRIC_KEY } from './chat/telemetry/chatTelemetryController' export class TelemetryService extends CodeWhispererServiceToken { private userContext: UserContext | undefined private optOutPreference!: OptOutPreference - private enableTelemetryEventsToDestination!: boolean + private enableTelemetryEventsToDestination: boolean private telemetry: Telemetry private credentialsType: CredentialsType private credentialsProvider: CredentialsProvider @@ -57,6 +64,7 @@ export class TelemetryService extends CodeWhispererServiceToken { this.credentialsType = credentialsType this.telemetry = telemetry this.logging = logging + this.enableTelemetryEventsToDestination = true } public updateUserContext(userContext: UserContext | undefined): void { @@ -106,11 +114,11 @@ export class TelemetryService extends CodeWhispererServiceToken { } this.logging.log( - `Failed to sendTelemetryEvent to CodeWhisperer, requestId: ${requestId ?? ''}, message: ${error.message}` + `Failed to sendTelemetryEvent to CodeWhisperer, requestId: ${requestId ?? ''}, message: ${error?.message}` ) } - private invokeSendTelemetryEvent(event: TelemetryEvent) { + private async invokeSendTelemetryEvent(event: TelemetryEvent) { if (!this.shouldSendTelemetry()) { return } @@ -123,8 +131,11 @@ export class TelemetryService extends CodeWhispererServiceToken { if (this.optOutPreference !== undefined) { request.optOutPreference = this.optOutPreference } - - this.sendTelemetryEvent(request).then().catch(this.logSendTelemetryEventFailure) + try { + await this.sendTelemetryEvent(request) + } catch (error) { + this.logSendTelemetryEventFailure(error) + } } private getCWClientTelemetryInteractionType(interactionType: ChatInteractionType): ChatMessageInteractionType { @@ -132,7 +143,43 @@ export class TelemetryService extends CodeWhispererServiceToken { } public emitUserTriggerDecision(session: CodeWhispererSession, timeSinceLastUserModification?: number) { - const completionSessionResult = session.completionSessionResult ?? {} + if (this.enableTelemetryEventsToDestination) { + const data: CodeWhispererUserTriggerDecisionEvent = { + codewhispererSessionId: session.codewhispererSessionId || '', + codewhispererFirstRequestId: session.responseContext?.requestId || '', + credentialStartUrl: session.credentialStartUrl, + codewhispererSuggestionState: session.getAggregatedUserTriggerDecision(), + codewhispererCompletionType: + session.suggestions.length > 0 ? getCompletionType(session.suggestions[0]) : undefined, + codewhispererLanguage: session.language, + codewhispererTriggerType: session.triggerType, + codewhispererAutomatedTriggerType: session.autoTriggerType, + codewhispererTriggerCharacter: + session.autoTriggerType === 'SpecialCharacters' ? session.triggerCharacter : undefined, + codewhispererLineNumber: session.startPosition.line, + codewhispererCursorOffset: session.startPosition.character, + codewhispererSuggestionCount: session.suggestions.length, + codewhispererClassifierResult: session.classifierResult, + codewhispererClassifierThreshold: session.classifierThreshold, + codewhispererTotalShownTime: session.totalSessionDisplayTime || 0, + codewhispererTypeaheadLength: session.typeaheadLength || 0, + // Global time between any 2 document changes + codewhispererTimeSinceLastDocumentChange: timeSinceLastUserModification, + codewhispererTimeSinceLastUserDecision: session.previousTriggerDecisionTime + ? session.startTime - session.previousTriggerDecisionTime + : undefined, + codewhispererTimeToFirstRecommendation: session.timeToFirstRecommendation, + codewhispererPreviousSuggestionState: session.previousTriggerDecision, + codewhispererSupplementalContextTimeout: session.supplementalMetadata?.isProcessTimeout, + codewhispererSupplementalContextIsUtg: session.supplementalMetadata?.isUtg, + codewhispererSupplementalContextLength: session.supplementalMetadata?.contentsLength, + codewhispererCustomizationArn: session.customizationArn, + } + this.telemetry.emitMetric({ + name: 'codewhisperer_userTriggerDecision', + data: data, + }) + } const acceptedSuggestion = session.suggestions.find(s => s.itemId === session.acceptedSuggestionId) const generatedLines = acceptedSuggestion === undefined || acceptedSuggestion.content.trim() === '' @@ -185,6 +232,16 @@ export class TelemetryService extends CodeWhispererServiceToken { if (options?.conversationId === undefined) { return } + if (this.enableTelemetryEventsToDestination) { + this.telemetry.emitMetric({ + name: ChatTelemetryEventName.InteractWithMessage, + data: { + ...metric, + [CONVERSATION_ID_METRIC_KEY]: options.conversationId, + credentialStartUrl: this.credentialsProvider.getConnectionMetadata()?.sso?.startUrl, + }, + }) + } const event: ChatInteractWithMessageEvent = { conversationId: options.conversationId, messageId: metric.cwsprChatMessageId, @@ -209,6 +266,18 @@ export class TelemetryService extends CodeWhispererServiceToken { modificationPercentage: number customizationArn?: string }) { + if (this.enableTelemetryEventsToDestination) { + this.telemetry.emitMetric({ + name: ChatTelemetryEventName.ModifyCode, + data: { + [CONVERSATION_ID_METRIC_KEY]: params.conversationId, + cwsprChatMessageId: params.messageId, + cwsprChatModificationPercentage: params.modificationPercentage, + codewhispererCustomizationArn: params.customizationArn, + credentialStartUrl: this.credentialsProvider.getConnectionMetadata()?.sso?.startUrl, + }, + }) + } this.invokeSendTelemetryEvent({ chatUserModificationEvent: params, }) @@ -241,12 +310,30 @@ export class TelemetryService extends CodeWhispererServiceToken { }) } - public emitCodeCoverageEvent(params: { - languageId: CodewhispererLanguage - acceptedCharacterCount: number - totalCharacterCount: number - customizationArn?: string - }) { + public emitCodeCoverageEvent( + params: { + languageId: CodewhispererLanguage + acceptedCharacterCount: number + totalCharacterCount: number + customizationArn?: string + }, + additionalParams: Partial<{ + percentage: number + successCount: number + }> + ) { + if (this.enableTelemetryEventsToDestination) { + this.telemetry.emitMetric({ + name: 'codewhisperer_codePercentage', + data: { + codewhispererTotalTokens: params.totalCharacterCount, + codewhispererLanguage: params.languageId, + codewhispererSuggestedTokens: params.acceptedCharacterCount, + codewhispererPercentage: additionalParams.percentage, + successCount: additionalParams.successCount, + }, + }) + } const event: CodeCoverageEvent = { programmingLanguage: { languageName: getRuntimeLanguage(params.languageId), @@ -262,25 +349,66 @@ export class TelemetryService extends CodeWhispererServiceToken { }) } - public emitChatAddMessage(params: { - conversationId?: string - messageId?: string - customizationArn?: string - userIntent?: UserIntent - hasCodeSnippet?: boolean - programmingLanguage?: CodewhispererLanguage - activeEditorTotalCharacters?: number - timeToFirstChunkMilliseconds?: number - timeBetweenChunks?: number[] - fullResponselatency?: number - requestLength?: number - responseLength?: number - numberOfCodeBlocks?: number - hasProjectLevelContext?: number - }) { + public emitChatAddMessage( + params: { + conversationId?: string + messageId?: string + customizationArn?: string + userIntent?: UserIntent + hasCodeSnippet?: boolean + programmingLanguage?: CodewhispererLanguage + activeEditorTotalCharacters?: number + timeToFirstChunkMilliseconds?: number + timeBetweenChunks?: number[] + fullResponselatency?: number + requestLength?: number + responseLength?: number + numberOfCodeBlocks?: number + hasProjectLevelContext?: number + }, + additionalParams: Partial<{ + chatTriggerInteraction: string + chatResponseCode: number + chatSourceLinkCount?: number + chatReferencesCount?: number + chatFollowUpCount?: number + chatConversationType: ChatConversationType + chatActiveEditorImportCount?: number + }> + ) { if (!params.conversationId || !params.messageId) { return } + + if (this.enableTelemetryEventsToDestination) { + this.telemetry.emitMetric({ + name: ChatTelemetryEventName.AddMessage, + data: { + credentialStartUrl: this.credentialsProvider.getConnectionMetadata()?.sso?.startUrl, + [CONVERSATION_ID_METRIC_KEY]: params.conversationId, + cwsprChatHasCodeSnippet: params.hasCodeSnippet, + cwsprChatTriggerInteraction: additionalParams.chatTriggerInteraction, + cwsprChatMessageId: params.messageId, + cwsprChatUserIntent: params.userIntent, + cwsprChatProgrammingLanguage: params.programmingLanguage, + cwsprChatResponseCodeSnippetCount: params.numberOfCodeBlocks, + cwsprChatResponseCode: additionalParams.chatResponseCode, + cwsprChatSourceLinkCount: additionalParams.chatSourceLinkCount, + cwsprChatReferencesCount: additionalParams.chatReferencesCount, + cwsprChatFollowUpCount: additionalParams.chatFollowUpCount, + cwsprTimeToFirstChunk: params.timeToFirstChunkMilliseconds, + cwsprChatFullResponseLatency: params.fullResponselatency, + cwsprChatTimeBetweenChunks: params.timeBetweenChunks, + cwsprChatRequestLength: params.requestLength, + cwsprChatResponseLength: params.responseLength, + cwsprChatConversationType: additionalParams.chatConversationType, + cwsprChatActiveEditorTotalCharacters: params.activeEditorTotalCharacters, + cwsprChatActiveEditorImportCount: additionalParams.chatActiveEditorImportCount, + codewhispererCustomizationArn: params.customizationArn, + }, + }) + } + const event: ChatAddMessageEvent = { conversationId: params.conversationId, messageId: params.messageId,