Skip to content

Commit

Permalink
Emit chat and inline events to destination (#574)
Browse files Browse the repository at this point in the history 
* Add logging support in case of failures from STE call

* Emit ChatAddMessage and ChatInteractWithMessage to destination based on client flag

* Emit ChatUserModification to destination based on client flag

* Inline completion events to destination behind a flag

* Add unit test to check no event is send to destination when sendTelemetryEventsToDestination flag is disabled

* changing the default value of enableTelemetryEventsToDestination to true

* Fix the error function to catch STE call gracefully

* Disable unnecessary configuration call and remove unused imports

---------

Co-authored-by: Paras <[email protected]>
Co-authored-by: Viktor Shcherba <[email protected]>
  • Loading branch information
@pras0131 @viktorsaws
3 people authored Nov 8, 2024
1 parent 609eb88 commit 2124762
Show file tree
Hide file tree
Showing 10 changed files with 1,166 additions and 445 deletions.
7 changes: 6 additions & 1 deletion server/aws-lsp-codewhisperer/src/language-server/chat/chatController.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ describe('ChatController', () => {
let chatController: ChatController
let telemetryService: TelemetryService
let invokeSendTelemetryEventStub: sinon.SinonStub
let telemetry: Telemetry

beforeEach(() => {
sendMessageStub = sinon.stub(CodeWhispererStreaming.prototype, 'sendMessage').callsFake(() => {
Expand Down Expand Up @@ -120,7 +121,11 @@ describe('ChatController', () => {
}),
}

telemetryService = new TelemetryService(mockCredentialsProvider, 'bearer', {} as Telemetry, logging, {})
telemetry = {
emitMetric: sinon.stub(),
onClientTelemetry: sinon.stub(),
}
telemetryService = new TelemetryService(mockCredentialsProvider, 'bearer', telemetry, logging, {})
invokeSendTelemetryEventStub = sinon.stub(telemetryService, 'sendTelemetryEvent' as any)
chatController = new ChatController(chatSessionManagementService, testFeatures, telemetryService)
})
Expand Down
8 changes: 6 additions & 2 deletions server/aws-lsp-codewhisperer/src/language-server/chat/chatController.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -415,8 +415,12 @@ export class ChatController implements ChatHandlers {
if (qConfig) {
this.#customizationArn = undefinedIfEmpty(qConfig.customization)
this.#log(`Chat configuration updated to use ${this.#customizationArn}`)
const enableTelemetryEventsToDestination = qConfig['enableTelemetryEventsToDestination'] === true
this.#telemetryService.updateEnableTelemetryEventsToDestination(enableTelemetryEventsToDestination)
/*
The flag enableTelemetryEventsToDestination is set to true temporarily. It's value will be determined through destination
configuration post all events migration to STE. It'll be replaced by qConfig['enableTelemetryEventsToDestination'] === true
*/
// const enableTelemetryEventsToDestination = true
// this.#telemetryService.updateEnableTelemetryEventsToDestination(enableTelemetryEventsToDestination)
const optOutTelemetryPreference = qConfig['optOutTelemetry'] === true ? 'OPTOUT' : 'OPTIN'
this.#telemetryService.updateOptOutPreference(optOutTelemetryPreference)
}
Expand Down
88 changes: 23 additions & 65 deletions server/aws-lsp-codewhisperer/src/language-server/chat/telemetry/chatTelemetryController.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -121,16 +121,6 @@ export class ChatTelemetryController {
}

public emitModifyCodeMetric(entry: AcceptedSuggestionChatEntry, percentage: number) {
const data: Omit<ModifyCodeEvent, 'cwsprChatConversationId'> = {
cwsprChatMessageId: entry.messageId,
cwsprChatModificationPercentage: percentage,
codewhispererCustomizationArn: entry.customizationArn,
}

this.emitConversationMetric({
name: ChatTelemetryEventName.ModifyCode,
data,
})
this.#telemetryService.emitChatUserModificationEvent({
conversationId: entry.conversationId,
messageId: entry.messageId,
Expand Down Expand Up @@ -175,48 +165,31 @@ export class ChatTelemetryController {

public emitAddMessageMetric(tabId: string, metric: Partial<CombinedConversationEvent>) {
const conversationId = this.getConversationId(tabId)
this.#telemetryService.emitChatAddMessage({
conversationId: conversationId,
messageId: metric.cwsprChatMessageId,
customizationArn: metric.codewhispererCustomizationArn,
userIntent: metric.cwsprChatUserIntent,
hasCodeSnippet: metric.cwsprChatHasCodeSnippet,
programmingLanguage: metric.cwsprChatProgrammingLanguage as CodewhispererLanguage,
activeEditorTotalCharacters: metric.cwsprChatActiveEditorTotalCharacters,
timeToFirstChunkMilliseconds: metric.cwsprTimeToFirstChunk,
timeBetweenChunks: metric.cwsprChatTimeBetweenChunks,
fullResponselatency: metric.cwsprChatFullResponseLatency,
requestLength: metric.cwsprChatRequestLength,
responseLength: metric.cwsprChatResponseLength,
numberOfCodeBlocks: metric.cwsprChatResponseCodeSnippetCount,
})
this.emitConversationMetric(
this.#telemetryService.emitChatAddMessage(
{
name: ChatTelemetryEventName.AddMessage,
data: {
cwsprChatHasCodeSnippet: metric.cwsprChatHasCodeSnippet,
cwsprChatTriggerInteraction: metric.cwsprChatTriggerInteraction,
cwsprChatMessageId: metric.cwsprChatMessageId,
cwsprChatUserIntent: metric.cwsprChatUserIntent,
cwsprChatProgrammingLanguage: metric.cwsprChatProgrammingLanguage,
cwsprChatResponseCodeSnippetCount: metric.cwsprChatResponseCodeSnippetCount,
cwsprChatResponseCode: metric.cwsprChatResponseCode,
cwsprChatSourceLinkCount: metric.cwsprChatSourceLinkCount,
cwsprChatReferencesCount: metric.cwsprChatReferencesCount,
cwsprChatFollowUpCount: metric.cwsprChatFollowUpCount,
cwsprTimeToFirstChunk: metric.cwsprTimeToFirstChunk,
cwsprChatFullResponseLatency: metric.cwsprChatFullResponseLatency,
cwsprChatTimeBetweenChunks: metric.cwsprChatTimeBetweenChunks,
cwsprChatRequestLength: metric.cwsprChatRequestLength,
cwsprChatResponseLength: metric.cwsprChatResponseLength,
cwsprChatConversationType: metric.cwsprChatConversationType,
cwsprChatActiveEditorTotalCharacters: metric.cwsprChatActiveEditorTotalCharacters,
cwsprChatActiveEditorImportCount: metric.cwsprChatActiveEditorImportCount,
codewhispererCustomizationArn: metric.codewhispererCustomizationArn,
// not possible: cwsprChatResponseType: metric.cwsprChatResponseType,
},
conversationId: conversationId,
messageId: metric.cwsprChatMessageId,
customizationArn: metric.codewhispererCustomizationArn,
userIntent: metric.cwsprChatUserIntent,
hasCodeSnippet: metric.cwsprChatHasCodeSnippet,
programmingLanguage: metric.cwsprChatProgrammingLanguage as CodewhispererLanguage,
activeEditorTotalCharacters: metric.cwsprChatActiveEditorTotalCharacters,
timeToFirstChunkMilliseconds: metric.cwsprTimeToFirstChunk,
timeBetweenChunks: metric.cwsprChatTimeBetweenChunks,
fullResponselatency: metric.cwsprChatFullResponseLatency,
requestLength: metric.cwsprChatRequestLength,
responseLength: metric.cwsprChatResponseLength,
numberOfCodeBlocks: metric.cwsprChatResponseCodeSnippetCount,
},
tabId
{
chatTriggerInteraction: metric.cwsprChatTriggerInteraction,
chatResponseCode: metric.cwsprChatResponseCode,
chatSourceLinkCount: metric.cwsprChatSourceLinkCount,
chatReferencesCount: metric.cwsprChatReferencesCount,
chatFollowUpCount: metric.cwsprChatFollowUpCount,
chatConversationType: metric.cwsprChatConversationType,
chatActiveEditorImportCount: metric.cwsprChatActiveEditorImportCount,
}
)
// Store the customization value associated with the message
if (metric.cwsprChatMessageId && metric.codewhispererCustomizationArn) {
Expand Down Expand Up @@ -250,13 +223,6 @@ export class ChatTelemetryController {
this.#telemetryService.emitChatInteractWithMessage(metric, {
conversationId: this.getConversationId(tabId),
})
this.emitConversationMetric(
{
name: ChatTelemetryEventName.InteractWithMessage,
data: metric,
},
tabId
)
}

public emitMessageResponseError(tabId: string, metric: Partial<CombinedConversationEvent>) {
Expand Down Expand Up @@ -364,10 +330,6 @@ export class ChatTelemetryController {
this.#telemetryService.emitChatInteractWithMessage(voteData, {
conversationId: this.getConversationId(params.tabId),
})
this.emitConversationMetric({
name: ChatTelemetryEventName.InteractWithMessage,
data: voteData,
})
break
case ChatUIEventName.InsertToCursorPosition:
case ChatUIEventName.CopyToClipboard:
Expand All @@ -394,10 +356,6 @@ export class ChatTelemetryController {
? params.code?.split('\n').length
: undefined,
})
this.emitConversationMetric({
name: ChatTelemetryEventName.InteractWithMessage,
data: interactData,
})
break
case ChatUIEventName.LinkClick:
case ChatUIEventName.InfoLinkClick:
Expand Down
50 changes: 8 additions & 42 deletions server/aws-lsp-codewhisperer/src/language-server/codeWhispererServer.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -177,56 +177,18 @@ const emitUserTriggerDecisionTelemetry = (
return
}

emitAggregatedUserTriggerDecisionTelemetry(telemetry, telemetryService, session, timeSinceLastUserModification)
emitAggregatedUserTriggerDecisionTelemetry(telemetryService, session, timeSinceLastUserModification)
emitUserDecisionTelemetry(telemetry, session)

session.reportedUserDecision = true
}

const emitAggregatedUserTriggerDecisionTelemetry = (
telemetry: Telemetry,
telemetryService: TelemetryService,
session: CodeWhispererSession,
timeSinceLastUserModification?: number
) => {
telemetryService.emitUserTriggerDecision(session, timeSinceLastUserModification)
// TODO: the below emitted event to the Toolkit DWH will be moved to telemetryService as well.
const data: CodeWhispererUserTriggerDecisionEvent = {
codewhispererSessionId: session.codewhispererSessionId || '',
codewhispererFirstRequestId: session.responseContext?.requestId || '',
credentialStartUrl: session.credentialStartUrl,
codewhispererSuggestionState: session.getAggregatedUserTriggerDecision(),
codewhispererCompletionType:
session.suggestions.length > 0 ? getCompletionType(session.suggestions[0]) : undefined,
codewhispererLanguage: session.language,
codewhispererTriggerType: session.triggerType,
codewhispererAutomatedTriggerType: session.autoTriggerType,
codewhispererTriggerCharacter:
session.autoTriggerType === 'SpecialCharacters' ? session.triggerCharacter : undefined,
codewhispererLineNumber: session.startPosition.line,
codewhispererCursorOffset: session.startPosition.character,
codewhispererSuggestionCount: session.suggestions.length,
codewhispererClassifierResult: session.classifierResult,
codewhispererClassifierThreshold: session.classifierThreshold,
codewhispererTotalShownTime: session.totalSessionDisplayTime || 0,
codewhispererTypeaheadLength: session.typeaheadLength || 0,
// Global time between any 2 document changes
codewhispererTimeSinceLastDocumentChange: timeSinceLastUserModification,
codewhispererTimeSinceLastUserDecision: session.previousTriggerDecisionTime
? session.startTime - session.previousTriggerDecisionTime
: undefined,
codewhispererTimeToFirstRecommendation: session.timeToFirstRecommendation,
codewhispererPreviousSuggestionState: session.previousTriggerDecision,
codewhispererSupplementalContextTimeout: session.supplementalMetadata?.isProcessTimeout,
codewhispererSupplementalContextIsUtg: session.supplementalMetadata?.isUtg,
codewhispererSupplementalContextLength: session.supplementalMetadata?.contentsLength,
codewhispererCustomizationArn: session.customizationArn,
}

telemetry.emitMetric({
name: 'codewhisperer_userTriggerDecision',
data,
})
}

const emitUserDecisionTelemetry = (telemetry: Telemetry, session: CodeWhispererSession) => {
Expand Down Expand Up @@ -335,7 +297,7 @@ export const CodewhispererServerFactory =
// the context of a single response.
let includeSuggestionsWithCodeReferences = false

const codePercentageTracker = new CodePercentageTracker(telemetry, telemetryService)
const codePercentageTracker = new CodePercentageTracker(telemetryService)

const codeDiffTracker: CodeDiffTracker<AcceptedInlineSuggestionEntry> = new CodeDiffTracker(
workspace,
Expand Down Expand Up @@ -640,8 +602,12 @@ export const CodewhispererServerFactory =
logging.log(
`Inline completion configuration updated to use ${codeWhispererService.customizationArn}`
)
const enableTelemetryEventsToDestination = qConfig['enableTelemetryEventsToDestination'] === true
telemetryService.updateEnableTelemetryEventsToDestination(enableTelemetryEventsToDestination)
/*
The flag enableTelemetryEventsToDestination is set to true temporarily. It's value will be determined through destination
configuration post all events migration to STE. It'll be replaced by qConfig['enableTelemetryEventsToDestination'] === true
*/
// const enableTelemetryEventsToDestination = true
// telemetryService.updateEnableTelemetryEventsToDestination(enableTelemetryEventsToDestination)
const optOutTelemetryPreference = qConfig['optOutTelemetry'] === true ? 'OPTOUT' : 'OPTIN'
telemetryService.updateOptOutPreference(optOutTelemetryPreference)
}
Expand Down
24 changes: 15 additions & 9 deletions server/aws-lsp-codewhisperer/src/language-server/telemetry.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import sinon, { StubbedInstance, stubInterface } from 'ts-sinon'
import { TextDocument } from 'vscode-languageserver-textdocument'
import { CodewhispererServerFactory } from './codeWhispererServer'
import { CodeWhispererServiceBase, ResponseContext, Suggestion } from './codeWhispererService'
import { TelemetryService } from './telemetryService'

describe('CodeWhisperer Server', () => {
const HELLO_WORLD_IN_CSHARP = `
Expand Down Expand Up @@ -36,6 +37,7 @@ class HelloWorld
// for examples on how to mock just the SDK client
let service: StubbedInstance<CodeWhispererServiceBase>
let clock: sinon.SinonFakeTimers
let telemetryServiceSpy: sinon.SinonSpy

beforeEach(async () => {
clock = sinon.useFakeTimers()
Expand Down Expand Up @@ -69,6 +71,7 @@ class HelloWorld
})

it('should emit Code Percentage telemetry event every 5 minutes', async () => {
telemetryServiceSpy = sinon.spy(TelemetryService.prototype, 'emitCodeCoverageEvent')
await features.simulateTyping(SOME_FILE.uri, SOME_TYPING)

const updatedDocument = features.documents[SOME_FILE.uri]
Expand Down Expand Up @@ -107,16 +110,19 @@ class HelloWorld

clock.tick(5000 * 60)

sinon.assert.calledWithExactly(features.telemetry.emitMetric, {
name: 'codewhisperer_codePercentage',
data: {
codewhispererTotalTokens: totalInsertCharacters,
codewhispererLanguage: 'csharp',
codewhispererSuggestedTokens: codeWhispererCharacters,
codewhispererPercentage: codePercentage,
successCount: 1,
sinon.assert.calledWithExactly(
telemetryServiceSpy,
{
languageId: 'csharp',
customizationArn: undefined,
totalCharacterCount: totalInsertCharacters,
acceptedCharacterCount: codeWhispererCharacters,
},
})
{
percentage: codePercentage,
successCount: 1,
}
)
})
})
})
Loading

0 comments on commit 2124762

Please sign in to comment.