diff --git a/vscode/src/autoedits/adapters/cody-gateway.ts b/vscode/src/autoedits/adapters/cody-gateway.ts index fbb957a981f9..dcfacb4cd4ba 100644 --- a/vscode/src/autoedits/adapters/cody-gateway.ts +++ b/vscode/src/autoedits/adapters/cody-gateway.ts @@ -25,11 +25,11 @@ export class CodyGatewayAdapter implements AutoeditsModelAdapter { } } - private getMessageBody(option: AutoeditModelOptions): string { - const maxTokens = getMaxOutputTokensForAutoedits(option.codeToRewrite) + private getMessageBody(options: AutoeditModelOptions): string { + const maxTokens = getMaxOutputTokensForAutoedits(options.codeToRewrite) const body: FireworksCompatibleRequestParams = { stream: false, - model: option.model, + model: options.model, temperature: 0, max_tokens: maxTokens, response_format: { @@ -37,20 +37,20 @@ export class CodyGatewayAdapter implements AutoeditsModelAdapter { }, prediction: { type: 'content', - content: option.codeToRewrite, + content: options.codeToRewrite, }, rewrite_speculation: true, - user: option.userId || undefined, + user: options.userId || undefined, } - const request = option.isChatModel + const request = options.isChatModel ? { ...body, messages: getOpenaiCompatibleChatPrompt({ - systemMessage: option.prompt.systemMessage, - userMessage: option.prompt.userMessage, + systemMessage: options.prompt.systemMessage, + userMessage: options.prompt.userMessage, }), } - : { ...body, prompt: option.prompt.userMessage } + : { ...body, prompt: options.prompt.userMessage } return JSON.stringify(request) } } diff --git a/vscode/src/autoedits/analytics-logger/analytics-logger.test.ts b/vscode/src/autoedits/analytics-logger/analytics-logger.test.ts new file mode 100644 index 000000000000..40d22caf4152 --- /dev/null +++ b/vscode/src/autoedits/analytics-logger/analytics-logger.test.ts @@ -0,0 +1,309 @@ +import omit from 'lodash/omit' +import * as uuid from 'uuid' +import { type MockInstance, afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import * as vscode from 'vscode' + +import { mockAuthStatus, ps, telemetryRecorder } from '@sourcegraph/cody-shared' + +import { documentAndPosition } from '../../completions/test-helpers' +import * as sentryModule from '../../services/sentry/sentry' +import type { AutoeditModelOptions } from '../adapters/base' + +import { + AutoeditAnalyticsLogger, + type AutoeditRequestID, + autoeditSource, + autoeditTriggerKind, +} from './analytics-logger' + +describe('AutoeditAnalyticsLogger', () => { + let autoeditLogger: AutoeditAnalyticsLogger + let recordSpy: MockInstance + let stableIdCounter = 0 + + const { document, position } = documentAndPosition('█', 'typescript', 'file:///fake-file.ts') + + const modelOptions: AutoeditModelOptions = { + url: 'https://test-url.com/', + model: 'autoedit-model', + apiKey: 'api-key', + prompt: { + systemMessage: ps`This is test message`, + userMessage: ps`This is test prompt text`, + }, + codeToRewrite: 'This is test code to rewrite', + userId: 'test-user-id', + isChatModel: false, + } + + const requestStartMetadata: Parameters[0] = { + languageId: 'typescript', + model: 'autoedit-model', + traceId: 'trace-id', + triggerKind: autoeditTriggerKind.automatic, + codeToRewrite: 'Code to rewrite', + } + + function createAndAdvanceRequest({ + finalPhase, + prediction, + }: { finalPhase: 'suggested' | 'accepted' | 'rejected'; prediction: string }): AutoeditRequestID { + const requestId = autoeditLogger.createRequest(requestStartMetadata) + + autoeditLogger.markAsContextLoaded({ + requestId, + payload: { + contextSummary: { + strategy: 'none', + duration: 1.234, + totalChars: 10, + prefixChars: 5, + suffixChars: 5, + retrieverStats: {}, + }, + }, + }) + + // Stabilize latency for tests + vi.advanceTimersByTime(300) + + autoeditLogger.markAsLoaded({ + requestId, + modelOptions: modelOptions, + payload: { + prediction, + source: autoeditSource.network, + isFuzzyMatch: false, + responseHeaders: {}, + }, + }) + + autoeditLogger.markAsSuggested(requestId) + + if (finalPhase === 'accepted') { + autoeditLogger.markAsAccepted({ + requestId, + trackedRange: new vscode.Range(position, position), + position, + document, + prediction, + }) + } + + if (finalPhase === 'rejected') { + autoeditLogger.markAsRejected(requestId) + } + + return requestId + } + + beforeEach(() => { + autoeditLogger = new AutoeditAnalyticsLogger() + recordSpy = vi.spyOn(telemetryRecorder, 'recordEvent') + mockAuthStatus() + + stableIdCounter = 0 + vi.spyOn(uuid, 'v4').mockImplementation(() => `stable-id-for-tests-${++stableIdCounter}`) + + vi.useFakeTimers() + }) + + afterEach(() => { + vi.resetAllMocks() + vi.clearAllTimers() + }) + + it('logs a suggestion lifecycle (started -> contextLoaded -> loaded -> suggested -> accepted)', () => { + const prediction = 'console.log("Hello from autoedit!")' + const requestId = createAndAdvanceRequest({ + finalPhase: 'accepted', + prediction, + }) + + // Invalid transition attempt + autoeditLogger.markAsAccepted({ + requestId, + trackedRange: new vscode.Range(position, position), + position, + document, + prediction, + }) + + expect(recordSpy).toHaveBeenCalledTimes(3) + expect(recordSpy).toHaveBeenNthCalledWith(1, 'cody.autoedit', 'suggested', expect.any(Object)) + expect(recordSpy).toHaveBeenNthCalledWith(2, 'cody.autoedit', 'accepted', expect.any(Object)) + expect(recordSpy).toHaveBeenNthCalledWith( + 3, + 'cody.autoedit', + 'invalidTransitionToAccepted', + undefined + ) + + const suggestedEventPayload = recordSpy.mock.calls[0].at(2) + expect(suggestedEventPayload).toMatchInlineSnapshot(` + { + "billingMetadata": { + "category": "billable", + "product": "cody", + }, + "interactionID": "stable-id-for-tests-2", + "metadata": { + "charCount": 35, + "contextSummary.duration": 1.234, + "contextSummary.prefixChars": 5, + "contextSummary.suffixChars": 5, + "contextSummary.totalChars": 10, + "displayDuration": 0, + "isAccepted": 1, + "isDisjoint": 0, + "isFullyOutsideOfVisibleRanges": 1, + "isFuzzyMatch": 0, + "isPartiallyOutsideOfVisibleRanges": 1, + "isSelectionStale": 1, + "latency": 300, + "lineCount": 1, + "noActiveTextEditor": 0, + "otherCompletionProviderEnabled": 0, + "outsideOfActiveEditor": 1, + "recordsPrivateMetadataTranscript": 1, + "source": 1, + "suggestionsStartedSinceLastSuggestion": 0, + "triggerKind": 1, + "windowNotFocused": 1, + }, + "privateMetadata": { + "codeToRewrite": "Code to rewrite", + "contextSummary": { + "duration": 1.234, + "prefixChars": 5, + "retrieverStats": {}, + "strategy": "none", + "suffixChars": 5, + "totalChars": 10, + }, + "gatewayLatency": undefined, + "id": "stable-id-for-tests-2", + "languageId": "typescript", + "model": "autoedit-model", + "otherCompletionProviders": [], + "prediction": "console.log("Hello from autoedit!")", + "responseHeaders": {}, + "traceId": "trace-id", + "upstreamLatency": undefined, + }, + "version": 0, + } + `) + + const acceptedEventPayload = recordSpy.mock.calls[1].at(2) + // Accepted and suggested event payloads are only different by `billingMetadata`. + expect(acceptedEventPayload.billingMetadata).toMatchInlineSnapshot(` + { + "category": "core", + "product": "cody", + } + `) + + expect(omit(acceptedEventPayload, 'billingMetadata')).toEqual( + omit(suggestedEventPayload, 'billingMetadata') + ) + }) + + it('reuses the autoedit suggestion ID for the same prediction text', () => { + const prediction = 'function foo() {}\n' + + // First request (started -> contextLoaded -> loaded -> suggested -> rejected) + // The request ID should remain "in use" + createAndAdvanceRequest({ + finalPhase: 'rejected', + prediction, + }) + + // After acceptance, ID can no longer be reused + createAndAdvanceRequest({ + finalPhase: 'accepted', + prediction, + }) + + // Analytics event should use the new stable ID + createAndAdvanceRequest({ + finalPhase: 'rejected', + prediction, + }) + + expect(recordSpy).toHaveBeenCalledTimes(4) + expect(recordSpy).toHaveBeenNthCalledWith(1, 'cody.autoedit', 'suggested', expect.any(Object)) + expect(recordSpy).toHaveBeenNthCalledWith(2, 'cody.autoedit', 'suggested', expect.any(Object)) + expect(recordSpy).toHaveBeenNthCalledWith(3, 'cody.autoedit', 'accepted', expect.any(Object)) + expect(recordSpy).toHaveBeenNthCalledWith(4, 'cody.autoedit', 'suggested', expect.any(Object)) + + const suggestedEvent1 = recordSpy.mock.calls[0].at(2) + const suggestedEvent2 = recordSpy.mock.calls[1].at(2) + const suggestedEvent3 = recordSpy.mock.calls[3].at(2) + + // First two suggested calls should reuse the same stable ID + expect(suggestedEvent1.privateMetadata.id).toEqual('stable-id-for-tests-2') + expect(suggestedEvent2.privateMetadata.id).toEqual('stable-id-for-tests-2') + // The third one should be different because we just accepted a completion + // which removes the stable ID from the cache. + expect(suggestedEvent3.privateMetadata.id).toEqual('stable-id-for-tests-5') + }) + + it('logs `discarded` if the suggestion was not suggested for any reason', () => { + const requestId = autoeditLogger.createRequest(requestStartMetadata) + autoeditLogger.markAsContextLoaded({ requestId, payload: { contextSummary: undefined } }) + autoeditLogger.markAsDiscarded(requestId) + + expect(recordSpy).toHaveBeenCalledTimes(1) + expect(recordSpy).toHaveBeenCalledWith('cody.autoedit', 'discarded', expect.any(Object)) + }) + + it('handles invalid transitions by logging debug events (invalidTransitionToXYZ)', () => { + const requestId = autoeditLogger.createRequest(requestStartMetadata) + + // Both calls below are invalid transitions, so the logger logs debug events + autoeditLogger.markAsSuggested(requestId) + autoeditLogger.markAsRejected(requestId) + + expect(recordSpy).toHaveBeenCalledTimes(2) + expect(recordSpy).toHaveBeenNthCalledWith( + 1, + 'cody.autoedit', + 'invalidTransitionToSuggested', + undefined + ) + expect(recordSpy).toHaveBeenNthCalledWith( + 2, + 'cody.autoedit', + 'invalidTransitionToRejected', + undefined + ) + }) + + it('throttles repeated error logs, capturing the first occurrence immediately', () => { + // Force error logs to be reported: + vi.spyOn(sentryModule, 'shouldErrorBeReported').mockReturnValue(true) + + const error = new Error('Deliberate test error for autoedit') + autoeditLogger.logError(error) + + // First occurrence logs right away + expect(recordSpy).toHaveBeenCalledTimes(1) + expect(recordSpy).toHaveBeenCalledWith( + 'cody.autoedit', + 'error', + expect.objectContaining({ + version: 0, + metadata: { count: 1 }, + privateMetadata: expect.objectContaining({ + message: 'Deliberate test error for autoedit', + }), + }) + ) + + // Repeated calls should not log immediately + autoeditLogger.logError(error) + autoeditLogger.logError(error) + expect(recordSpy).toHaveBeenCalledTimes(1) + }) +}) diff --git a/vscode/src/autoedits/analytics-logger/analytics-logger.ts b/vscode/src/autoedits/analytics-logger/analytics-logger.ts new file mode 100644 index 000000000000..8440a44f12da --- /dev/null +++ b/vscode/src/autoedits/analytics-logger/analytics-logger.ts @@ -0,0 +1,636 @@ +import capitalize from 'lodash/capitalize' +import { LRUCache } from 'lru-cache' +import * as uuid from 'uuid' +import * as vscode from 'vscode' + +import { + type BillingCategory, + type BillingProduct, + isDotComAuthed, + isNetworkError, + telemetryRecorder, +} from '@sourcegraph/cody-shared' +import type { TelemetryEventParameters } from '@sourcegraph/telemetry' + +import { getOtherCompletionProvider } from '../../completions/analytics-logger' +import type { ContextSummary } from '../../completions/context/context-mixer' +import { lines } from '../../completions/text-processing' +import { type CodeGenEventMetadata, charactersLogger } from '../../services/CharactersLogger' +import { upstreamHealthProvider } from '../../services/UpstreamHealthProvider' +import { captureException, shouldErrorBeReported } from '../../services/sentry/sentry' +import { splitSafeMetadata } from '../../services/telemetry-v2' + +import type { AutoeditModelOptions } from '../adapters/base' +import { AutoeditSuggestionIdRegistry } from './suggestion-id-registry' + +/** + * This file implements a state machine to manage the lifecycle of an autoedit request. + * Each phase of the request is represented by a distinct state interface, and metadata + * evolves as the request progresses. + * + * 1. Each autoedit request phase (e.g., `started`, `loaded`, `accepted`) is represented by a + * `state` interface that extends `AutoeditBaseState` and adds phase-specific fields. + * + * 2. Valid transitions between phases are enforced using the `validRequestTransitions` map, + * ensuring logical progression through the request lifecycle. + * + * 3. The `payload` field in each state encapsulates the exact list of fields that we plan to send + * to our analytics backend. + * + * 4. Other top-level `state` fields are saved only for bookkeeping and won't end up at our + * analytics backend. This ensures we don't send unintentional or redundant information to + * the analytics backend. + * + * 5. Metadata is progressively enriched as the request transitions between states. + * + * 6. Eventually, once we reach one of the terminal states and log its current `payload`. + */ + +/** + * Defines the possible phases of our autoedit request state machine. + */ +type Phase = + /** The autoedit request has started. */ + | 'started' + /** The context for the autoedit has been loaded. */ + | 'contextLoaded' + /** The autoedit suggestion has been loaded — we have a prediction string. */ + | 'loaded' + /** The autoedit suggestion has been suggested to the user. */ + | 'suggested' + /** The user has accepted the suggestion. */ + | 'accepted' + /** The user has rejected the suggestion. */ + | 'rejected' + /** The autoedit request was discarded by our heuristics before being suggested to a user */ + | 'discarded' + +/** + * Defines which phases can transition to which other phases. + */ +const validRequestTransitions = { + started: ['contextLoaded', 'discarded'], + contextLoaded: ['loaded', 'discarded'], + loaded: ['suggested', 'discarded'], + suggested: ['accepted', 'rejected'], + accepted: [], + rejected: [], + discarded: [], +} as const satisfies Record + +export const autoeditTriggerKind = { + /** Suggestion was triggered automatically while editing. */ + automatic: 1, + + /** Suggestion was triggered manually by the user invoking the keyboard shortcut. */ + manual: 2, + + /** When the user uses the suggest widget to cycle through different suggestions. */ + suggestWidget: 3, + + /** Suggestion was triggered automatically by the selection change event. */ + cursor: 4, +} as const + +/** We use numeric keys to send these to the analytics backend */ +type AutoeditTriggerKindMetadata = (typeof autoeditTriggerKind)[keyof typeof autoeditTriggerKind] + +interface AutoeditStartedMetadata { + /** Document language ID (e.g., 'typescript'). */ + languageId: string + + /** Model used by Cody client to request the autosuggestion suggestion. */ + model: string + + /** Optional trace ID for cross-service correlation, if your environment provides it. */ + traceId: string + + /** Describes how the autoedit request was triggered by the user. */ + triggerKind: AutoeditTriggerKindMetadata + + /** + * The code to rewrite by autoedit. + * 🚨 SECURITY: included only for DotCom users. + */ + codeToRewrite?: string + + /** True if other autoedit/completion providers might also be active (e.g., Copilot). */ + otherCompletionProviderEnabled: boolean + + /** The exact list of other providers that are active, if known. */ + otherCompletionProviders: string[] + + /** The round trip timings to reach the Sourcegraph and Cody Gateway instances. */ + upstreamLatency?: number + gatewayLatency?: number +} + +interface AutoeditContextLoadedMetadata extends AutoeditStartedMetadata { + /** + * Information about the context retrieval process that lead to this autoedit request. Refer + * to the documentation of {@link ContextSummary} + */ + contextSummary?: ContextSummary +} + +/** + * A stable ID that identifies a particular autoedit suggestion. If the same text + * and context recurs, we reuse this ID to avoid double-counting. + */ +export type AutoeditSuggestionID = string & { readonly _brand: 'AutoeditSuggestionID' } + +export const autoeditSource = { + /** Autoedit originated from a request to our backend for the suggestion. */ + network: 1, + /** Autoedit originated from a client cached suggestion. */ + cache: 2, +} as const + +/** We use numeric keys to send these to the analytics backend */ +type AutoeditSourceMetadata = (typeof autoeditSource)[keyof typeof autoeditSource] + +interface AutoeditLoadedMetadata extends AutoeditContextLoadedMetadata { + /** + * An ID to uniquely identify a suggest autoedit. Note: It is possible for this ID to be part + * of two suggested events. This happens when the exact same autoedit text is shown again at + * the exact same location. We count this as the same autoedit and thus use the same ID. + */ + id: AutoeditSuggestionID + + /** Total lines in the suggestion. */ + lineCount: number + + /** Total characters in the suggestion. */ + charCount: number + + /** + * Prediction text snippet of the suggestion. + * Might be `undefined` if too long. + * 🚨 SECURITY: included only for DotCom users. + */ + prediction?: string + + /** The source of the suggestion, e.g. 'network', 'cache', etc. */ + source?: AutoeditSourceMetadata + + /** True if we fuzzy-matched this suggestion from a local or remote cache. */ + isFuzzyMatch?: boolean + + /** Optional set of relevant response headers (e.g. from Cody Gateway). */ + responseHeaders?: Record + + /** Time (ms) to generate or load the suggestion after it was started. */ + latency: number +} + +interface AutoEditFinalMetadata extends AutoeditLoadedMetadata { + /** Displayed to the user for this many milliseconds. */ + displayDuration: number + /** True if the suggestion was explicitly/intentionally accepted. */ + isAccepted: boolean + /** The number of the auto-edits started since the last suggestion was shown. */ + suggestionsStartedSinceLastSuggestion: number +} + +interface AutoeditAcceptedEventPayload + extends AutoEditFinalMetadata, + Omit {} + +interface AutoeditRejectedEventPayload extends AutoEditFinalMetadata {} +interface AutoeditDiscardedEventPayload extends AutoeditContextLoadedMetadata {} + +/** + * An ephemeral ID for a single “request” from creation to acceptance or rejection. + */ +export type AutoeditRequestID = string & { readonly _brand: 'AutoeditRequestID' } + +/** + * The base fields common to all request states. We track ephemeral times and + * the partial payload. Once we reach a certain phase, we log the payload as a telemetry event. + */ +interface AutoeditBaseState { + requestId: AutoeditRequestID + /** Current phase of the autoedit request */ + phase: Phase +} + +interface StartedState extends AutoeditBaseState { + phase: 'started' + /** Time (ms) when we started computing or requesting the suggestion. */ + startedAt: number + /** Partial payload for this phase. Will be augmented with more info as we progress. */ + payload: AutoeditStartedMetadata +} + +interface ContextLoadedState extends Omit { + phase: 'contextLoaded' + payload: AutoeditContextLoadedMetadata +} + +interface LoadedState extends Omit { + phase: 'loaded' + /** Timestamp when the suggestion completed generation/loading. */ + loadedAt: number + payload: AutoeditLoadedMetadata +} + +interface SuggestedState extends Omit { + phase: 'suggested' + /** Timestamp when the suggestion was first shown to the user. */ + suggestedAt: number +} + +interface AcceptedState extends Omit { + phase: 'accepted' + /** Timestamp when the user accepted the suggestion. */ + acceptedAt: number + /** Timestamp when the suggestion was logged to our analytics backend. This is to avoid double-logging. */ + suggestionLoggedAt?: number + payload: AutoeditAcceptedEventPayload +} + +interface RejectedState extends Omit { + phase: 'rejected' + /** Timestamp when the suggestion was logged to our analytics backend. This is to avoid double-logging. */ + suggestionLoggedAt?: number + payload: AutoeditRejectedEventPayload +} + +interface DiscardedState extends Omit { + phase: 'discarded' + /** Timestamp when the suggestion was logged to our analytics backend. This is to avoid double-logging. */ + suggestionLoggedAt?: number + payload: AutoeditDiscardedEventPayload +} + +interface PhaseStates { + started: StartedState + contextLoaded: ContextLoadedState + loaded: LoadedState + suggested: SuggestedState + accepted: AcceptedState + rejected: RejectedState + discarded: DiscardedState +} + +/** + * Using the validTransitions definition, we can derive which "from phases" lead to a given next phase, + * and map that to the correct PhaseStates[fromPhase]. + */ +type PreviousPossiblePhaseFrom = { + [F in Phase]: T extends (typeof validRequestTransitions)[F][number] ? PhaseStates[F] : never +}[Phase] + +type AutoeditRequestState = PhaseStates[Phase] + +type AutoeditEventAction = + | 'suggested' + | 'accepted' + | 'discarded' + | 'error' + | `invalidTransitionTo${Capitalize}` + +/** + * Specialized string type for referencing error messages in our rate-limiting map. + */ +type AutoeditErrorMessage = string & { readonly _brand: 'AutoeditErrorMessage' } + +export class AutoeditAnalyticsLogger { + /** + * Stores ephemeral AutoeditRequestState for each request ID. + */ + private activeRequests = new LRUCache({ max: 20 }) + + /** + * Encapsulates the logic for reusing stable suggestion IDs for repeated text/context. + */ + private suggestionIdRegistry = new AutoeditSuggestionIdRegistry() + + /** + * Tracks repeated errors via their message key to avoid spamming logs. + */ + private errorCounts = new Map() + private autoeditsStartedSinceLastSuggestion = 0 + private ERROR_THROTTLE_INTERVAL_MS = 10 * 60 * 1000 // 10 minutes + + /** + * Creates a new ephemeral request with initial metadata. At this stage, we do not have the prediction yet. + */ + public createRequest( + payload: Required< + Pick< + AutoeditStartedMetadata, + 'languageId' | 'model' | 'traceId' | 'triggerKind' | 'codeToRewrite' + > + > + ): AutoeditRequestID { + const { codeToRewrite, ...restPayload } = payload + const requestId = uuid.v4() as AutoeditRequestID + const otherCompletionProviders = getOtherCompletionProvider() + + const request: StartedState = { + requestId, + phase: 'started', + startedAt: getTimeNowInMillis(), + payload: { + otherCompletionProviderEnabled: otherCompletionProviders.length > 0, + otherCompletionProviders, + upstreamLatency: upstreamHealthProvider.getUpstreamLatency(), + gatewayLatency: upstreamHealthProvider.getGatewayLatency(), + // 🚨 SECURITY: included only for DotCom users. + codeToRewrite: isDotComAuthed() ? codeToRewrite : undefined, + ...restPayload, + }, + } + + this.activeRequests.set(requestId, request) + this.autoeditsStartedSinceLastSuggestion++ + return requestId + } + + public markAsContextLoaded({ + requestId, + payload, + }: { + requestId: AutoeditRequestID + payload: Pick + }): void { + this.tryTransitionTo(requestId, 'contextLoaded', request => ({ + ...request, + payload: { + ...request.payload, + contextSummary: payload.contextSummary, + }, + })) + } + + /** + * Mark when the suggestion finished generating/loading. This is also where + * we finally receive the prediction text, create a stable suggestion ID, + * and store the full suggestion metadata in ephemeral state. + */ + public markAsLoaded({ + requestId, + modelOptions, + payload, + }: { + requestId: AutoeditRequestID + modelOptions: AutoeditModelOptions + payload: Required< + Pick + > + }): void { + const { prediction, source, isFuzzyMatch, responseHeaders } = payload + const stableId = this.suggestionIdRegistry.getOrCreate(modelOptions, prediction) + const loadedAt = getTimeNowInMillis() + + this.tryTransitionTo(requestId, 'loaded', request => ({ + ...request, + loadedAt, + payload: { + ...request.payload, + id: stableId, + lineCount: lines(prediction).length, + charCount: prediction.length, + // 🚨 SECURITY: included only for DotCom users. + prediction: isDotComAuthed() && prediction.length < 300 ? prediction : undefined, + source, + isFuzzyMatch, + responseHeaders, + latency: loadedAt - request.startedAt, + }, + })) + } + + public markAsSuggested(requestId: AutoeditRequestID): SuggestedState | null { + const result = this.tryTransitionTo(requestId, 'suggested', currentRequest => ({ + ...currentRequest, + suggestedAt: getTimeNowInMillis(), + })) + + if (!result) { + return null + } + + // Reset the number of the auto-edits started since the last suggestion. + this.autoeditsStartedSinceLastSuggestion = 0 + + return result.updatedRequest + } + + public markAsAccepted({ + requestId, + trackedRange, + position, + document, + prediction, + }: { + requestId: AutoeditRequestID + trackedRange?: vscode.Range + position: vscode.Position + document: vscode.TextDocument + prediction: string + }): void { + const acceptedAt = getTimeNowInMillis() + + const result = this.tryTransitionTo(requestId, 'accepted', request => { + // Ensure the AutoeditSuggestionID is never reused by removing it from the suggestion id registry + this.suggestionIdRegistry.deleteEntryIfValueExists(request.payload.id) + + // Calculate metadata required for PCW. + const rangeForCharacterMetadata = trackedRange || new vscode.Range(position, position) + const { charsDeleted, charsInserted, ...charactersLoggerMetadata } = + charactersLogger.getChangeEventMetadataForCodyCodeGenEvents({ + document, + contentChanges: [ + { + range: rangeForCharacterMetadata, + rangeOffset: document.offsetAt(rangeForCharacterMetadata.start), + rangeLength: 0, + text: prediction, + }, + ], + reason: undefined, + }) + + return { + ...request, + acceptedAt, + payload: { + ...request.payload, + ...charactersLoggerMetadata, + isAccepted: true, + displayDuration: acceptedAt - request.suggestedAt, + suggestionsStartedSinceLastSuggestion: this.autoeditsStartedSinceLastSuggestion, + }, + } + }) + + if (result?.updatedRequest) { + this.writeAutoeditRequestEvent('suggested', result.updatedRequest) + this.writeAutoeditRequestEvent('accepted', result.updatedRequest) + + this.activeRequests.delete(result.updatedRequest.requestId) + } + } + + public markAsRejected(requestId: AutoeditRequestID): void { + const result = this.tryTransitionTo(requestId, 'rejected', request => ({ + ...request, + payload: { + ...request.payload, + isAccepted: false, + displayDuration: getTimeNowInMillis() - request.suggestedAt, + suggestionsStartedSinceLastSuggestion: this.autoeditsStartedSinceLastSuggestion, + }, + })) + + if (result?.updatedRequest) { + this.writeAutoeditRequestEvent('suggested', result.updatedRequest) + + // Suggestions are kept in the LRU cache for longer. This is because they + // can still become visible if e.g. they are served from the cache and we + // need to retain the ability to mark them as seen. + } + } + + public markAsDiscarded(requestId: AutoeditRequestID): void { + const result = this.tryTransitionTo(requestId, 'discarded', currentRequest => currentRequest) + + if (result?.updatedRequest) { + this.writeAutoeditRequestEvent('discarded', result.updatedRequest) + this.activeRequests.delete(result.updatedRequest.requestId) + } + } + + private tryTransitionTo

( + requestId: AutoeditRequestID, + nextPhase: P, + patch: (currentRequest: PreviousPossiblePhaseFrom

) => Omit + ): { currentRequest: PreviousPossiblePhaseFrom

; updatedRequest: PhaseStates[P] } | null { + const currentRequest = this.getRequestIfReadyForNextPhase(requestId, nextPhase) + + if (!currentRequest) { + return null + } + + const updatedRequest = { + ...currentRequest, + ...patch(currentRequest), + phase: nextPhase, + } as PhaseStates[P] + + this.activeRequests.set(requestId, updatedRequest) + return { updatedRequest, currentRequest } + } + + /** + * Retrieves the request if it is in a phase that can transition to nextPhase, + * returning null if not found or if the transition is invalid. Uses the derived + * PreviousPossiblePhaseFrom type so that the returned State has the correct fields. + */ + private getRequestIfReadyForNextPhase( + requestId: AutoeditRequestID, + nextPhase: T + ): PreviousPossiblePhaseFrom | null { + const request = this.activeRequests.get(requestId) + + if ( + !request || + !(validRequestTransitions[request.phase] as readonly Phase[]).includes(nextPhase) + ) { + this.writeDebugBookkeepingEvent( + `invalidTransitionTo${capitalize(nextPhase) as Capitalize}` + ) + + return null + } + + return request as PreviousPossiblePhaseFrom + } + + private writeAutoeditRequestEvent( + action: AutoeditEventAction, + state: AcceptedState | RejectedState | DiscardedState + ): void { + const { suggestionLoggedAt, payload } = state + + if (action === 'suggested' && suggestionLoggedAt) { + return + } + + // Update the request state to mark the suggestion as logged. + state.suggestionLoggedAt = getTimeNowInMillis() + + const { metadata, privateMetadata } = splitSafeMetadata(payload) + this.writeAutoeditEvent(action, { + version: 0, + // Extract `id` from payload into the first-class `interactionId` field. + interactionID: 'id' in payload ? payload.id : undefined, + metadata: { + ...metadata, + recordsPrivateMetadataTranscript: 'prediction' in privateMetadata ? 1 : 0, + }, + privateMetadata, + billingMetadata: { + product: 'cody', + // TODO: double check with the analytics team + // whether we should be categorizing the different completion event types. + category: action === 'suggested' ? 'billable' : 'core', + }, + }) + } + + private writeAutoeditEvent( + action: AutoeditEventAction, + params?: TelemetryEventParameters<{ [key: string]: number }, BillingProduct, BillingCategory> + ): void { + telemetryRecorder.recordEvent('cody.autoedit', action, params) + } + + /** + * Rate-limited error logging, capturing exceptions with Sentry and grouping repeated logs. + */ + public logError(error: Error): void { + if (!shouldErrorBeReported(error, false)) { + return + } + captureException(error) + + const messageKey = error.message as AutoeditErrorMessage + const traceId = isNetworkError(error) ? error.traceId : undefined + + const currentCount = this.errorCounts.get(messageKey) ?? 0 + if (currentCount === 0) { + this.writeAutoeditEvent('error', { + version: 0, + metadata: { count: 1 }, + privateMetadata: { message: error.message, traceId }, + }) + + // After the interval, flush repeated errors + setTimeout(() => { + const finalCount = this.errorCounts.get(messageKey) ?? 0 + if (finalCount > 0) { + this.writeAutoeditEvent('error', { + version: 0, + metadata: { count: finalCount }, + privateMetadata: { message: error.message, traceId }, + }) + } + this.errorCounts.set(messageKey, 0) + }, this.ERROR_THROTTLE_INTERVAL_MS) + } + this.errorCounts.set(messageKey, currentCount + 1) + } + + private writeDebugBookkeepingEvent(action: `invalidTransitionTo${Capitalize}`): void { + this.writeAutoeditEvent(action) + } +} + +export const autoeditAnalyticsLogger = new AutoeditAnalyticsLogger() + +function getTimeNowInMillis(): number { + return Math.floor(performance.now()) +} diff --git a/vscode/src/autoedits/analytics-logger/index.ts b/vscode/src/autoedits/analytics-logger/index.ts new file mode 100644 index 000000000000..6ae02d5165b1 --- /dev/null +++ b/vscode/src/autoedits/analytics-logger/index.ts @@ -0,0 +1 @@ +export * from './analytics-logger' diff --git a/vscode/src/autoedits/analytics-logger/suggestion-id-registry.ts b/vscode/src/autoedits/analytics-logger/suggestion-id-registry.ts new file mode 100644 index 000000000000..503cac896800 --- /dev/null +++ b/vscode/src/autoedits/analytics-logger/suggestion-id-registry.ts @@ -0,0 +1,57 @@ +import { LRUCache } from 'lru-cache' +import * as uuid from 'uuid' + +import type { AutoeditModelOptions } from '../adapters/base' +import type { AutoeditSuggestionID } from './analytics-logger' + +/** + * A specialized string type for the stable “suggestion key” in caches. + */ +export type AutoeditSuggestionKey = string & { readonly _brand: 'AutoeditSuggestionKey' } + +/** + * A small helper class that generates or retrieves stable AutoeditSuggestionIDs. + * This encapsulates the logic of deduplicating identical suggestions. + */ +export class AutoeditSuggestionIdRegistry { + private suggestionIdCache = new LRUCache({ max: 50 }) + + /** + * Produce a stable suggestion ID for the given context + text, reusing + * previously generated IDs for duplicates. + */ + public getOrCreate(options: AutoeditModelOptions, prediction: string): AutoeditSuggestionID { + const key = this.getAutoeditSuggestionKey(options, prediction) + let stableId = this.suggestionIdCache.get(key) + if (!stableId) { + stableId = uuid.v4() as AutoeditSuggestionID + this.suggestionIdCache.set(key, stableId) + } + return stableId + } + + /** + * Creates a stable string key that identifies the same suggestion across repeated displays. + */ + private getAutoeditSuggestionKey( + params: AutoeditModelOptions, + prediction: string + ): AutoeditSuggestionKey { + const key = `${params.prompt.systemMessage}█${params.prompt.userMessage}█${prediction}` + return key as AutoeditSuggestionKey + } + + public deleteEntryIfValueExists(id: AutoeditSuggestionID): void { + let matchingKey: string | null = null + + this.suggestionIdCache.forEach((value, key) => { + if (value === id) { + matchingKey = key + } + }) + + if (matchingKey) { + this.suggestionIdCache.delete(matchingKey) + } + } +} diff --git a/vscode/src/completions/analytics-logger.ts b/vscode/src/completions/analytics-logger.ts index c96925210d4d..c4b2ed7d341d 100644 --- a/vscode/src/completions/analytics-logger.ts +++ b/vscode/src/completions/analytics-logger.ts @@ -1279,7 +1279,7 @@ const otherCompletionProviders = [ 'TabNine.tabnine-vscode', 'Venthe.fauxpilot', ] -function getOtherCompletionProvider(): string[] { +export function getOtherCompletionProvider(): string[] { return otherCompletionProviders.filter(id => vscode.extensions.getExtension(id)?.isActive) }