diff --git a/vscode/src/autoedits/adapters/cody-gateway.ts b/vscode/src/autoedits/adapters/cody-gateway.ts index fbb957a981f9..dcfacb4cd4ba 100644 --- a/vscode/src/autoedits/adapters/cody-gateway.ts +++ b/vscode/src/autoedits/adapters/cody-gateway.ts @@ -25,11 +25,11 @@ export class CodyGatewayAdapter implements AutoeditsModelAdapter { } } - private getMessageBody(option: AutoeditModelOptions): string { - const maxTokens = getMaxOutputTokensForAutoedits(option.codeToRewrite) + private getMessageBody(options: AutoeditModelOptions): string { + const maxTokens = getMaxOutputTokensForAutoedits(options.codeToRewrite) const body: FireworksCompatibleRequestParams = { stream: false, - model: option.model, + model: options.model, temperature: 0, max_tokens: maxTokens, response_format: { @@ -37,20 +37,20 @@ export class CodyGatewayAdapter implements AutoeditsModelAdapter { }, prediction: { type: 'content', - content: option.codeToRewrite, + content: options.codeToRewrite, }, rewrite_speculation: true, - user: option.userId || undefined, + user: options.userId || undefined, } - const request = option.isChatModel + const request = options.isChatModel ? { ...body, messages: getOpenaiCompatibleChatPrompt({ - systemMessage: option.prompt.systemMessage, - userMessage: option.prompt.userMessage, + systemMessage: options.prompt.systemMessage, + userMessage: options.prompt.userMessage, }), } - : { ...body, prompt: option.prompt.userMessage } + : { ...body, prompt: options.prompt.userMessage } return JSON.stringify(request) } } diff --git a/vscode/src/autoedits/analytics-logger/analytics-logger.test.ts b/vscode/src/autoedits/analytics-logger/analytics-logger.test.ts new file mode 100644 index 000000000000..47406cc58dd0 --- /dev/null +++ b/vscode/src/autoedits/analytics-logger/analytics-logger.test.ts @@ -0,0 +1,317 @@ +import { type MockInstance, afterEach, beforeEach, describe, expect, it, vi } from 'vitest' +import * as vscode from 'vscode' + +import { ps, telemetryRecorder } from '@sourcegraph/cody-shared' + +import * as sentryModule from '../../services/sentry/sentry' +import type { AutoeditModelOptions } from '../adapters/base' + +import { AutoeditAnalyticsLogger } from './analytics-logger' + +// Ensure we can override shouldErrorBeReported in each test. +vi.mock('../../services/sentry/sentry', async () => { + const actual: typeof import('../../services/sentry/sentry') = await vi.importActual( + '../../services/sentry/sentry' + ) + return { + ...actual, + shouldErrorBeReported: vi.fn(), + } +}) + +describe('AutoeditAnalyticsLogger', () => { + let autoeditLogger: AutoeditAnalyticsLogger + let recordSpy: MockInstance + const fakeDocument = { + offsetAt: () => 0, + uri: { toString: () => 'file:///fake-file.ts' }, + } as unknown as vscode.TextDocument + const fakePosition = new vscode.Position(0, 0) + const defaultModelOptions: AutoeditModelOptions = { + url: 'https://my-test-url.com/', + model: 'my-autoedit-model', + apiKey: 'my-api-key', + prompt: { + systemMessage: ps`This is test message`, + userMessage: ps`This is test prompt text`, + }, + codeToRewrite: 'This is test code to rewrite', + userId: 'test-user-id', + isChatModel: false, + } + + beforeEach(() => { + autoeditLogger = new AutoeditAnalyticsLogger() + recordSpy = vi.spyOn(telemetryRecorder, 'recordEvent') + }) + + afterEach(() => { + vi.resetAllMocks() + }) + + it('logs a suggestion lifecycle (started -> contextLoaded -> loaded -> suggested -> read -> accepted)', () => { + // 1. Create session + const sessionId = autoeditLogger.createSession({ + languageId: 'typescript', + model: 'my-autoedit-model', + traceId: 'trace-xyz', + }) + + // 2. Mark context loaded + autoeditLogger.markAsContextLoaded({ + sessionId, + payload: { + contextSummary: { + strategy: 'none', + duration: 1.234, + totalChars: 10, + prefixChars: 5, + suffixChars: 5, + retrieverStats: {}, + }, + }, + }) + + // 3. Mark loaded + const prediction = 'console.log("Hello from autoedit!")' + autoeditLogger.markAsLoaded({ + sessionId, + modelOptions: defaultModelOptions, + payload: { + prediction, + source: 'network', + isFuzzyMatch: false, + responseHeaders: {}, + }, + }) + + // 4. Mark suggested + autoeditLogger.markAsSuggested(sessionId) + + // 5. Mark read + autoeditLogger.markAsRead(sessionId) + + // 6. Mark accepted + autoeditLogger.markAsAccepted({ + sessionId, + trackedRange: new vscode.Range(fakePosition, fakePosition), + position: fakePosition, + document: fakeDocument, + prediction, + }) + + // Since the logger short-circuits after logging once (by setting suggestionLoggedAt), + // we see exactly ONE event record with action = "suggested". + // We only check that it's "cody.autoedit", "suggested", and an object with certain keys. + expect(recordSpy).toHaveBeenCalledTimes(1) + expect(recordSpy).toHaveBeenCalledWith( + 'cody.autoedit', + 'suggested', + expect.objectContaining({ + version: 0, + billingMetadata: expect.any(Object), + metadata: expect.any(Object), + privateMetadata: expect.any(Object), + }) + ) + }) + + it('reuses the autoedit suggestion ID for the same prediction text', () => { + const prediction = 'function foo() {}\n' + + // FIRST SESSION (started -> contextLoaded -> loaded -> suggested) + const session1 = autoeditLogger.createSession({ + languageId: 'typescript', + model: 'my-autoedit-model', + traceId: 'trace-abc', + }) + autoeditLogger.markAsContextLoaded({ + sessionId: session1, + payload: { contextSummary: undefined }, + }) + autoeditLogger.markAsLoaded({ + sessionId: session1, + modelOptions: defaultModelOptions, + payload: { + prediction, + source: 'network', + isFuzzyMatch: false, + responseHeaders: {}, + }, + }) + autoeditLogger.markAsSuggested(session1) + // We do NOT accept or reject so that the ID remains "in use." + + // SECOND SESSION with the same text + const session2 = autoeditLogger.createSession({ + languageId: 'typescript', + model: 'my-autoedit-model', + traceId: 'trace-def', + }) + autoeditLogger.markAsContextLoaded({ + sessionId: session2, + payload: { contextSummary: undefined }, + }) + autoeditLogger.markAsLoaded({ + sessionId: session2, + modelOptions: defaultModelOptions, + payload: { + prediction, + source: 'cache', + isFuzzyMatch: true, + responseHeaders: {}, + }, + }) + autoeditLogger.markAsSuggested(session2) + + // Accept the second session to finalize it + autoeditLogger.markAsAccepted({ + sessionId: session2, + trackedRange: new vscode.Range(fakePosition, fakePosition), + position: fakePosition, + document: fakeDocument, + prediction, + }) + + // After acceptance, ID can no longer be reused + const session3 = autoeditLogger.createSession({ + languageId: 'typescript', + model: 'my-autoedit-model', + traceId: 'trace-ghi', + }) + autoeditLogger.markAsContextLoaded({ + sessionId: session3, + payload: { contextSummary: undefined }, + }) + autoeditLogger.markAsLoaded({ + sessionId: session3, + modelOptions: defaultModelOptions, + payload: { + prediction, + source: 'cache', + isFuzzyMatch: true, + responseHeaders: {}, + }, + }) + + // Expect 1 telemetry call from the acceptance on session2 + expect(recordSpy).toHaveBeenCalledTimes(1) + expect(recordSpy).toHaveBeenCalledWith('cody.autoedit', 'suggested', expect.any(Object)) + }) + + it('logs noResponse if no suggestion was produced', () => { + // Start a session but never actually produce a suggestion + const sessionId = autoeditLogger.createSession({ + languageId: 'typescript', + model: 'my-autoedit-model', + traceId: 'trace-nr', + }) + autoeditLogger.markAsContextLoaded({ + sessionId, + payload: { contextSummary: undefined }, + }) + autoeditLogger.markAsNoResponse(sessionId) + + // We see a single telemetry event ("noResponse"), with any standard shape + expect(recordSpy).toHaveBeenCalledTimes(1) + expect(recordSpy).toHaveBeenCalledWith( + 'cody.autoedit', + 'noResponse', + expect.objectContaining({ + version: 0, + }) + ) + }) + + it('logs a rejection event after suggestion', () => { + // A valid chain: started -> contextLoaded -> loaded -> suggested -> rejected + const sessionId = autoeditLogger.createSession({ + languageId: 'typescript', + model: 'my-autoedit-model', + traceId: 'trace-rej', + }) + autoeditLogger.markAsContextLoaded({ + sessionId, + payload: { contextSummary: undefined }, + }) + autoeditLogger.markAsLoaded({ + sessionId, + modelOptions: defaultModelOptions, + payload: { + prediction: 'console.warn("reject test")', + source: 'network', + isFuzzyMatch: false, + responseHeaders: {}, + }, + }) + autoeditLogger.markAsSuggested(sessionId) + + // The user rejects + autoeditLogger.markAsRejected(sessionId) + + // The logger lumps final data into the single "suggested" event call. + expect(recordSpy).toHaveBeenCalledTimes(1) + expect(recordSpy).toHaveBeenNthCalledWith( + 1, + 'cody.autoedit', + 'suggested', + expect.objectContaining({ + version: 0, + }) + ) + }) + + it('handles invalid transitions by logging debug events (invalidTransitionToXYZ)', () => { + const sessionId = autoeditLogger.createSession({ + languageId: 'typescript', + model: 'my-autoedit-model', + traceId: 'trace-bad', + }) + + // Both calls below are invalid transitions, so the logger logs debug events + autoeditLogger.markAsSuggested(sessionId) + autoeditLogger.markAsRejected(sessionId) + + // "invalidTransitionTosuggested" and then "invalidTransitionTorejected" + expect(recordSpy).toHaveBeenCalledTimes(2) + expect(recordSpy).toHaveBeenNthCalledWith( + 1, + 'cody.autoedit', + 'invalidTransitionTosuggested', + undefined + ) + expect(recordSpy).toHaveBeenNthCalledWith( + 2, + 'cody.autoedit', + 'invalidTransitionTorejected', + undefined + ) + }) + + it('throttles repeated error logs, capturing the first occurrence immediately', () => { + // Force error logs to be reported: + vi.spyOn(sentryModule, 'shouldErrorBeReported').mockReturnValue(true) + + const error = new Error('Deliberate test error for autoedit') + autoeditLogger.logError(error) + + // First occurrence logs right away + expect(recordSpy).toHaveBeenCalledTimes(1) + expect(recordSpy).toHaveBeenCalledWith( + 'cody.autoedit', + 'error', + expect.objectContaining({ + version: 0, + metadata: { count: 1 }, + privateMetadata: expect.objectContaining({ + message: 'Deliberate test error for autoedit', + }), + }) + ) + + // Repeated calls should not log immediately + autoeditLogger.logError(error) + autoeditLogger.logError(error) + expect(recordSpy).toHaveBeenCalledTimes(1) + }) +}) diff --git a/vscode/src/autoedits/analytics-logger/analytics-logger.ts b/vscode/src/autoedits/analytics-logger/analytics-logger.ts new file mode 100644 index 000000000000..a2ab384a051a --- /dev/null +++ b/vscode/src/autoedits/analytics-logger/analytics-logger.ts @@ -0,0 +1,560 @@ +import { LRUCache } from 'lru-cache' +import * as uuid from 'uuid' +import * as vscode from 'vscode' + +import { + type BillingCategory, + type BillingProduct, + isNetworkError, + telemetryRecorder, +} from '@sourcegraph/cody-shared' +import type { TelemetryEventParameters } from '@sourcegraph/telemetry' + +import { getOtherCompletionProvider } from '../../completions/analytics-logger' +import type { ContextSummary } from '../../completions/context/context-mixer' +import { lines } from '../../completions/text-processing' +import { type CodeGenEventMetadata, charactersLogger } from '../../services/CharactersLogger' +import { upstreamHealthProvider } from '../../services/UpstreamHealthProvider' +import { captureException, shouldErrorBeReported } from '../../services/sentry/sentry' +import { splitSafeMetadata } from '../../services/telemetry-v2' + +import type { AutoeditModelOptions } from '../adapters/base' +import { AutoeditSuggestionIdRegistry } from './suggestion-id-registry' + +/** + * A stable ID that identifies a particular autoedit suggestion. If the same text + * and context recurs, we reuse this ID to avoid double-counting. + */ +export type AutoeditSuggestionID = string & { readonly _brand: 'AutoeditSuggestionID' } + +/** + * An ephemeral ID for a single “session” from creation to acceptance or rejection. + */ +type AutoeditSessionID = string & { readonly _brand: 'AutoeditSessionID' } + +/** + * Specialized string type for referencing error messages in our rate-limiting map. + */ +type AutoeditErrorMessage = string & { readonly _brand: 'AutoeditErrorMessage' } + +interface AutoeditStartedMetadata { + /** Document language ID (e.g., 'typescript'). */ + languageId: string + + /** Model used by Cody client to request the autosuggestion suggestion. */ + model: string + + /** Optional trace ID for cross-service correlation, if your environment provides it. */ + traceId: string + /** True if other autoedit/completion providers might also be active (e.g., Copilot). */ + + otherCompletionProviderEnabled: boolean + + /** The exact list of other providers that are active, if known. */ + otherCompletionProviders: string[] + + /** The round trip timings to reach the Sourcegraph and Cody Gateway instances. */ + upstreamLatency?: number + gatewayLatency?: number +} + +interface AutoeditContextLoadedMetadata extends AutoeditStartedMetadata { + /** + * Information about the context retrieval process that lead to this autocomplete request. Refer + * to the documentation of {@link ContextSummary} + */ + contextSummary?: ContextSummary +} + +interface AutoeditLoadedMetadata extends AutoeditContextLoadedMetadata { + /** + * An ID to uniquely identify a suggest completion. Note: It is possible for this ID to be part + * of two suggested events. This happens when the exact same completion text is shown again at + * the exact same location. We count this as the same completion and thus use the same ID. + */ + id: AutoeditSuggestionID + + /** Total lines in the suggestion. */ + lineCount: number + + /** Total characters in the suggestion. */ + charCount: number + + /** Prediction text snippet of the suggestion. */ + prediction?: string + + /** The source of the suggestion, e.g. 'network', 'cache', etc. */ + source?: string + + /** True if we fuzzy-matched this suggestion from a local or remote cache. */ + isFuzzyMatch?: boolean + + /** Optional set of relevant response headers (e.g. from Cody Gateway). */ + responseHeaders?: Record + + /** Time (ms) to generate or load the suggestion after it was started. */ + latency: number +} + +interface AutoEditFinalMetadata extends AutoeditLoadedMetadata { + /** Displayed to the user for this many milliseconds. */ + displayDuration: number + /** Whether the user actually had enough time to see the suggestion. (markAsRead) */ + isRead: boolean + /** True if the suggestion was explicitly/intentionally accepted */ + isAccepted: boolean + /** The number of completions we requested until this one was suggested. */ + suggestionsStartedSinceLastSuggestion: number +} + +interface AutoeditAcceptedEventPayload + extends AutoEditFinalMetadata, + Omit {} + +interface AutoeditRejectedEventPayload extends AutoEditFinalMetadata {} +interface AutoeditNoResponseEventPayload extends AutoeditContextLoadedMetadata {} + +/** + * Defines the possible phases of our autoedit session state machine. + */ +type Phase = + | 'started' + | 'contextLoaded' + | 'loaded' + | 'suggested' + | 'read' + | 'accepted' + | 'rejected' + | 'noResponse' + +/** + * Defines which phases can transition to which other phases. + */ +const validSessionTransitions = { + started: ['contextLoaded', 'noResponse'], + contextLoaded: ['loaded', 'noResponse'], + loaded: ['suggested'], + suggested: ['read', 'accepted', 'rejected'], + read: ['accepted', 'rejected'], + accepted: [], + rejected: [], + noResponse: [], +} as const satisfies Record + +/** + * The base fields common to all session states. We track ephemeral times and + * the partial payload. Once we reach a certain phase, we log the payload as a telemetry event. + */ +interface AutoeditBaseState { + sessionId: AutoeditSessionID + /** Current phase of the autoedit session */ + phase: Phase +} + +interface StartedState extends AutoeditBaseState { + phase: 'started' + /** Time (ms) when we started computing or requesting the suggestion. */ + startedAt: number + /** Partial payload for this phase. Will be augmented with more info as we progress. */ + payload: AutoeditStartedMetadata +} + +interface ContextLoadedState extends Omit { + phase: 'contextLoaded' + payload: AutoeditContextLoadedMetadata +} + +interface LoadedState extends Omit { + phase: 'loaded' + /** Timestamp when the suggestion completed generation/loading. */ + loadedAt: number + payload: AutoeditLoadedMetadata +} + +interface SuggestedState extends Omit { + phase: 'suggested' + /** Timestamp when the suggestion was first shown to the user. */ + suggestedAt: number +} + +interface ReadState extends Omit { + phase: 'read' + /** Timestamp when the suggestion can be considered as read by a user. */ + readAt: number +} + +interface AcceptedState extends Omit { + phase: 'accepted' + /** Timestamp when the user accepted the suggestion. */ + acceptedAt: number + /** Timestamp when the suggestion was logged to our analytics backend. This is to avoid double-logging. */ + suggestionLoggedAt?: number + payload: AutoeditAcceptedEventPayload +} + +interface RejectedState extends Omit { + phase: 'rejected' + /** Timestamp when the suggestion was logged to our analytics backend. This is to avoid double-logging. */ + suggestionLoggedAt?: number + payload: AutoeditRejectedEventPayload +} + +interface NoResponseState extends Omit { + phase: 'noResponse' + /** Timestamp when the suggestion was logged to our analytics backend. This is to avoid double-logging. */ + suggestionLoggedAt?: number + payload: AutoeditNoResponseEventPayload +} + +interface PhaseStates { + started: StartedState + contextLoaded: ContextLoadedState + loaded: LoadedState + suggested: SuggestedState + read: ReadState + accepted: AcceptedState + rejected: RejectedState + noResponse: NoResponseState +} + +/** + * Using the validTransitions definition, we can derive which "from phases" lead to a given next phase, + * and map that to the correct PhaseStates[fromPhase]. + */ +type PreviousPossiblePhaseFrom = { + [F in Phase]: T extends (typeof validSessionTransitions)[F][number] ? PhaseStates[F] : never +}[Phase] + +type AutoeditSessionState = PhaseStates[Phase] + +type AutoeditEventAction = + | 'suggested' + | 'accepted' + | 'noResponse' + | 'error' + | `invalidTransitionTo${Phase}` + +export class AutoeditAnalyticsLogger { + /** + * Stores ephemeral AutoeditSessionState for each session ID. + */ + private activeSessions = new LRUCache({ max: 20 }) + + /** + * Encapsulates the logic for reusing stable suggestion IDs for repeated text/context. + */ + private suggestionIdRegistry = new AutoeditSuggestionIdRegistry() + + /** + * Tracks repeated errors via their message key to avoid spamming logs. + */ + private errorCounts = new Map() + private autoeditsStartedSinceLastSuggestion = 0 + private ERROR_THROTTLE_INTERVAL_MS = 10 * 60 * 1000 // 10 minutes + + /** + * Creates a new ephemeral session with initial metadata. At this stage, we do not have the prediction yet. + */ + public createSession( + payload: Pick + ): AutoeditSessionID { + const sessionId = uuid.v4() as AutoeditSessionID + const otherCompletionProviders = getOtherCompletionProvider() + + const session: StartedState = { + sessionId, + phase: 'started', + startedAt: getTimeNowInMillis(), + payload: { + otherCompletionProviderEnabled: otherCompletionProviders.length > 0, + otherCompletionProviders, + upstreamLatency: upstreamHealthProvider.getUpstreamLatency(), + gatewayLatency: upstreamHealthProvider.getGatewayLatency(), + ...payload, + }, + } + + this.activeSessions.set(sessionId, session) + this.autoeditsStartedSinceLastSuggestion++ + return sessionId + } + + public markAsContextLoaded({ + sessionId, + payload, + }: { + sessionId: AutoeditSessionID + payload: Pick + }): void { + this.tryTransitionTo(sessionId, 'contextLoaded', session => ({ + ...session, + payload: { + ...session.payload, + contextSummary: payload.contextSummary, + }, + })) + } + + /** + * Mark when the suggestion finished generating/loading. This is also where + * we finally receive the prediction text, create a stable suggestion ID, + * and store the full suggestion metadata in ephemeral state. + */ + public markAsLoaded({ + sessionId, + modelOptions, + payload, + }: { + sessionId: AutoeditSessionID + modelOptions: AutoeditModelOptions + payload: Required< + Pick + > + }): void { + const { prediction, source, isFuzzyMatch, responseHeaders } = payload + const stableId = this.suggestionIdRegistry.getOrCreate(modelOptions, prediction) + const loadedAt = getTimeNowInMillis() + + this.tryTransitionTo(sessionId, 'loaded', session => ({ + ...session, + loadedAt, + payload: { + ...session.payload, + id: stableId, + lineCount: lines(prediction).length, + charCount: prediction.length, + prediction: prediction.length < 300 ? prediction : undefined, + source, + isFuzzyMatch, + responseHeaders, + latency: loadedAt - session.startedAt, + }, + })) + } + + public markAsSuggested(sessionId: AutoeditSessionID): SuggestedState | null { + const result = this.tryTransitionTo(sessionId, 'suggested', currentSession => ({ + ...currentSession, + suggestedAt: getTimeNowInMillis(), + })) + + if (!result) { + return null + } + + return result.updatedSession + } + + public markAsRead(sessionId: AutoeditSessionID): void { + this.tryTransitionTo(sessionId, 'read', currentSession => ({ + ...currentSession, + readAt: getTimeNowInMillis(), + })) + } + + public markAsAccepted({ + sessionId, + trackedRange, + position, + document, + prediction, + }: { + sessionId: AutoeditSessionID + trackedRange?: vscode.Range + position: vscode.Position + document: vscode.TextDocument + prediction: string + }): void { + const rangeForCharacterMetadata = trackedRange || new vscode.Range(position, position) + const { charsDeleted, charsInserted, ...charactersLoggerMetadata } = + charactersLogger.getChangeEventMetadataForCodyCodeGenEvents({ + document, + contentChanges: [ + { + range: rangeForCharacterMetadata, + rangeOffset: document.offsetAt(rangeForCharacterMetadata.start), + rangeLength: 0, + text: prediction, + }, + ], + reason: undefined, + }) + + const acceptedAt = getTimeNowInMillis() + + const result = this.tryTransitionTo(sessionId, 'accepted', session => { + // Ensure the AutoeditSuggestionID is never reused by removing it from the suggestion id registry + this.suggestionIdRegistry.deleteEntryIfValueExists(session.payload.id) + + return { + ...session, + acceptedAt, + payload: { + ...session.payload, + ...charactersLoggerMetadata, + isRead: true, + isAccepted: true, + displayDuration: acceptedAt - session.suggestedAt, + suggestionsStartedSinceLastSuggestion: this.autoeditsStartedSinceLastSuggestion, + }, + } + }) + + if (result?.updatedSession) { + this.writeAutoeditSessionEvent('suggested', result.updatedSession) + this.writeAutoeditSessionEvent('accepted', result.updatedSession) + this.activeSessions.delete(result.updatedSession.sessionId) + } + } + + public markAsRejected(sessionId: AutoeditSessionID): void { + const result = this.tryTransitionTo(sessionId, 'rejected', session => ({ + ...session, + payload: { + ...session.payload, + isRead: session.phase === 'read', + isAccepted: false, + displayDuration: getTimeNowInMillis() - session.suggestedAt, + suggestionsStartedSinceLastSuggestion: this.autoeditsStartedSinceLastSuggestion, + }, + })) + + if (result?.updatedSession) { + this.writeAutoeditSessionEvent('suggested', result.updatedSession) + + // Suggestions are kept in the LRU cache for longer. This is because they + // can still become visible if e.g. they are served from the cache and we + // need to retain the ability to mark them as seen. + } + } + + /** + * If the suggestion was never provided at all (“noResponse”), treat as a specialized reject. + */ + public markAsNoResponse(sessionId: AutoeditSessionID): void { + const result = this.tryTransitionTo(sessionId, 'noResponse', currentSession => currentSession) + + if (result?.updatedSession) { + this.writeAutoeditSessionEvent('noResponse', result.updatedSession) + this.activeSessions.delete(result.updatedSession.sessionId) + } + } + + private tryTransitionTo

( + sessionId: AutoeditSessionID, + nextPhase: P, + patch: (currentSession: PreviousPossiblePhaseFrom

) => Omit + ): { currentSession: PreviousPossiblePhaseFrom

; updatedSession: PhaseStates[P] } | null { + const currentSession = this.getSessionIfReadyForNextPhase(sessionId, nextPhase) + + if (!currentSession) { + return null + } + + const updatedSession = { ...currentSession, ...patch, phase: nextPhase } as PhaseStates[P] + this.activeSessions.set(sessionId, updatedSession) + + return { updatedSession, currentSession } + } + + /** + * Retrieves the session if it is in a phase that can transition to nextPhase, + * returning null if not found or if the transition is invalid. Uses the derived + * PreviousPossiblePhaseFrom type so that the returned State has the correct fields. + */ + private getSessionIfReadyForNextPhase( + sessionId: AutoeditSessionID, + nextPhase: T + ): PreviousPossiblePhaseFrom | null { + const session = this.activeSessions.get(sessionId) + + if ( + !session || + !(validSessionTransitions[session.phase] as readonly Phase[]).includes(nextPhase) + ) { + this.writeDebugBookkeepingEvent(`invalidTransitionTo${nextPhase}`) + return null + } + + return session as PreviousPossiblePhaseFrom + } + + private writeAutoeditSessionEvent( + action: AutoeditEventAction, + state: AcceptedState | RejectedState | NoResponseState + ): void { + if (state.suggestionLoggedAt) { + return + } + + // Update the session state to mark the suggestion as logged. + state.suggestionLoggedAt = getTimeNowInMillis() + + const { metadata, privateMetadata } = splitSafeMetadata(state.payload) + this.writeAutoeditEvent(action, { + version: 0, + metadata, + privateMetadata, + billingMetadata: { + product: 'cody', + category: state.phase === 'accepted' ? 'core' : 'billable', + }, + }) + + // Reset the number of the auto-edits started since the last suggestion. + this.autoeditsStartedSinceLastSuggestion = 0 + } + + private writeAutoeditEvent( + action: AutoeditEventAction, + params?: TelemetryEventParameters<{ [key: string]: number }, BillingProduct, BillingCategory> + ): void { + telemetryRecorder.recordEvent('cody.autoedit', action, params) + } + + /** + * Rate-limited error logging, capturing exceptions with Sentry and grouping repeated logs. + */ + public logError(error: Error): void { + if (!shouldErrorBeReported(error, false)) { + return + } + captureException(error) + + const messageKey = error.message as AutoeditErrorMessage + const traceId = isNetworkError(error) ? error.traceId : undefined + + const currentCount = this.errorCounts.get(messageKey) ?? 0 + if (currentCount === 0) { + this.writeAutoeditEvent('error', { + version: 0, + metadata: { count: 1 }, + privateMetadata: { message: error.message, traceId }, + }) + + // After the interval, flush repeated errors + setTimeout(() => { + const finalCount = this.errorCounts.get(messageKey) ?? 0 + if (finalCount > 0) { + this.writeAutoeditEvent('error', { + version: 0, + metadata: { count: finalCount }, + privateMetadata: { message: error.message, traceId }, + }) + } + this.errorCounts.set(messageKey, 0) + }, this.ERROR_THROTTLE_INTERVAL_MS) + } + this.errorCounts.set(messageKey, currentCount + 1) + } + + private writeDebugBookkeepingEvent(action: `invalidTransitionTo${Phase}`): void { + this.writeAutoeditEvent(action) + } +} + +export const autoeditAnalyticsLogger = new AutoeditAnalyticsLogger() + +function getTimeNowInMillis(): number { + return Math.floor(performance.now()) +} diff --git a/vscode/src/autoedits/analytics-logger/index.ts b/vscode/src/autoedits/analytics-logger/index.ts new file mode 100644 index 000000000000..6ae02d5165b1 --- /dev/null +++ b/vscode/src/autoedits/analytics-logger/index.ts @@ -0,0 +1 @@ +export * from './analytics-logger' diff --git a/vscode/src/autoedits/analytics-logger/suggestion-id-registry.ts b/vscode/src/autoedits/analytics-logger/suggestion-id-registry.ts new file mode 100644 index 000000000000..503cac896800 --- /dev/null +++ b/vscode/src/autoedits/analytics-logger/suggestion-id-registry.ts @@ -0,0 +1,57 @@ +import { LRUCache } from 'lru-cache' +import * as uuid from 'uuid' + +import type { AutoeditModelOptions } from '../adapters/base' +import type { AutoeditSuggestionID } from './analytics-logger' + +/** + * A specialized string type for the stable “suggestion key” in caches. + */ +export type AutoeditSuggestionKey = string & { readonly _brand: 'AutoeditSuggestionKey' } + +/** + * A small helper class that generates or retrieves stable AutoeditSuggestionIDs. + * This encapsulates the logic of deduplicating identical suggestions. + */ +export class AutoeditSuggestionIdRegistry { + private suggestionIdCache = new LRUCache({ max: 50 }) + + /** + * Produce a stable suggestion ID for the given context + text, reusing + * previously generated IDs for duplicates. + */ + public getOrCreate(options: AutoeditModelOptions, prediction: string): AutoeditSuggestionID { + const key = this.getAutoeditSuggestionKey(options, prediction) + let stableId = this.suggestionIdCache.get(key) + if (!stableId) { + stableId = uuid.v4() as AutoeditSuggestionID + this.suggestionIdCache.set(key, stableId) + } + return stableId + } + + /** + * Creates a stable string key that identifies the same suggestion across repeated displays. + */ + private getAutoeditSuggestionKey( + params: AutoeditModelOptions, + prediction: string + ): AutoeditSuggestionKey { + const key = `${params.prompt.systemMessage}█${params.prompt.userMessage}█${prediction}` + return key as AutoeditSuggestionKey + } + + public deleteEntryIfValueExists(id: AutoeditSuggestionID): void { + let matchingKey: string | null = null + + this.suggestionIdCache.forEach((value, key) => { + if (value === id) { + matchingKey = key + } + }) + + if (matchingKey) { + this.suggestionIdCache.delete(matchingKey) + } + } +} diff --git a/vscode/src/completions/analytics-logger.ts b/vscode/src/completions/analytics-logger.ts index c96925210d4d..c4b2ed7d341d 100644 --- a/vscode/src/completions/analytics-logger.ts +++ b/vscode/src/completions/analytics-logger.ts @@ -1279,7 +1279,7 @@ const otherCompletionProviders = [ 'TabNine.tabnine-vscode', 'Venthe.fauxpilot', ] -function getOtherCompletionProvider(): string[] { +export function getOtherCompletionProvider(): string[] { return otherCompletionProviders.filter(id => vscode.extensions.getExtension(id)?.isActive) }