Skip to content

Commit

Permalink
autoedits e2e tests (#6425)
Browse files Browse the repository at this point in the history
## Context
1. End to end testing setup for the `auto-edits`
2. https://playwright.dev/docs/test-snapshots is used to detect the
differences in the rendering for the auto-edit diff view
3. Closes
https://linear.app/sourcegraph/issue/CODY-4570/implement-a-manual-trigger-command

## Test plan
Added unit test and e2e tests
  • Loading branch information
hitesh-1997 authored Dec 26, 2024
1 parent 38d0921 commit b06f903
Show file tree
Hide file tree
Showing 15 changed files with 937 additions and 112 deletions.
4 changes: 3 additions & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -176,7 +176,9 @@ jobs:
if: ${{ failure() }}
with:
name: playwright-recordings ${{ matrix.runner }}
path: playwright/
path: |
playwright/
vscode/test-results/
build:
runs-on: ubuntu-latest
Expand Down
10 changes: 5 additions & 5 deletions vscode/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -597,6 +597,11 @@
"command": "cody.command.insertCodeToNewFile",
"title": "Save Code to New File...",
"enablement": "cody.activated"
},
{
"command": "cody.command.autoedits-manual-trigger",
"title": "Autoedits Manual Trigger",
"enablement": "cody.activated && config.cody.experimental.autoedits.enabled"
}
],
"keybindings": [
Expand Down Expand Up @@ -759,11 +764,6 @@
"key": "escape",
"when": "editorTextFocus && cody.activated && cody.supersuggest.active"
},
{
"command": "cody.experimental.suggest",
"key": "ctrl+shift+enter",
"when": "cody.activated && config.cody.experimental.autoedits.enabled"
},
{
"command": "cody.supersuggest.testExample",
"key": "ctrl+alt+enter",
Expand Down
39 changes: 25 additions & 14 deletions vscode/src/autoedits/autoedits-provider.ts
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import { FilterPredictionBasedOnRecentEdits } from './filter-prediction-edits'
import { autoeditsLogger } from './logger'
import type { CodeToReplaceData } from './prompt/prompt-utils'
import { ShortTermPromptStrategy } from './prompt/short-term-diff-prompt-strategy'
import type { DecorationInfo } from './renderer/decorators/base'
import { DefaultDecorator } from './renderer/decorators/default-decorator'
import { InlineDiffDecorator } from './renderer/decorators/inline-diff-decorator'
import { getDecorationInfo } from './renderer/diff-utils'
Expand All @@ -29,7 +30,7 @@ import { AutoEditsDefaultRendererManager, type AutoEditsRendererManager } from '
import {
extractAutoEditResponseFromCurrentDocumentCommentTemplate,
shrinkReplacerTextToCodeToReplaceRange,
} from './renderer/renderer-testing'
} from './renderer/mock-renderer'
import { shrinkPredictionUntilSuffix } from './shrink-prediction'
import { isPredictedTextAlreadyInSuffix } from './utils'

Expand Down Expand Up @@ -186,14 +187,7 @@ export class AutoeditsProvider implements vscode.InlineCompletionItemProvider, v
)
return null
}

const currentFileText = document.getText()
const predictedFileText =
currentFileText.slice(0, document.offsetAt(codeToReplaceData.range.start)) +
prediction +
currentFileText.slice(document.offsetAt(codeToReplaceData.range.end))

const decorationInfo = getDecorationInfo(currentFileText, predictedFileText)
const decorationInfo = getDecorationInfoFromPrediction(document, prediction, codeToReplaceData)

if (
isPredictedTextAlreadyInSuffix({
Expand All @@ -210,15 +204,14 @@ export class AutoeditsProvider implements vscode.InlineCompletionItemProvider, v
}

const { inlineCompletions } =
await this.rendererManager.maybeRenderDecorationsAndTryMakeInlineCompletionResponse(
await this.rendererManager.maybeRenderDecorationsAndTryMakeInlineCompletionResponse({
prediction,
codeToReplaceData,
document,
position,
docContext,
decorationInfo
)

decorationInfo,
})
return inlineCompletions
}

Expand All @@ -244,7 +237,10 @@ export class AutoeditsProvider implements vscode.InlineCompletionItemProvider, v

let response: string | undefined = undefined
if (autoeditsProviderConfig.isMockResponseFromCurrentDocumentTemplateEnabled) {
const responseMetadata = extractAutoEditResponseFromCurrentDocumentCommentTemplate()
const responseMetadata = extractAutoEditResponseFromCurrentDocumentCommentTemplate(
document,
position
)

if (responseMetadata) {
response = shrinkReplacerTextToCodeToReplaceRange(responseMetadata, codeToReplace)
Expand Down Expand Up @@ -289,3 +285,18 @@ export class AutoeditsProvider implements vscode.InlineCompletionItemProvider, v
}
}
}

export function getDecorationInfoFromPrediction(
document: vscode.TextDocument,
prediction: string,
codeToReplaceData: CodeToReplaceData
): DecorationInfo {
const currentFileText = document.getText()
const predictedFileText =
currentFileText.slice(0, document.offsetAt(codeToReplaceData.range.start)) +
prediction +
currentFileText.slice(document.offsetAt(codeToReplaceData.range.end))

const decorationInfo = getDecorationInfo(currentFileText, predictedFileText)
return decorationInfo
}
37 changes: 37 additions & 0 deletions vscode/src/autoedits/prompt/test-helper.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
import dedent from 'dedent'
import { getCurrentDocContext } from '../../completions/get-current-doc-context'
import { documentAndPosition } from '../../completions/test-helpers'
import { type CodeToReplaceData, getCurrentFilePromptComponents } from '../prompt/prompt-utils'

interface CodeToReplaceTestOptions {
maxPrefixLength: number
maxSuffixLength: number
maxPrefixLinesInArea: number
maxSuffixLinesInArea: number
codeToRewritePrefixLines: number
codeToRewriteSuffixLines: number
}

export function createCodeToReplaceDataForTest(
code: TemplateStringsArray,
options: CodeToReplaceTestOptions,
...values: unknown[]
): CodeToReplaceData {
const { document, position } = documentAndPosition(dedent(code, values))
const docContext = getCurrentDocContext({
document,
position,
maxPrefixLength: options.maxPrefixLength,
maxSuffixLength: options.maxSuffixLength,
})

return getCurrentFilePromptComponents({
docContext,
position,
document,
maxPrefixLinesInArea: options.maxPrefixLinesInArea,
maxSuffixLinesInArea: options.maxSuffixLinesInArea,
codeToRewritePrefixLines: options.codeToRewritePrefixLines,
codeToRewriteSuffixLines: options.codeToRewriteSuffixLines,
}).codeToReplace
}
22 changes: 10 additions & 12 deletions vscode/src/autoedits/renderer/inline-manager.ts
Original file line number Diff line number Diff line change
@@ -1,11 +1,9 @@
import { isFileURI } from '@sourcegraph/cody-shared'
import * as vscode from 'vscode'

import { type DocumentContext, isFileURI } from '@sourcegraph/cody-shared'

import { completionMatchesSuffix } from '../../completions/is-completion-visible'
import { getNewLineChar } from '../../completions/text-processing'
import { autoeditsLogger } from '../logger'
import type { CodeToReplaceData } from '../prompt/prompt-utils'
import type { AutoeditRendererManagerArgs } from './manager'

import type {
AddedLineInfo,
Expand Down Expand Up @@ -53,14 +51,14 @@ export class AutoEditsInlineRendererManager
}
}

async maybeRenderDecorationsAndTryMakeInlineCompletionResponse(
prediction: string,
codeToReplaceData: CodeToReplaceData,
document: vscode.TextDocument,
position: vscode.Position,
docContext: DocumentContext,
decorationInfo: DecorationInfo
): Promise<{
async maybeRenderDecorationsAndTryMakeInlineCompletionResponse({
prediction,
codeToReplaceData,
document,
position,
docContext,
decorationInfo,
}: AutoeditRendererManagerArgs): Promise<{
inlineCompletions: vscode.InlineCompletionItem[] | null
updatedDecorationInfo: DecorationInfo
}> {
Expand Down
151 changes: 151 additions & 0 deletions vscode/src/autoedits/renderer/manager.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,151 @@
import dedent from 'dedent'
import { beforeEach, describe, expect, it } from 'vitest'
import type * as vscode from 'vscode'
import { getCurrentDocContext } from '../../completions/get-current-doc-context'
import { documentAndPosition } from '../../completions/test-helpers'
import { getDecorationInfoFromPrediction } from '../autoedits-provider'
import type { CodeToReplaceData } from '../prompt/prompt-utils'
import { createCodeToReplaceDataForTest } from '../prompt/test-helper'
import { AutoEditsDefaultRendererManager } from '../renderer/manager'
import { DefaultDecorator } from './decorators/default-decorator'
import type { AutoeditRendererManagerArgs } from './manager'

function getCodeToReplaceForManager(
code: TemplateStringsArray,
...values: unknown[]
): CodeToReplaceData {
return createCodeToReplaceDataForTest(
code,
{
maxPrefixLength: 100,
maxSuffixLength: 100,
maxPrefixLinesInArea: 2,
maxSuffixLinesInArea: 2,
codeToRewritePrefixLines: 1,
codeToRewriteSuffixLines: 1,
},
...values
)
}

describe('AutoEditsDefaultRendererManager', () => {
const getAutoeditRendererManagerArgs = (
documentText: string,
prediction: string
): AutoeditRendererManagerArgs => {
const { document, position } = documentAndPosition(documentText)
const docContext = getCurrentDocContext({
document,
position,
maxPrefixLength: 100,
maxSuffixLength: 100,
})
const codeToReplaceData = getCodeToReplaceForManager`${documentText}`
const decorationInfo = getDecorationInfoFromPrediction(document, prediction, codeToReplaceData)
return {
prediction,
codeToReplaceData,
document,
position,
docContext,
decorationInfo,
}
}

describe('maybeRenderDecorationsAndTryMakeInlineCompletionResponse', () => {
let manager: AutoEditsDefaultRendererManager

beforeEach(() => {
manager = new AutoEditsDefaultRendererManager(
(editor: vscode.TextEditor) => new DefaultDecorator(editor)
)
})

const assertInlineCompletionItems = (
items: vscode.InlineCompletionItem[],
expectedCompletion: string
) => {
expect(items).toHaveLength(1)
expect(items[0].insertText).toEqual(expectedCompletion)
}

it('should return single line inline completion when possible', async () => {
const documentText = dedent`const a = 1
const b = 2
const c = 3
console█
function greet() { console.log("Hello") }
const x = 10
console.log(x)
console.log("end")
`
const prediction = dedent`const c = 3
console.log(a, b, c)
function greet() { console.log("Hello") }
`
const args = getAutoeditRendererManagerArgs(documentText, prediction)
const result = await manager.maybeRenderDecorationsAndTryMakeInlineCompletionResponse(args)
expect(result).toBeDefined()
assertInlineCompletionItems(
result.inlineCompletions!,
dedent`
console.log(a, b, c)
`
)
})

it('should return multi line inline completion when possible', async () => {
const documentText = dedent`const a = 1
const b = 2
const c = 3
console█
function greet() { console.log("Hello") }
const x = 10
console.log(x)
console.log("end")
`
const prediction = dedent`const c = 3
console.log(a, b, c)
const d = 10
const e = 20
function greet() { console.log("Hello") }
`
const args = getAutoeditRendererManagerArgs(documentText, prediction)
const result = await manager.maybeRenderDecorationsAndTryMakeInlineCompletionResponse(args)
expect(result).toBeDefined()
assertInlineCompletionItems(
result.inlineCompletions!,
dedent`
console.log(a, b, c)
const d = 10
const e = 20
`
)
})

it('should return single line inline completion when the suffix is present on same line', async () => {
const documentText = dedent`const a = 1
const b = 2
const c = 3
console█c)
function greet() { console.log("Hello") }
const x = 10
console.log(x)
console.log("end")
`
const prediction = dedent`const c = 3
console.log(a, b, c)
function greet() { console.log("Hello") }
`
const args = getAutoeditRendererManagerArgs(documentText, prediction)
const result = await manager.maybeRenderDecorationsAndTryMakeInlineCompletionResponse(args)
expect(result).toBeDefined()
assertInlineCompletionItems(
result.inlineCompletions!,
dedent`
console.log(a, b, c)
`
)
})
})
})
Loading

0 comments on commit b06f903

Please sign in to comment.