Skip to content

Commit

Permalink
test(amazonq): generateZipTestGen is unreliable (#6290)
Browse files Browse the repository at this point in the history
## Problem

Flaky test `generateZipTestGen` sometimes fails with
```
  1) zipUtil
       generateZipTestGen
         Should generate zip for test generation successfully:
     Error: done() called multiple times in test <zipUtil generateZipTestGen Should generate zip for test generation successfully> of file /Users/runner/work/aws-toolkit-vscode/aws-toolkit-vscode/packages/amazonq/dist/test/unit/codewhisperer/util/zipUtil.test.js; in addition, done() received error: EntryNotFound (FileSystemError): Error: ENOENT: no such file or directory, scandir '/test/zip/utgRequiredArtifactsDir'
    at Function.e (/private/tmp/.vscode-test/vscode-darwin-arm64-1.83.0/Visual Studio Code.app/Contents/Resources/app/out/vs/workbench/api/node/extensionHostProcess.js:127:26740)
    at Object.readDirectory (/private/tmp/.vscode-test/vscode-darwin-arm64-1.83.0/Visual Studio Code.app/Contents/Resources/app/out/vs/workbench/api/node/extensionHostProcess.js:127:24744)
    at async processDirectory (/Users/runner/work/aws-toolkit-vscode/aws-toolkit-vscode/packages/core/src/codewhisperer/util/zipUtil.ts:190:29)
    at async ZipUtil.processMetadataDir (/Users/runner/work/aws-toolkit-vscode/aws-toolkit-vscode/packages/core/src/codewhisperer/util/zipUtil.ts:212:9)
    at async ZipUtil.zipProject (/Users/runner/work/aws-toolkit-vscode/aws-toolkit-vscode/packages/core/src/codewhisperer/util/zipUtil.ts:230:13)
    at async ZipUtil.generateZipTestGen (/Users/runner/work/aws-toolkit-vscode/aws-toolkit-vscode/packages/core/src/codewhisperer/util/zipUtil.ts:602:41)
    at async Context.<anonymous> (/Users/runner/work/aws-toolkit-vscode/aws-toolkit-vscode/packages/amazonq/test/unit/codewhisperer/util/zipUtil.test.ts:159:28) {
  code: 'FileNotFound'
```

Issue: #6160


## Solution

Based on the error, the test is trying to readdir on a mock dirpath even
though the entire `zipProject` method should be stubbed. This can be
confirmed by commenting out the following line:


https://github.com/aws/aws-toolkit-vscode/blob/aa332da854c9d17ba258b459c622500911b1b0d9/packages/amazonq/test/unit/codewhisperer/util/zipUtil.test.ts#L135

which will give the same error

```
Error: ENOENT: no such file or directory, scandir '/test/zip/utgRequiredArtifactsDir'
```

So instead of stubbing which seems to be unreliable, we can let the test
actually create the zip file.

---

- Treat all work as PUBLIC. Private `feature/x` branches will not be
squash-merged at release time.
- Your code changes must meet the guidelines in
[CONTRIBUTING.md](https://github.com/aws/aws-toolkit-vscode/blob/master/CONTRIBUTING.md#guidelines).
- License: I confirm that my contribution is made under the terms of the
Apache 2.0 license.
  • Loading branch information
ctlai95 authored Jan 6, 2025
1 parent 3d90772 commit e9e2965
Showing 1 changed file with 34 additions and 89 deletions.
123 changes: 34 additions & 89 deletions packages/amazonq/test/unit/codewhisperer/util/zipUtil.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@ import vscode from 'vscode'
import sinon from 'sinon'
import { join } from 'path'
import { getTestWorkspaceFolder } from 'aws-core-vscode/test'
import { CodeAnalysisScope, ZipUtil } from 'aws-core-vscode/codewhisperer'
import { CodeAnalysisScope, CodeWhispererConstants, ZipUtil } from 'aws-core-vscode/codewhisperer'
import { codeScanTruncDirPrefix } from 'aws-core-vscode/codewhisperer'
import { ToolkitError } from 'aws-core-vscode/shared'
import { tempDirPath, ToolkitError } from 'aws-core-vscode/shared'
import { LspClient } from 'aws-core-vscode/amazonq'
import { fs } from 'aws-core-vscode/shared'
import path from 'path'
Expand Down Expand Up @@ -142,97 +142,58 @@ describe('zipUtil', function () {

describe('generateZipTestGen', function () {
let zipUtil: ZipUtil
let mockFs: sinon.SinonStubbedInstance<typeof fs>
const projectPath = '/test/project'
const zipDirPath = '/test/zip'
const zipFilePath = '/test/zip/test.zip'
let getZipDirPathStub: sinon.SinonStub
let testTempDirPath: string

beforeEach(function () {
zipUtil = new ZipUtil()
mockFs = sinon.stub(fs)

const mockRepoMapPath = '/path/to/repoMapData.json'
mockFs.exists.withArgs(mockRepoMapPath).resolves(true)
sinon.stub(LspClient, 'instance').get(() => ({
getRepoMapJSON: sinon.stub().resolves(mockRepoMapPath),
}))

sinon.stub(zipUtil, 'getZipDirPath').returns(zipDirPath)
sinon.stub(zipUtil as any, 'zipProject').resolves(zipFilePath)
testTempDirPath = path.join(tempDirPath, CodeWhispererConstants.TestGenerationTruncDirPrefix)
getZipDirPathStub = sinon.stub(zipUtil, 'getZipDirPath')
getZipDirPathStub.callsFake(() => testTempDirPath)
})

afterEach(function () {
sinon.restore()
})

it('Should generate zip for test generation successfully', async function () {
mockFs.stat.resolves({
type: vscode.FileType.File,
size: 1000,
ctime: Date.now(),
mtime: Date.now(),
} as vscode.FileStat)

mockFs.readFileBytes.resolves(Buffer.from('test content'))
it('should generate zip for test generation successfully', async function () {
const mkdirSpy = sinon.spy(fs, 'mkdir')

// Fix: Create a Set from the array
zipUtil['_totalSize'] = 500
zipUtil['_totalBuildSize'] = 200
zipUtil['_totalLines'] = 100
zipUtil['_language'] = 'typescript'
zipUtil['_pickedSourceFiles'] = new Set(['file1.ts', 'file2.ts'])
const result = await zipUtil.generateZipTestGen(appRoot, false)

const result = await zipUtil.generateZipTestGen(projectPath, false)

assert.ok(mockFs.mkdir.calledWith(path.join(zipDirPath, 'utgRequiredArtifactsDir')))
assert.ok(mkdirSpy.calledWith(path.join(testTempDirPath, 'utgRequiredArtifactsDir')))
assert.ok(
mockFs.mkdir.calledWith(path.join(zipDirPath, 'utgRequiredArtifactsDir', 'buildAndExecuteLogDir'))
mkdirSpy.calledWith(path.join(testTempDirPath, 'utgRequiredArtifactsDir', 'buildAndExecuteLogDir'))
)
assert.ok(mockFs.mkdir.calledWith(path.join(zipDirPath, 'utgRequiredArtifactsDir', 'repoMapData')))
assert.ok(mockFs.mkdir.calledWith(path.join(zipDirPath, 'utgRequiredArtifactsDir', 'testCoverageDir')))

// assert.ok(
// mockFs.copy.calledWith(
// '/path/to/repoMapData.json',
// path.join(zipDirPath, 'utgRequiredArtifactsDir', 'repoMapData', 'repoMapData.json')
// )
// )

assert.strictEqual(result.rootDir, zipDirPath)
assert.strictEqual(result.zipFilePath, zipFilePath)
assert.strictEqual(result.srcPayloadSizeInBytes, 500)
assert.strictEqual(result.buildPayloadSizeInBytes, 200)
assert.strictEqual(result.zipFileSizeInBytes, 1000)
assert.strictEqual(result.lines, 100)
assert.strictEqual(result.language, 'typescript')
assert.deepStrictEqual(Array.from(result.scannedFiles), ['file1.ts', 'file2.ts'])
})

// it('Should handle LSP client error', async function () {
// // Override the default stub with one that rejects
// sinon.stub(LspClient, 'instance').get(() => ({
// getRepoMapJSON: sinon.stub().rejects(new Error('LSP error')),
// }))
assert.ok(mkdirSpy.calledWith(path.join(testTempDirPath, 'utgRequiredArtifactsDir', 'repoMapData')))
assert.ok(mkdirSpy.calledWith(path.join(testTempDirPath, 'utgRequiredArtifactsDir', 'testCoverageDir')))

// await assert.rejects(() => zipUtil.generateZipTestGen(projectPath), /LSP error/)
// })
assert.strictEqual(result.rootDir, testTempDirPath)
assert.strictEqual(result.zipFilePath, testTempDirPath + CodeWhispererConstants.codeScanZipExt)
assert.ok(result.srcPayloadSizeInBytes > 0)
assert.strictEqual(result.buildPayloadSizeInBytes, 0)
assert.ok(result.zipFileSizeInBytes > 0)
assert.strictEqual(result.lines, 150)
assert.strictEqual(result.language, 'java')
assert.strictEqual(result.scannedFiles.size, 4)
})

it('Should handle file system errors during directory creation', async function () {
sinon.stub(LspClient, 'instance').get(() => ({
getRepoMapJSON: sinon.stub().resolves('{"mock": "data"}'),
}))
mockFs.mkdir.rejects(new Error('Directory creation failed'))
sinon.stub(fs, 'mkdir').rejects(new Error('Directory creation failed'))

await assert.rejects(() => zipUtil.generateZipTestGen(projectPath, false), /Directory creation failed/)
await assert.rejects(() => zipUtil.generateZipTestGen(appRoot, false), /Directory creation failed/)
})

it('Should handle zip project errors', async function () {
sinon.stub(LspClient, 'instance').get(() => ({
getRepoMapJSON: sinon.stub().resolves('{"mock": "data"}'),
}))
;(zipUtil as any).zipProject.rejects(new Error('Zip failed'))
sinon.stub(zipUtil, 'zipProject' as keyof ZipUtil).rejects(new Error('Zip failed'))

await assert.rejects(() => zipUtil.generateZipTestGen(projectPath, false), /Zip failed/)
await assert.rejects(() => zipUtil.generateZipTestGen(appRoot, false), /Zip failed/)
})

it('Should handle file copy to downloads folder error', async function () {
Expand All @@ -241,31 +202,15 @@ describe('zipUtil', function () {
getRepoMapJSON: sinon.stub().resolves('{"mock": "data"}'),
}))

// Mock file operations
const mockFs = {
mkdir: sinon.stub().resolves(),
copy: sinon.stub().rejects(new Error('Copy failed')),
exists: sinon.stub().resolves(true),
stat: sinon.stub().resolves({
type: vscode.FileType.File,
size: 1000,
ctime: Date.now(),
mtime: Date.now(),
} as vscode.FileStat),
}

// Since the function now uses Promise.all for directory creation and file operations,
// we need to ensure the mkdir succeeds but the copy fails
fs.mkdir = mockFs.mkdir
fs.copy = mockFs.copy
fs.exists = mockFs.exists
fs.stat = mockFs.stat

await assert.rejects(() => zipUtil.generateZipTestGen(projectPath, false), /Copy failed/)
const mkdirSpy = sinon.spy(fs, 'mkdir')
sinon.stub(fs, 'exists').resolves(true)
sinon.stub(fs, 'copy').rejects(new Error('Copy failed'))

await assert.rejects(() => zipUtil.generateZipTestGen(appRoot, false), /Copy failed/)

// Verify mkdir was called for all directories
assert(mockFs.mkdir.called, 'mkdir should have been called')
assert.strictEqual(mockFs.mkdir.callCount, 4, 'mkdir should have been called 4 times')
assert(mkdirSpy.called, 'mkdir should have been called')
assert.strictEqual(mkdirSpy.callCount, 4, 'mkdir should have been called 4 times')
})
})
})

0 comments on commit e9e2965

Please sign in to comment.