From 39dc730e95dc4b6e0d2ddc375d0644983f6d71a0 Mon Sep 17 00:00:00 2001 From: Alena Khineika Date: Mon, 18 Nov 2024 15:49:00 +0100 Subject: [PATCH 1/4] feat(participant): in-app notification to introduce users to copilot extension VSCODE-633 --- src/mdbExtensionController.ts | 51 +++++++ src/storage/storageController.ts | 2 + src/telemetry/telemetryService.ts | 19 ++- src/test/suite/mdbExtensionController.test.ts | 137 ++++++++++++++++++ 4 files changed, 208 insertions(+), 1 deletion(-) diff --git a/src/mdbExtensionController.ts b/src/mdbExtensionController.ts index 0f8e6bdf6..eccc7a460 100644 --- a/src/mdbExtensionController.ts +++ b/src/mdbExtensionController.ts @@ -165,6 +165,7 @@ export default class MDBExtensionController implements vscode.Disposable { this.registerCommands(); this.showOverviewPageIfRecentlyInstalled(); void this.showSurveyForEstablishedUsers(); + void this.showCopilotIntroductionForEstablishedUsers(); } registerCommands = (): void => { @@ -918,6 +919,56 @@ export default class MDBExtensionController implements vscode.Disposable { } } + async showCopilotIntroductionForEstablishedUsers(): Promise { + const hasBeenShownAlready = + this._storageController.get( + StorageVariables.GLOBAL_COPILOT_INTRODUCTION_SHOWN + ) === true; + + // Show the toast when it hasn't been show to the + // user yet, and they have saved connections + // -> they haven't just started using this extension. + if (hasBeenShownAlready || !this._connectionStorage.hasSavedConnections()) { + return; + } + + const copilot = vscode.extensions.getExtension('github.copilot-chat'); + + const action = 'Chat with @MongoDB'; + const text = + 'Generate queries, interact with documentation, and explore your database schema using the MongoDB Copilot extension. Give it a try!'; + const result = await vscode.window.showInformationMessage( + text, + {}, + { + title: action, + } + ); + if (result?.title === action) { + await vscode.commands.executeCommand('workbench.action.chat.newChat'); + await vscode.commands.executeCommand( + 'workbench.action.chat.clearHistory' + ); + await vscode.commands.executeCommand('workbench.action.chat.open', { + query: '@MongoDB', + isPartialQuery: true, + }); + this._telemetryService.trackCopilotIntroductionClicked({ + is_copilot_active: !!copilot?.isActive, + }); + } else { + this._telemetryService.trackCopilotIntroductionDismissed({ + is_copilot_active: !!copilot?.isActive, + }); + } + + // Whether action was taken or the prompt dismissed, we won't show this again. + void this._storageController.update( + StorageVariables.GLOBAL_COPILOT_INTRODUCTION_SHOWN, + true + ); + } + async showSurveyForEstablishedUsers(): Promise { const surveyId = '9viN9wcbsC3zvHyg7'; diff --git a/src/storage/storageController.ts b/src/storage/storageController.ts index e19eae149..9bce41701 100644 --- a/src/storage/storageController.ts +++ b/src/storage/storageController.ts @@ -7,6 +7,7 @@ export enum StorageVariables { // Only exists on globalState. GLOBAL_HAS_BEEN_SHOWN_INITIAL_VIEW = 'GLOBAL_HAS_BEEN_SHOWN_INITIAL_VIEW', GLOBAL_SURVEY_SHOWN = 'GLOBAL_SURVEY_SHOWN', + GLOBAL_COPILOT_INTRODUCTION_SHOWN = 'GLOBAL_COPILOT_INTRODUCTION_SHOWN', GLOBAL_SAVED_CONNECTIONS = 'GLOBAL_SAVED_CONNECTIONS', // Analytics user identify. GLOBAL_USER_ID = 'GLOBAL_USER_ID', @@ -53,6 +54,7 @@ interface StorageVariableContents { [StorageVariables.GLOBAL_ANONYMOUS_ID]: string; [StorageVariables.GLOBAL_HAS_BEEN_SHOWN_INITIAL_VIEW]: boolean; [StorageVariables.GLOBAL_SURVEY_SHOWN]: string; + [StorageVariables.GLOBAL_COPILOT_INTRODUCTION_SHOWN]: boolean; [StorageVariables.GLOBAL_SAVED_CONNECTIONS]: ConnectionsFromStorage; [StorageVariables.WORKSPACE_SAVED_CONNECTIONS]: ConnectionsFromStorage; [StorageVariables.COPILOT_HAS_BEEN_SHOWN_WELCOME_MESSAGE]: boolean; diff --git a/src/telemetry/telemetryService.ts b/src/telemetry/telemetryService.ts index 8a58258dc..46e2791e2 100644 --- a/src/telemetry/telemetryService.ts +++ b/src/telemetry/telemetryService.ts @@ -135,6 +135,10 @@ export type ParticipantResponseProperties = { output_length: number; }; +export type CopilotIntroductionProperties = { + is_copilot_active: boolean; +}; + export function chatResultFeedbackKindToTelemetryValue( kind: vscode.ChatResultFeedbackKind ): TelemetryFeedbackKind { @@ -167,7 +171,8 @@ type TelemetryEventProperties = | ParticipantFeedbackProperties | ParticipantResponseFailedProperties | ParticipantPromptProperties - | ParticipantResponseProperties; + | ParticipantResponseProperties + | CopilotIntroductionProperties; export enum TelemetryEventTypes { PLAYGROUND_CODE_EXECUTED = 'Playground Code Executed', @@ -192,6 +197,8 @@ export enum TelemetryEventTypes { PARTICIPANT_RESPONSE_FAILED = 'Participant Response Failed', PARTICIPANT_PROMPT_SUBMITTED = 'Participant Prompt Submitted', PARTICIPANT_RESPONSE_GENERATED = 'Participant Response Generated', + COPILOT_INTRODUCTION_CLICKED = 'Copilot Introduction Clicked', + COPILOT_INTRODUCTION_DISMISSED = 'Copilot Introduction Dismissed', } /** @@ -489,4 +496,14 @@ export default class TelemetryService { trackCopilotParticipantResponse(props: ParticipantResponseProperties): void { this.track(TelemetryEventTypes.PARTICIPANT_RESPONSE_GENERATED, props); } + + trackCopilotIntroductionClicked(props: CopilotIntroductionProperties): void { + this.track(TelemetryEventTypes.COPILOT_INTRODUCTION_CLICKED, props); + } + + trackCopilotIntroductionDismissed( + props: CopilotIntroductionProperties + ): void { + this.track(TelemetryEventTypes.COPILOT_INTRODUCTION_DISMISSED, props); + } } diff --git a/src/test/suite/mdbExtensionController.test.ts b/src/test/suite/mdbExtensionController.test.ts index 5e11b1cc3..eae0aee6d 100644 --- a/src/test/suite/mdbExtensionController.test.ts +++ b/src/test/suite/mdbExtensionController.test.ts @@ -1846,6 +1846,143 @@ suite('MDBExtensionController Test Suite', function () { }); }); }); + + suite('copilot introduction prompt', function () { + suite( + "when a user hasn't been shown the copilot introduction prompt yet, and they have connections saved", + () => { + [ + { + description: 'clicked the button', + value: { title: 'Chat with @MongoDB' }, + }, + { description: 'dismissed', value: undefined }, + ].forEach((reaction) => { + suite(`user ${reaction.description}`, () => { + let connectionsUpdateStub: SinonStub; + let executeCommandStub: SinonStub; + beforeEach(async () => { + showInformationMessageStub.resolves(reaction.value); + executeCommandStub = sandbox.stub( + vscode.commands, + 'executeCommand' + ); + sandbox.replace( + mdbTestExtension.testExtensionController._storageController, + 'get', + sandbox.fake.returns(undefined) + ); + sandbox.replace( + mdbTestExtension.testExtensionController._connectionStorage, + 'hasSavedConnections', + sandbox.fake.returns(true) + ); + connectionsUpdateStub = sandbox.stub( + mdbTestExtension.testExtensionController._storageController, + 'update' + ); + connectionsUpdateStub.resolves(undefined); + await mdbTestExtension.testExtensionController.showCopilotIntroductionForEstablishedUsers(); + }); + + afterEach(() => { + sandbox.restore(); + }); + + test('they are shown the copilot introduction prompt', () => { + assert(showInformationMessageStub.called); + assert.strictEqual( + showInformationMessageStub.firstCall.args[0], + 'Generate queries, interact with documentation, and explore your database schema using the MongoDB Copilot extension. Give it a try!' + ); + }); + + test('the link was open if and only if they click the button', () => { + if (reaction.value === undefined) { + assert(executeCommandStub.notCalled); + } + if (reaction.value) { + assert(executeCommandStub.called); + assert.strictEqual( + executeCommandStub.firstCall.args[0], + 'workbench.action.chat.newChat' + ); + } + }); + + test("it sets that they've been shown the copilot introduction", () => { + assert(connectionsUpdateStub.called); + assert.strictEqual( + connectionsUpdateStub.firstCall.args[0], + StorageVariables.GLOBAL_COPILOT_INTRODUCTION_SHOWN + ); + assert.strictEqual( + connectionsUpdateStub.firstCall.args[1], + true + ); + }); + }); + }); + } + ); + + suite( + 'when a user has been shown the copilot introduction prompt already', + () => { + let connectionsUpdateStub: SinonStub; + beforeEach(() => { + sandbox.replace( + mdbTestExtension.testExtensionController._storageController, + 'get', + sandbox.fake.returns(true) // copilot introduction has been shown + ); + sandbox.replace( + mdbTestExtension.testExtensionController._connectionStorage, + 'hasSavedConnections', + sandbox.fake.returns(true) + ); + connectionsUpdateStub = sandbox.stub( + mdbTestExtension.testExtensionController._storageController, + 'update' + ); + connectionsUpdateStub.resolves(undefined); + + void mdbTestExtension.testExtensionController.showCopilotIntroductionForEstablishedUsers(); + }); + + test('they are not shown the copilot introduction prompt', () => { + assert(showInformationMessageStub.notCalled); + }); + } + ); + + suite('when a has no connections saved', () => { + let connectionsUpdateStub: SinonStub; + beforeEach(() => { + sandbox.replace( + mdbTestExtension.testExtensionController._storageController, + 'get', + sandbox.fake.returns(undefined) + ); + sandbox.replace( + mdbTestExtension.testExtensionController._connectionStorage, + 'hasSavedConnections', + sandbox.fake.returns(false) // no connections yet - this might be the first install + ); + connectionsUpdateStub = sandbox.stub( + mdbTestExtension.testExtensionController._storageController, + 'update' + ); + connectionsUpdateStub.resolves(undefined); + + void mdbTestExtension.testExtensionController.showCopilotIntroductionForEstablishedUsers(); + }); + + test('they are not shown the copilot introduction prompt', () => { + assert(showInformationMessageStub.notCalled); + }); + }); + }); }); test('mdb.participantViewRawSchemaOutput command opens a json document with the output', async () => { From bd1ca848f80e9de8595732e80dc0a1ce5d7e67dc Mon Sep 17 00:00:00 2001 From: Alena Khineika Date: Tue, 19 Nov 2024 15:03:34 +0100 Subject: [PATCH 2/4] feat: show copilot introduction only if survey not shown --- src/mdbExtensionController.ts | 11 ++++- src/test/suite/mdbExtensionController.test.ts | 43 +++++++++++++++++++ 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/src/mdbExtensionController.ts b/src/mdbExtensionController.ts index eccc7a460..4ac01ac56 100644 --- a/src/mdbExtensionController.ts +++ b/src/mdbExtensionController.ts @@ -72,6 +72,7 @@ export default class MDBExtensionController implements vscode.Disposable { _editDocumentCodeLensProvider: EditDocumentCodeLensProvider; _exportToLanguageCodeLensProvider: ExportToLanguageCodeLensProvider; _participantController: ParticipantController; + _surveyShown = false; constructor( context: vscode.ExtensionContext, @@ -920,7 +921,7 @@ export default class MDBExtensionController implements vscode.Disposable { } async showCopilotIntroductionForEstablishedUsers(): Promise { - const hasBeenShownAlready = + const copilotIntroductionShown = this._storageController.get( StorageVariables.GLOBAL_COPILOT_INTRODUCTION_SHOWN ) === true; @@ -928,7 +929,11 @@ export default class MDBExtensionController implements vscode.Disposable { // Show the toast when it hasn't been show to the // user yet, and they have saved connections // -> they haven't just started using this extension. - if (hasBeenShownAlready || !this._connectionStorage.hasSavedConnections()) { + if ( + this._surveyShown || + copilotIntroductionShown || + !this._connectionStorage.hasSavedConnections() + ) { return; } @@ -986,6 +991,8 @@ export default class MDBExtensionController implements vscode.Disposable { return; } + this._surveyShown = true; + const action = 'Share your thoughts'; const text = 'How can we make the MongoDB extension better for you?'; const link = 'https://forms.gle/9viN9wcbsC3zvHyg7'; diff --git a/src/test/suite/mdbExtensionController.test.ts b/src/test/suite/mdbExtensionController.test.ts index eae0aee6d..cddfcb243 100644 --- a/src/test/suite/mdbExtensionController.test.ts +++ b/src/test/suite/mdbExtensionController.test.ts @@ -1867,6 +1867,14 @@ suite('MDBExtensionController Test Suite', function () { vscode.commands, 'executeCommand' ); + sandbox + .stub( + mdbTestExtension.testExtensionController, + '_surveyShown' + ) + .get(function getterFn() { + return false; + }); sandbox.replace( mdbTestExtension.testExtensionController._storageController, 'get', @@ -1959,6 +1967,11 @@ suite('MDBExtensionController Test Suite', function () { suite('when a has no connections saved', () => { let connectionsUpdateStub: SinonStub; beforeEach(() => { + sandbox + .stub(mdbTestExtension.testExtensionController, '_surveyShown') + .get(function getterFn() { + return false; + }); sandbox.replace( mdbTestExtension.testExtensionController._storageController, 'get', @@ -1982,6 +1995,36 @@ suite('MDBExtensionController Test Suite', function () { assert(showInformationMessageStub.notCalled); }); }); + + suite( + 'when a user has been shown the survey prompt during this extension launch', + () => { + let connectionsUpdateStub: SinonStub; + beforeEach(() => { + sandbox + .stub(mdbTestExtension.testExtensionController, '_surveyShown') + .get(function getterFn() { + return true; + }); + sandbox.replace( + mdbTestExtension.testExtensionController._connectionStorage, + 'hasSavedConnections', + sandbox.fake.returns(true) + ); + connectionsUpdateStub = sandbox.stub( + mdbTestExtension.testExtensionController._storageController, + 'update' + ); + connectionsUpdateStub.resolves(undefined); + + void mdbTestExtension.testExtensionController.showCopilotIntroductionForEstablishedUsers(); + }); + + test('they are not shown the copilot introduction prompt', () => { + assert(showInformationMessageStub.notCalled); + }); + } + ); }); }); From a3be0b290cdb0926777e998cc18d7b273a2cb2fd Mon Sep 17 00:00:00 2001 From: Alena Khineika Date: Fri, 22 Nov 2024 13:20:25 +0100 Subject: [PATCH 3/4] feat: startupNotificationShown --- src/mdbExtensionController.ts | 11 +- src/test/suite/mdbExtensionController.test.ts | 125 ++++++++++++------ 2 files changed, 95 insertions(+), 41 deletions(-) diff --git a/src/mdbExtensionController.ts b/src/mdbExtensionController.ts index c252dd662..9c6b3e253 100644 --- a/src/mdbExtensionController.ts +++ b/src/mdbExtensionController.ts @@ -72,7 +72,7 @@ export default class MDBExtensionController implements vscode.Disposable { _editDocumentCodeLensProvider: EditDocumentCodeLensProvider; _exportToLanguageCodeLensProvider: ExportToLanguageCodeLensProvider; _participantController: ParticipantController; - _surveyShown = false; + _startupNotificationShown = false; constructor( context: vscode.ExtensionContext, @@ -921,14 +921,14 @@ export default class MDBExtensionController implements vscode.Disposable { // user yet, and they have saved connections // -> they haven't just started using this extension. if ( - this._surveyShown || + this._startupNotificationShown || copilotIntroductionShown || !this._connectionStorage.hasSavedConnections() ) { return; } - const copilot = vscode.extensions.getExtension('github.copilot-chat'); + this._startupNotificationShown = true; const action = 'Chat with @MongoDB'; const text = @@ -940,6 +940,8 @@ export default class MDBExtensionController implements vscode.Disposable { title: action, } ); + + const copilot = vscode.extensions.getExtension('github.copilot-chat'); if (result?.title === action) { await vscode.commands.executeCommand('workbench.action.chat.newChat'); await vscode.commands.executeCommand( @@ -976,13 +978,14 @@ export default class MDBExtensionController implements vscode.Disposable { // user yet, and they have saved connections // -> they haven't just started using this extension if ( + this._startupNotificationShown || hasBeenShownSurveyAlready || !this._connectionStorage.hasSavedConnections() ) { return; } - this._surveyShown = true; + this._startupNotificationShown = true; const action = 'Share your thoughts'; const text = 'How can we make the MongoDB extension better for you?'; diff --git a/src/test/suite/mdbExtensionController.test.ts b/src/test/suite/mdbExtensionController.test.ts index 80aa682e5..dac0bb16a 100644 --- a/src/test/suite/mdbExtensionController.test.ts +++ b/src/test/suite/mdbExtensionController.test.ts @@ -1716,6 +1716,26 @@ suite('MDBExtensionController Test Suite', function () { }); suite('survey prompt', function () { + suite( + 'when a user has been shown the startup notification already', + function () { + beforeEach(() => { + sandbox + .stub( + mdbTestExtension.testExtensionController, + '_startupNotificationShown' + ) + .get(function getterFn() { + return true; + }); + }); + + test('they are not shown the survey prompt', () => { + assert(showInformationMessageStub.notCalled); + }); + } + ); + suite( "when a user hasn't been shown the survey prompt yet, and they have connections saved", () => { @@ -1730,6 +1750,15 @@ suite('MDBExtensionController Test Suite', function () { let connectionsUpdateStub: SinonStub; let uriParseStub: SinonStub; beforeEach(async () => { + sandbox + .stub( + mdbTestExtension.testExtensionController, + '_startupNotificationShown' + ) + .set(function setterFn() {}) + .get(function getterFn() { + return false; + }); showInformationMessageStub.resolves(reaction.value); openExternalStub.resolves(undefined); sandbox.replace( @@ -1796,6 +1825,15 @@ suite('MDBExtensionController Test Suite', function () { suite('when a user has been shown the survey prompt already', () => { let connectionsUpdateStub: SinonStub; beforeEach(() => { + sandbox + .stub( + mdbTestExtension.testExtensionController, + '_startupNotificationShown' + ) + .set(function setterFn() {}) + .get(function getterFn() { + return false; + }); sandbox.replace( mdbTestExtension.testExtensionController._storageController, 'get', @@ -1823,6 +1861,15 @@ suite('MDBExtensionController Test Suite', function () { suite('when a has no connections saved', () => { let connectionsUpdateStub: SinonStub; beforeEach(() => { + sandbox + .stub( + mdbTestExtension.testExtensionController, + '_startupNotificationShown' + ) + .set(function setterFn() {}) + .get(function getterFn() { + return false; + }); sandbox.replace( mdbTestExtension.testExtensionController._storageController, 'get', @@ -1849,6 +1896,26 @@ suite('MDBExtensionController Test Suite', function () { }); suite('copilot introduction prompt', function () { + suite( + 'when a user has been shown the startup notification already', + function () { + beforeEach(() => { + sandbox + .stub( + mdbTestExtension.testExtensionController, + '_startupNotificationShown' + ) + .get(function getterFn() { + return true; + }); + }); + + test('they are not shown the copilot introduction prompt', () => { + assert(showInformationMessageStub.notCalled); + }); + } + ); + suite( "when a user hasn't been shown the copilot introduction prompt yet, and they have connections saved", () => { @@ -1863,19 +1930,20 @@ suite('MDBExtensionController Test Suite', function () { let connectionsUpdateStub: SinonStub; let executeCommandStub: SinonStub; beforeEach(async () => { - showInformationMessageStub.resolves(reaction.value); - executeCommandStub = sandbox.stub( - vscode.commands, - 'executeCommand' - ); sandbox .stub( mdbTestExtension.testExtensionController, - '_surveyShown' + '_startupNotificationShown' ) + .set(function setterFn() {}) .get(function getterFn() { return false; }); + showInformationMessageStub.resolves(reaction.value); + executeCommandStub = sandbox.stub( + vscode.commands, + 'executeCommand' + ); sandbox.replace( mdbTestExtension.testExtensionController._storageController, 'get', @@ -1940,6 +2008,15 @@ suite('MDBExtensionController Test Suite', function () { () => { let connectionsUpdateStub: SinonStub; beforeEach(() => { + sandbox + .stub( + mdbTestExtension.testExtensionController, + '_startupNotificationShown' + ) + .set(function setterFn() {}) + .get(function getterFn() { + return false; + }); sandbox.replace( mdbTestExtension.testExtensionController._storageController, 'get', @@ -1969,7 +2046,11 @@ suite('MDBExtensionController Test Suite', function () { let connectionsUpdateStub: SinonStub; beforeEach(() => { sandbox - .stub(mdbTestExtension.testExtensionController, '_surveyShown') + .stub( + mdbTestExtension.testExtensionController, + '_startupNotificationShown' + ) + .set(function setterFn() {}) .get(function getterFn() { return false; }); @@ -1996,36 +2077,6 @@ suite('MDBExtensionController Test Suite', function () { assert(showInformationMessageStub.notCalled); }); }); - - suite( - 'when a user has been shown the survey prompt during this extension launch', - () => { - let connectionsUpdateStub: SinonStub; - beforeEach(() => { - sandbox - .stub(mdbTestExtension.testExtensionController, '_surveyShown') - .get(function getterFn() { - return true; - }); - sandbox.replace( - mdbTestExtension.testExtensionController._connectionStorage, - 'hasSavedConnections', - sandbox.fake.returns(true) - ); - connectionsUpdateStub = sandbox.stub( - mdbTestExtension.testExtensionController._storageController, - 'update' - ); - connectionsUpdateStub.resolves(undefined); - - void mdbTestExtension.testExtensionController.showCopilotIntroductionForEstablishedUsers(); - }); - - test('they are not shown the copilot introduction prompt', () => { - assert(showInformationMessageStub.notCalled); - }); - } - ); }); }); From 1c9de49b6924fa7980fdcb6912b12f7e1247fc30 Mon Sep 17 00:00:00 2001 From: Alena Khineika Date: Fri, 22 Nov 2024 14:06:11 +0100 Subject: [PATCH 4/4] docs: update comments --- src/mdbExtensionController.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/mdbExtensionController.ts b/src/mdbExtensionController.ts index 9c6b3e253..55ec199f4 100644 --- a/src/mdbExtensionController.ts +++ b/src/mdbExtensionController.ts @@ -917,8 +917,8 @@ export default class MDBExtensionController implements vscode.Disposable { StorageVariables.GLOBAL_COPILOT_INTRODUCTION_SHOWN ) === true; - // Show the toast when it hasn't been show to the - // user yet, and they have saved connections + // Show the toast when startup notifications have not been shown + // to the user yet and they have saved connections // -> they haven't just started using this extension. if ( this._startupNotificationShown || @@ -974,8 +974,8 @@ export default class MDBExtensionController implements vscode.Disposable { this._storageController.get(StorageVariables.GLOBAL_SURVEY_SHOWN) === surveyId; - // Show the survey when it hasn't been show to the - // user yet, and they have saved connections + // Show the toast when startup notifications have not been shown + // to the user yet and they have saved connections // -> they haven't just started using this extension if ( this._startupNotificationShown ||