diff --git a/.github/workflows/draft-release.yaml b/.github/workflows/draft-release.yaml index 2246bc3ac..62ec8fb49 100644 --- a/.github/workflows/draft-release.yaml +++ b/.github/workflows/draft-release.yaml @@ -80,6 +80,7 @@ jobs: uses: ./.github/workflows/actions/test-and-build with: SEGMENT_KEY: ${{ secrets.SEGMENT_KEY_PROD }} + MONGODB_DOCS_CHATBOT_BASE_URI: ${{ secrets.MONGODB_DOCS_CHATBOT_BASE_URI_PROD }} ARTIFACTORY_HOST: ${{ secrets.ARTIFACTORY_HOST }} ARTIFACTORY_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }} ARTIFACTORY_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }} diff --git a/.github/workflows/test-and-build.yaml b/.github/workflows/test-and-build.yaml index ceaaa04f3..73fc6ef88 100644 --- a/.github/workflows/test-and-build.yaml +++ b/.github/workflows/test-and-build.yaml @@ -36,6 +36,7 @@ jobs: uses: ./.github/workflows/actions/test-and-build with: SEGMENT_KEY: ${{ secrets.SEGMENT_KEY_PROD }} + MONGODB_DOCS_CHATBOT_BASE_URI: ${{ secrets.MONGODB_DOCS_CHATBOT_BASE_URI_PROD }} ARTIFACTORY_HOST: ${{ secrets.ARTIFACTORY_HOST }} ARTIFACTORY_PASSWORD: ${{ secrets.ARTIFACTORY_PASSWORD }} ARTIFACTORY_USERNAME: ${{ secrets.ARTIFACTORY_USERNAME }} diff --git a/package-lock.json b/package-lock.json index dbf2cddcf..2ffe1c822 100644 --- a/package-lock.json +++ b/package-lock.json @@ -93,6 +93,7 @@ "mocha-junit-reporter": "^2.2.1", "mocha-multi": "^1.1.7", "mongodb-client-encryption": "^6.0.1", + "mongodb-rag-core": "^0.4.1", "mongodb-runner": "^5.6.4", "node-fetch": "^2.7.0", "node-loader": "^0.6.0", @@ -145,6 +146,30 @@ "node": ">=6.0.0" } }, + "node_modules/@anthropic-ai/sdk": { + "version": "0.25.2", + "resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.25.2.tgz", + "integrity": "sha512-F1Hck/asswwidFLtGdMg3XYgRxEUfygNbpkq5KEaEGsHNaSfxeX18/uZGQCL0oQNcj/tYNx8BaFXVwRhFDi45g==", + "dev": true, + "dependencies": { + "@types/node": "^18.11.18", + "@types/node-fetch": "^2.6.4", + "abort-controller": "^3.0.0", + "agentkeepalive": "^4.2.1", + "form-data-encoder": "1.7.2", + "formdata-node": "^4.3.2", + "node-fetch": "^2.6.7" + } + }, + "node_modules/@anthropic-ai/sdk/node_modules/@types/node": { + "version": "18.19.50", + "resolved": "https://registry.npmjs.org/@types/node/-/node-18.19.50.tgz", + "integrity": "sha512-xonK+NRrMBRtkL1hVCc3G+uXtjh1Al4opBLjqVmipe5ZAaBYWW6cNAiBVZ1BvmkBhep698rP3UM3aRAdSALuhg==", + "dev": true, + "dependencies": { + "undici-types": "~5.26.4" + } + }, "node_modules/@asamuzakjp/dom-selector": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/@asamuzakjp/dom-selector/-/dom-selector-2.0.2.tgz", @@ -856,6 +881,35 @@ } } }, + "node_modules/@azure-rest/core-client": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@azure-rest/core-client/-/core-client-1.4.0.tgz", + "integrity": "sha512-ozTDPBVUDR5eOnMIwhggbnVmOrka4fXCs8n8mvUo4WLLc38kki6bAOByDoVZZPz/pZy2jMt2kwfpvy/UjALj6w==", + "dev": true, + "dependencies": { + "@azure/abort-controller": "^2.0.0", + "@azure/core-auth": "^1.3.0", + "@azure/core-rest-pipeline": "^1.5.0", + "@azure/core-tracing": "^1.0.1", + "@azure/core-util": "^1.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, + "node_modules/@azure-rest/core-client/node_modules/@azure/abort-controller": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.1.2.tgz", + "integrity": "sha512-nBrLsEWm4J2u5LpAPjxADTlq3trDgVZZXHNKabeXZtpq3d3AbN/KGO82R87rdDz5/lYB024rtEf10/q0urNgsA==", + "dev": true, + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@azure/abort-controller": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-1.1.0.tgz", @@ -968,6 +1022,18 @@ "node": ">= 14" } }, + "node_modules/@azure/core-sse": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/@azure/core-sse/-/core-sse-2.1.3.tgz", + "integrity": "sha512-KSSdIKy8kvWCpYr8Hzpu22j3wcXsVTYE0IlgmI1T/aHvBDsLgV91y90UTfVWnuiuApRLCCVC4gS09ApBGOmYQA==", + "dev": true, + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@azure/core-tracing": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.1.2.tgz", @@ -1103,6 +1169,24 @@ "node": ">=16" } }, + "node_modules/@azure/openai": { + "version": "1.0.0-beta.12", + "resolved": "https://registry.npmjs.org/@azure/openai/-/openai-1.0.0-beta.12.tgz", + "integrity": "sha512-qKblxr6oVa8GsyNzY+/Ub9VmEsPYKhBrUrPaNEQiM+qrxnBPVm9kaeqGFFb/U78Q2zOabmhF9ctYt3xBW0nWnQ==", + "dev": true, + "dependencies": { + "@azure-rest/core-client": "^1.1.7", + "@azure/core-auth": "^1.4.0", + "@azure/core-rest-pipeline": "^1.13.0", + "@azure/core-sse": "^2.0.0", + "@azure/core-util": "^1.4.0", + "@azure/logger": "^1.0.3", + "tslib": "^2.4.0" + }, + "engines": { + "node": ">=18.0.0" + } + }, "node_modules/@babel/code-frame": { "version": "7.24.7", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.24.7.tgz", @@ -1756,6 +1840,15 @@ "w3c-keyname": "^2.2.4" } }, + "node_modules/@colors/colors": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@colors/colors/-/colors-1.6.0.tgz", + "integrity": "sha512-Ir+AOibqzrIsL6ajt3Rz3LskB7OiMVHqltZmspbW/TJuTVuyOMirVqAkjfY6JISiLHgyNqicAC8AyHHGzNd/dA==", + "dev": true, + "engines": { + "node": ">=0.1.90" + } + }, "node_modules/@cspotcode/source-map-support": { "version": "0.8.1", "resolved": "https://registry.npmjs.org/@cspotcode/source-map-support/-/source-map-support-0.8.1.tgz", @@ -1778,6 +1871,17 @@ "@jridgewell/sourcemap-codec": "^1.4.10" } }, + "node_modules/@dabh/diagnostics": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/@dabh/diagnostics/-/diagnostics-2.0.3.tgz", + "integrity": "sha512-hrlQOIi7hAfzsMqlGSFyVucrx38O+j6wiGOf//H2ecvIEqYN4ADBSS2iLMh5UFyDunCNniUIPk/q3riFv45xRA==", + "dev": true, + "dependencies": { + "colorspace": "1.1.x", + "enabled": "2.0.x", + "kuler": "^2.0.0" + } + }, "node_modules/@discoveryjs/json-ext": { "version": "0.5.7", "resolved": "https://registry.npmjs.org/@discoveryjs/json-ext/-/json-ext-0.5.7.tgz", @@ -2263,6 +2367,130 @@ "@jridgewell/sourcemap-codec": "^1.4.14" } }, + "node_modules/@langchain/anthropic": { + "version": "0.2.18", + "resolved": "https://registry.npmjs.org/@langchain/anthropic/-/anthropic-0.2.18.tgz", + "integrity": "sha512-4ZDTxMwGKTPRAi2Supu/faBSmwPIm/ga5QlazyO78Mf/8QbDR2DcvM5394FAy+X/nRAfnMbyXteO5IRJm653gw==", + "dev": true, + "dependencies": { + "@anthropic-ai/sdk": "^0.25.2", + "@langchain/core": ">=0.2.21 <0.3.0", + "fast-xml-parser": "^4.4.1", + "zod": "^3.22.4", + "zod-to-json-schema": "^3.22.4" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@langchain/core": { + "version": "0.2.32", + "resolved": "https://registry.npmjs.org/@langchain/core/-/core-0.2.32.tgz", + "integrity": "sha512-S27M+9Qou2qtcLfFGEvANkJ/zHq5XApeQsR6Q4I7C6v9x07eoYr558h6vVy6WQmKcksgbCIJ854ikwp173wBjA==", + "dev": true, + "dependencies": { + "ansi-styles": "^5.0.0", + "camelcase": "6", + "decamelize": "1.2.0", + "js-tiktoken": "^1.0.12", + "langsmith": "^0.1.43", + "mustache": "^4.2.0", + "p-queue": "^6.6.2", + "p-retry": "4", + "uuid": "^10.0.0", + "zod": "^3.22.4", + "zod-to-json-schema": "^3.22.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@langchain/core/node_modules/@types/uuid": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-7gqG38EyHgyP1S+7+xomFtL+ZNHcKv6DwNaCZmJmo1vgMugyF3TCnXVg4t1uk89mLNwnLtnY3TpOpCOyp1/xHQ==", + "dev": true + }, + "node_modules/@langchain/core/node_modules/ansi-styles": { + "version": "5.2.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-5.2.0.tgz", + "integrity": "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/@langchain/core/node_modules/decamelize": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/decamelize/-/decamelize-1.2.0.tgz", + "integrity": "sha512-z2S+W9X73hAUUki+N+9Za2lBlun89zigOyGrsax+KUQ6wKW4ZoWpEYBkGhQjwAjjDCkWxhY0VKEhk8wzY7F5cA==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/@langchain/core/node_modules/langsmith": { + "version": "0.1.55", + "resolved": "https://registry.npmjs.org/langsmith/-/langsmith-0.1.55.tgz", + "integrity": "sha512-6NVtI04UUnIY59I/imOX02FG/QMGfqStu8tiJtyyreKMv2GAN0EE9Z5Ap1wzOe6v8ukEcV3NwEO2LYOPwup1PQ==", + "dev": true, + "dependencies": { + "@types/uuid": "^10.0.0", + "commander": "^10.0.1", + "p-queue": "^6.6.2", + "p-retry": "4", + "semver": "^7.6.3", + "uuid": "^10.0.0" + }, + "peerDependencies": { + "@langchain/core": "*", + "langchain": "*", + "openai": "*" + }, + "peerDependenciesMeta": { + "@langchain/core": { + "optional": true + }, + "langchain": { + "optional": true + }, + "openai": { + "optional": true + } + } + }, + "node_modules/@langchain/core/node_modules/uuid": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-10.0.0.tgz", + "integrity": "sha512-8XkAphELsDnEGrDxUOHB3RGvXz6TeuYSGEZBOjtTtPm2lwhGBjLgOzLHB63IUWfBpNucQjND6d3AOudO+H3RWQ==", + "dev": true, + "funding": [ + "https://github.com/sponsors/broofa", + "https://github.com/sponsors/ctavan" + ], + "bin": { + "uuid": "dist/bin/uuid" + } + }, + "node_modules/@langchain/openai": { + "version": "0.2.10", + "resolved": "https://registry.npmjs.org/@langchain/openai/-/openai-0.2.10.tgz", + "integrity": "sha512-ph5sYDAmhP55Fs3TW3+LXiqF+r/5zaaNO2tur9p2Otr8KWNDSgp5ezfPki1WWfuUJVoSQ+6HDYtr6n2V5N1Lew==", + "dev": true, + "dependencies": { + "@langchain/core": ">=0.2.26 <0.3.0", + "js-tiktoken": "^1.0.12", + "openai": "^4.57.3", + "zod": "^3.22.4", + "zod-to-json-schema": "^3.22.3" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/@leafygreen-ui/a11y": { "version": "1.4.13", "resolved": "https://registry.npmjs.org/@leafygreen-ui/a11y/-/a11y-1.4.13.tgz", @@ -5823,6 +6051,12 @@ "@types/node": "*" } }, + "node_modules/@types/retry": { + "version": "0.12.0", + "resolved": "https://registry.npmjs.org/@types/retry/-/retry-0.12.0.tgz", + "integrity": "sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==", + "dev": true + }, "node_modules/@types/scheduler": { "version": "0.16.3", "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.3.tgz", @@ -5877,6 +6111,12 @@ "undici-types": "~5.26.4" } }, + "node_modules/@types/triple-beam": { + "version": "1.3.5", + "resolved": "https://registry.npmjs.org/@types/triple-beam/-/triple-beam-1.3.5.tgz", + "integrity": "sha512-6WaYesThRMCl19iryMYP7/x2OVgCtbIVflDGFpWnb9irXI3UjYE4AzmYuiUKY1AJstGijoY+MgUszMgRxIYTYw==", + "dev": true + }, "node_modules/@types/use-sync-external-store": { "version": "0.0.3", "resolved": "https://registry.npmjs.org/@types/use-sync-external-store/-/use-sync-external-store-0.0.3.tgz", @@ -7138,6 +7378,12 @@ "node": ">=4" } }, + "node_modules/async": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/async/-/async-3.2.6.tgz", + "integrity": "sha512-htCUDlxyyCLMgaM3xXg0C0LW2xqfuQ6p05pCEIsXuyQ+a1koYKTuBMzRNwmybfLgvJDMd0r1LTn4+E0Ti6C2AA==", + "dev": true + }, "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -7160,6 +7406,17 @@ "resolved": "https://registry.npmjs.org/aws4/-/aws4-1.12.0.tgz", "integrity": "sha512-NmWvPnx0F1SfrQbYwOi7OeaNGokp9XhzNioJ/CSBs8Qa4vxug81mhJEAVZwxXuBmYB5KDRfMq/F3RR0BIU7sWg==" }, + "node_modules/axios": { + "version": "1.7.7", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.7.tgz", + "integrity": "sha512-S4kL7XrjgBmvdGut0sN3yJxqYzrDOnivkBiN0OFs6hLiUam3UPvswUo0kqGyhqUZGEOytHyumEdXsAkgCOUf3Q==", + "dev": true, + "dependencies": { + "follow-redirects": "^1.15.6", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, "node_modules/azure-devops-node-api": { "version": "12.5.0", "resolved": "https://registry.npmjs.org/azure-devops-node-api/-/azure-devops-node-api-12.5.0.tgz", @@ -7813,6 +8070,18 @@ "node": ">=6" } }, + "node_modules/camelcase": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", + "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", + "dev": true, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/caniuse-lite": { "version": "1.0.30001651", "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001651.tgz", @@ -8120,6 +8389,16 @@ "node": ">=16" } }, + "node_modules/color": { + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/color/-/color-3.2.1.tgz", + "integrity": "sha512-aBl7dZI9ENN6fUGC7mWpMTPNHmWUSNan9tuWN6ahh5ZLNk9baLJOnSMlrQkHcrfFgz2/RigjUVAjdx36VcemKA==", + "dev": true, + "dependencies": { + "color-convert": "^1.9.3", + "color-string": "^1.6.0" + } + }, "node_modules/color-convert": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", @@ -8136,6 +8415,31 @@ "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==" }, + "node_modules/color-string": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/color-string/-/color-string-1.9.1.tgz", + "integrity": "sha512-shrVawQFojnZv6xM40anx4CkoDP+fZsw/ZerEMsW/pyzsRbElpsL/DBVW7q3ExxwusdNXI3lXpuhEZkzs8p5Eg==", + "dev": true, + "dependencies": { + "color-name": "^1.0.0", + "simple-swizzle": "^0.2.2" + } + }, + "node_modules/color/node_modules/color-convert": { + "version": "1.9.3", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", + "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", + "dev": true, + "dependencies": { + "color-name": "1.1.3" + } + }, + "node_modules/color/node_modules/color-name": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", + "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", + "dev": true + }, "node_modules/colorette": { "version": "2.0.20", "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.20.tgz", @@ -8151,6 +8455,16 @@ "node": ">=0.1.90" } }, + "node_modules/colorspace": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/colorspace/-/colorspace-1.1.4.tgz", + "integrity": "sha512-BgvKJiuVu1igBUF2kEjRCZXol6wiiGbY5ipL/oVPwm0BL9sIpMIzM8IK7vwuxIIzOXMV3Ey5w+vxhm0rR/TN8w==", + "dev": true, + "dependencies": { + "color": "^3.1.3", + "text-hex": "1.0.x" + } + }, "node_modules/combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -8172,6 +8486,15 @@ "node": ">=14" } }, + "node_modules/common-tags": { + "version": "1.8.2", + "resolved": "https://registry.npmjs.org/common-tags/-/common-tags-1.8.2.tgz", + "integrity": "sha512-gk/Z852D2Wtb//0I+kRFNKKE9dIIVirjoqPoA1wJU+XePVXZfGeBpk45+A1rKO4Q43prqWBNY/MiIeRLbPWUaA==", + "dev": true, + "engines": { + "node": ">=4.0.0" + } + }, "node_modules/compare-module-exports": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/compare-module-exports/-/compare-module-exports-2.1.0.tgz", @@ -9107,18 +9430,6 @@ "balanced-match": "^1.0.0" } }, - "node_modules/depcheck/node_modules/camelcase": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", - "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", - "dev": true, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/depcheck/node_modules/minimatch": { "version": "7.4.6", "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-7.4.6.tgz", @@ -9614,6 +9925,12 @@ "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", "devOptional": true }, + "node_modules/enabled": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/enabled/-/enabled-2.0.0.tgz", + "integrity": "sha512-AKrN98kuwOzMIdAizXGI86UFBoo26CL21UM763y1h/GMSJ4/OHU9k2YlsmBpyScFo/wbLzWQJBMCW4+IO3/+OQ==", + "dev": true + }, "node_modules/encodeurl": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz", @@ -10870,6 +11187,12 @@ "node": ">=0.10.0" } }, + "node_modules/exponential-backoff": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/exponential-backoff/-/exponential-backoff-3.1.1.tgz", + "integrity": "sha512-dX7e/LHVJ6W3DE1MHWi9S1EYzDESENfLrYohG2G++ovZrYOkm4Knwa0mc1cn84xJOR4KEU0WSchhLbd0UklbHw==", + "dev": true + }, "node_modules/express": { "version": "4.19.2", "resolved": "https://registry.npmjs.org/express/-/express-4.19.2.tgz", @@ -10924,6 +11247,18 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" }, + "node_modules/extend-shallow": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz", + "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==", + "dev": true, + "dependencies": { + "is-extendable": "^0.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/external-editor": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-3.1.0.tgz", @@ -11069,6 +11404,12 @@ "pend": "~1.2.0" } }, + "node_modules/fecha": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/fecha/-/fecha-4.2.3.tgz", + "integrity": "sha512-OP2IUU6HeYKJi3i0z4A19kHMQoLVs4Hc+DPqqxI2h/DPZHTm/vjsfC6P0b4jCMy14XizLBqvndQ+UilD7707Jw==", + "dev": true + }, "node_modules/fetch-blob": { "version": "3.2.0", "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", @@ -11242,6 +11583,12 @@ "integrity": "sha512-5nqDSxl8nn5BSNxyR3n4I6eDmbolI6WT+QqR547RwxQapgjQBmtktdP+HTBb/a/zLsbzERTONyUB5pefh5TtjQ==", "dev": true }, + "node_modules/fn.name": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/fn.name/-/fn.name-1.1.0.tgz", + "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==", + "dev": true + }, "node_modules/focus-trap": { "version": "6.9.4", "resolved": "https://registry.npmjs.org/focus-trap/-/focus-trap-6.9.4.tgz", @@ -11264,6 +11611,26 @@ "react-dom": ">=16.3.0" } }, + "node_modules/follow-redirects": { + "version": "1.15.9", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.9.tgz", + "integrity": "sha512-gew4GsXizNgdoRyqmyfMHyAmXsZDk6mHkSxZFCzW9gwlbtOW44CDtYavM+y+72qD/Vq2l550kMF52DT8fOLJqQ==", + "dev": true, + "funding": [ + { + "type": "individual", + "url": "https://github.com/sponsors/RubenVerborgh" + } + ], + "engines": { + "node": ">=4.0" + }, + "peerDependenciesMeta": { + "debug": { + "optional": true + } + } + }, "node_modules/for-each": { "version": "0.3.3", "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.3.tgz", @@ -11477,6 +11844,15 @@ "node": ">= 0.6" } }, + "node_modules/front-matter": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/front-matter/-/front-matter-4.0.2.tgz", + "integrity": "sha512-I8ZuJ/qG92NWX8i5x1Y8qyj3vizhXS31OxjKDu3LKP+7/qBgfIKValiZIEwoVoJKUHlhWtYrktkxV1XsX+pPlg==", + "dev": true, + "dependencies": { + "js-yaml": "^3.13.1" + } + }, "node_modules/fs-constants": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", @@ -11919,6 +12295,21 @@ "integrity": "sha512-EtKwoO6kxCL9WO5xipiHTZlSzBm7WLT627TqC/uVRd0HKmq8NXyebnNYxDoBi7wt8eTWrUrKXCOVaFq9x1kgag==", "dev": true }, + "node_modules/gray-matter": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/gray-matter/-/gray-matter-4.0.3.tgz", + "integrity": "sha512-5v6yZd4JK3eMI3FqqCouswVqwugaA9r4dNZB1wwcmrD02QkV5H0y7XBQW8QwQqEaZY1pM9aqORSORhJRdNK44Q==", + "dev": true, + "dependencies": { + "js-yaml": "^3.13.1", + "kind-of": "^6.0.2", + "section-matter": "^1.0.0", + "strip-bom-string": "^1.0.0" + }, + "engines": { + "node": ">=6.0" + } + }, "node_modules/gzip-size": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/gzip-size/-/gzip-size-6.0.0.tgz", @@ -12920,6 +13311,15 @@ "resolved": "https://registry.npmjs.org/is-electron-renderer/-/is-electron-renderer-2.0.1.tgz", "integrity": "sha512-pRlQnpaCFhDVPtkXkP+g9Ybv/CjbiQDjnKFQTEjpBfDKeV6dRDBczuFRDpM6DVfk2EjpMS8t5kwE5jPnqYl3zA==" }, + "node_modules/is-extendable": { + "version": "0.1.1", + "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz", + "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", @@ -13274,6 +13674,15 @@ "url": "https://github.com/sponsors/panva" } }, + "node_modules/js-tiktoken": { + "version": "1.0.14", + "resolved": "https://registry.npmjs.org/js-tiktoken/-/js-tiktoken-1.0.14.tgz", + "integrity": "sha512-Pk3l3WOgM9joguZY2k52+jH82RtABRgB5RdGFZNUGbOKGMVlNmafcPA3b0ITcCZPu1L9UclP1tne6aw7ZI4Myg==", + "dev": true, + "dependencies": { + "base64-js": "^1.5.1" + } + }, "node_modules/js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", @@ -13614,6 +14023,12 @@ "node": ">=0.10.0" } }, + "node_modules/kuler": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/kuler/-/kuler-2.0.0.tgz", + "integrity": "sha512-Xq9nH7KlWZmXAtodXDDRE7vs6DU1gTU8zYDHDiWLSip45Egwq3plLHzPn27NgvzL2r1LMPC1vdqh98sQxtqj4A==", + "dev": true + }, "node_modules/leven": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/leven/-/leven-3.1.0.tgz", @@ -13837,6 +14252,23 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/logform": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/logform/-/logform-2.6.1.tgz", + "integrity": "sha512-CdaO738xRapbKIMVn2m4F6KTj4j7ooJ8POVnebSgKo3KBz5axNXRAL7ZdRjIV6NOr2Uf4vjtRkxrFETOioCqSA==", + "dev": true, + "dependencies": { + "@colors/colors": "1.6.0", + "@types/triple-beam": "^1.3.2", + "fecha": "^4.2.0", + "ms": "^2.1.1", + "safe-stable-stringify": "^2.3.1", + "triple-beam": "^1.3.0" + }, + "engines": { + "node": ">= 12.0.0" + } + }, "node_modules/loose-envify": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", @@ -14704,6 +15136,65 @@ "lodash": "^4.17.21" } }, + "node_modules/mongodb-rag-core": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/mongodb-rag-core/-/mongodb-rag-core-0.4.1.tgz", + "integrity": "sha512-McgQyKeex2e2qR8PNo4zmn5XEAnPs5jmdtRZn5f9GAY1tQGjN+JS7ePYgLKwsEDUxRsR56rrtn9ExJgdFo9gFg==", + "dev": true, + "dependencies": { + "@azure/openai": "^1.0.0-beta.5", + "@langchain/anthropic": "^0.2.15", + "@langchain/core": "^0.2.27", + "@langchain/openai": "^0.2.7", + "common-tags": "^1", + "dotenv": "^16.3.1", + "exponential-backoff": "^3.1.1", + "front-matter": "^4.0.2", + "gray-matter": "^4.0.3", + "mongodb": "^6.3.0", + "openai": "^3", + "toml": "^3.0.0", + "typechat": "^0.0.10", + "winston": "^3", + "yaml": "^2.3.1", + "zod": "^3.21.4" + }, + "engines": { + "node": ">=18", + "npm": ">=8" + } + }, + "node_modules/mongodb-rag-core/node_modules/axios": { + "version": "0.26.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.26.1.tgz", + "integrity": "sha512-fPwcX4EvnSHuInCMItEhAGnaSEXRBjtzh9fOtsE6E1G6p7vl7edEeZe11QHf18+6+9gR5PbKV/sGKNaD8YaMeA==", + "dev": true, + "dependencies": { + "follow-redirects": "^1.14.8" + } + }, + "node_modules/mongodb-rag-core/node_modules/openai": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/openai/-/openai-3.3.0.tgz", + "integrity": "sha512-uqxI/Au+aPRnsaQRe8CojU0eCR7I0mBiKjD3sNMzY6DaC1ZVrc85u98mtJW6voDug8fgGN+DIZmTDxTthxb7dQ==", + "dev": true, + "dependencies": { + "axios": "^0.26.0", + "form-data": "^4.0.0" + } + }, + "node_modules/mongodb-rag-core/node_modules/yaml": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.5.1.tgz", + "integrity": "sha512-bLQOjaX/ADgQ20isPJRvF0iRUHIxVhYvr53Of7wGcWlO2jvtUlH5m87DsmulFVxRpNLOnI4tB6p/oh8D7kpn9Q==", + "dev": true, + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/mongodb-redact": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/mongodb-redact/-/mongodb-redact-1.1.3.tgz", @@ -14900,6 +15391,15 @@ "object-assign": "^4.1.0" } }, + "node_modules/mustache": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/mustache/-/mustache-4.2.0.tgz", + "integrity": "sha512-71ippSywq5Yb7/tVYyGbkBggbU8H3u5Rz56fH60jGFgr8uHwxs+aSKeqmluIVzM0m0kB7xQjKS6qPfd0b2ZoqQ==", + "dev": true, + "bin": { + "mustache": "bin/mustache" + } + }, "node_modules/mute-stream": { "version": "0.0.8", "resolved": "https://registry.npmjs.org/mute-stream/-/mute-stream-0.0.8.tgz", @@ -15579,6 +16079,15 @@ "wrappy": "1" } }, + "node_modules/one-time": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/one-time/-/one-time-1.0.0.tgz", + "integrity": "sha512-5DXOiRKwuSEcQ/l0kGCF6Q3jcADFv5tSmRaJck/OqkVFcOzutB134KRSfF0xDrL39MNnqxbHBbUUcjZIhTgb2g==", + "dev": true, + "dependencies": { + "fn.name": "1.x.x" + } + }, "node_modules/onetime": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", @@ -15611,14 +16120,14 @@ } }, "node_modules/openai": { - "version": "4.57.0", - "resolved": "https://registry.npmjs.org/openai/-/openai-4.57.0.tgz", - "integrity": "sha512-JnwBSIYqiZ3jYjB5f2in8hQ0PRA092c6m+/6dYB0MzK0BEbn+0dioxZsPLBm5idJbg9xzLNOiGVm2OSuhZ+BdQ==", + "version": "4.59.0", + "resolved": "https://registry.npmjs.org/openai/-/openai-4.59.0.tgz", + "integrity": "sha512-3bn7FypMt2R1ZDuO0+GcXgBEnVFhIzrpUkb47pQRoYvyfdZ2fQXcuP14aOc4C8F9FvCtZ/ElzJmVzVqnP4nHNg==", "dev": true, "dependencies": { "@types/node": "^18.11.18", "@types/node-fetch": "^2.6.4", - "@types/qs": "^6.9.7", + "@types/qs": "^6.9.15", "abort-controller": "^3.0.0", "agentkeepalive": "^4.2.1", "form-data-encoder": "1.7.2", @@ -15830,6 +16339,15 @@ "node": ">=8" } }, + "node_modules/p-finally": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/p-finally/-/p-finally-1.0.0.tgz", + "integrity": "sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==", + "dev": true, + "engines": { + "node": ">=4" + } + }, "node_modules/p-limit": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-2.3.0.tgz", @@ -15857,6 +16375,47 @@ "node": ">=8" } }, + "node_modules/p-queue": { + "version": "6.6.2", + "resolved": "https://registry.npmjs.org/p-queue/-/p-queue-6.6.2.tgz", + "integrity": "sha512-RwFpb72c/BhQLEXIZ5K2e+AhgNVmIejGlTgiB9MzZ0e93GRvqZ7uSi0dvRF7/XIXDeNkra2fNHBxTyPDGySpjQ==", + "dev": true, + "dependencies": { + "eventemitter3": "^4.0.4", + "p-timeout": "^3.2.0" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/p-retry": { + "version": "4.6.2", + "resolved": "https://registry.npmjs.org/p-retry/-/p-retry-4.6.2.tgz", + "integrity": "sha512-312Id396EbJdvRONlngUx0NydfrIQ5lsYu0znKVUzVvArzEIt08V1qhtyESbGVd1FGX7UKtiFp5uwKZdM8wIuQ==", + "dev": true, + "dependencies": { + "@types/retry": "0.12.0", + "retry": "^0.13.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/p-timeout": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/p-timeout/-/p-timeout-3.2.0.tgz", + "integrity": "sha512-rhIwUycgwwKcP9yTOOFK/AKsAopjjCakVqLHePO3CC6Mir1Z99xT+R63jZxAT5lFZLa2inS5h+ZS2GvR99/FBg==", + "dev": true, + "dependencies": { + "p-finally": "^1.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/p-try": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", @@ -16457,6 +17016,12 @@ "node": ">= 0.10" } }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", + "dev": true + }, "node_modules/pseudomap": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/pseudomap/-/pseudomap-1.0.2.tgz", @@ -17091,6 +17656,15 @@ "node": ">=8" } }, + "node_modules/retry": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/retry/-/retry-0.13.1.tgz", + "integrity": "sha512-XQBQ3I8W1Cge0Seh+6gjj03LbmRFWuoszgK9ooCpwYIrhhoO80pfq4cUkU5DkknwfOfFteRwlZ56PYOGYyFWdg==", + "dev": true, + "engines": { + "node": ">= 4" + } + }, "node_modules/reusify": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/reusify/-/reusify-1.0.4.tgz", @@ -17271,6 +17845,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/safe-stable-stringify": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.5.0.tgz", + "integrity": "sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA==", + "dev": true, + "engines": { + "node": ">=10" + } + }, "node_modules/safer-buffer": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", @@ -17324,6 +17907,19 @@ "url": "https://opencollective.com/webpack" } }, + "node_modules/section-matter": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/section-matter/-/section-matter-1.0.0.tgz", + "integrity": "sha512-vfD3pmTzGpufjScBh50YHKzEu2lxBWhVEHsNGoEXmCmn2hKGfeNLYMzCJpe8cD7gqX7TJluOVpBkAequ6dgMmA==", + "dev": true, + "dependencies": { + "extend-shallow": "^2.0.1", + "kind-of": "^6.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/seek-bzip": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/seek-bzip/-/seek-bzip-1.0.6.tgz", @@ -17349,9 +17945,9 @@ "integrity": "sha512-OwpTSOfy6xSs1+pwcNrv0RBMOzI39Lp3qQKUTPVVPRjCdNa5JH/oPRiqsesIskK8TVgmRiHwO4KXlV2Li9dANA==" }, "node_modules/semver": { - "version": "7.6.2", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.2.tgz", - "integrity": "sha512-FNAIBWCx9qcRhoHcgcJ0gvU7SN1lYU2ZXuSfl04bSC5OpvDHFyJCjdNHomPXxjQlCBU67YW64PzY7/VIEH7F2w==", + "version": "7.6.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", + "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", "bin": { "semver": "bin/semver.js" }, @@ -17600,6 +18196,21 @@ "simple-concat": "^1.0.0" } }, + "node_modules/simple-swizzle": { + "version": "0.2.2", + "resolved": "https://registry.npmjs.org/simple-swizzle/-/simple-swizzle-0.2.2.tgz", + "integrity": "sha512-JA//kQgZtbuY83m+xT+tXJkmJncGMTFT+C+g2h2R9uxkYIrE2yy9sgmcLhCnw57/WSD+Eh3J97FPEDFnbXnDUg==", + "dev": true, + "dependencies": { + "is-arrayish": "^0.3.1" + } + }, + "node_modules/simple-swizzle/node_modules/is-arrayish": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.3.2.tgz", + "integrity": "sha512-eVRqCvVlZbuw3GrM63ovNSNAeA1K16kaR/LRY/92w0zxQ5/1YzwblUX652i4Xs9RwAGjW9d9y6X88t8OaAJfWQ==", + "dev": true + }, "node_modules/sinon": { "version": "9.2.4", "resolved": "https://registry.npmjs.org/sinon/-/sinon-9.2.4.tgz", @@ -17937,6 +18548,15 @@ "nan": "^2.18.0" } }, + "node_modules/stack-trace": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", + "integrity": "sha512-KGzahc7puUKkzyMt+IqAep+TVNbKP+k2Lmwhub39m1AsTSkaDutx56aDCo+HLDzf/D26BIHTJWNiTG1KAJiQCg==", + "dev": true, + "engines": { + "node": "*" + } + }, "node_modules/stackframe": { "version": "1.3.4", "resolved": "https://registry.npmjs.org/stackframe/-/stackframe-1.3.4.tgz", @@ -18255,6 +18875,15 @@ "node": ">=8" } }, + "node_modules/strip-bom-string": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/strip-bom-string/-/strip-bom-string-1.0.0.tgz", + "integrity": "sha512-uCC2VHvQRYu+lMh4My/sFNmF2klFymLX1wHJeXnbEJERpV/ZsVuonzerjfrGpIGF7LBVa1O7i9kjiWvJiFck8g==", + "dev": true, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/strip-dirs": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/strip-dirs/-/strip-dirs-2.1.0.tgz", @@ -18691,6 +19320,12 @@ "integrity": "sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==", "dev": true }, + "node_modules/text-hex": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/text-hex/-/text-hex-1.0.0.tgz", + "integrity": "sha512-uuVGNWzgJ4yhRaNSiubPY7OjISw4sw4E5Uv0wbjp+OzcbmVU/rsT8ujgcXJhn9ypzsgr5vlzpPqP+MBBKcGvbg==", + "dev": true + }, "node_modules/text-table": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/text-table/-/text-table-0.2.0.tgz", @@ -18803,6 +19438,12 @@ "node": ">=0.6" } }, + "node_modules/toml": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/toml/-/toml-3.0.0.tgz", + "integrity": "sha512-y/mWCZinnvxjTKYhJ+pYxwD0mRLVvOtdS2Awbgxln6iEnt4rk0yBxeSBHkGJcPucRiG0e55mwWp+g/05rsrd6w==", + "dev": true + }, "node_modules/totalist": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/totalist/-/totalist-3.0.1.tgz", @@ -18849,6 +19490,15 @@ "node": ">=18" } }, + "node_modules/triple-beam": { + "version": "1.4.1", + "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.4.1.tgz", + "integrity": "sha512-aZbgViZrg1QNcG+LULa7nhZpJTZSLm/mXnHXnbAbjmN5aSa0y7V+wvv6+4WaBtpISJzThKy+PIPxc1Nq1EJ9mg==", + "dev": true, + "engines": { + "node": ">= 14.0.0" + } + }, "node_modules/ts-loader": { "version": "9.5.1", "resolved": "https://registry.npmjs.org/ts-loader/-/ts-loader-9.5.1.tgz", @@ -19048,6 +19698,32 @@ "node": ">= 0.6" } }, + "node_modules/typechat": { + "version": "0.0.10", + "resolved": "https://registry.npmjs.org/typechat/-/typechat-0.0.10.tgz", + "integrity": "sha512-iF/wLLaZWt4Q9WO8stpq3NKilAa4b8hnCD16EirdhaxzAYk80MCb1wnW1il7GhkMNJuhJUD38dxs8q4A/EdxJw==", + "dev": true, + "dependencies": { + "axios": "^1.4.0", + "typescript": "^5.1.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/typechat/node_modules/typescript": { + "version": "5.6.2", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.6.2.tgz", + "integrity": "sha512-NW8ByodCSNCwZeghjN3o+JX5OFH0Ojg6sadjEKY4huZ52TqbJTJnDo5+Tw98lSy63NZvi4n+ez5m2u5d4PkZyw==", + "dev": true, + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, "node_modules/typed-array-length": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/typed-array-length/-/typed-array-length-1.0.4.tgz", @@ -19757,6 +20433,100 @@ "integrity": "sha512-CC1bOL87PIWSBhDcTrdeLo6eGT7mCFtrg0uIJtqJUFyK+eJnzl8A1niH56uu7KMa5XFrtiV+AQuHO3n7DsHnLQ==", "dev": true }, + "node_modules/winston": { + "version": "3.14.2", + "resolved": "https://registry.npmjs.org/winston/-/winston-3.14.2.tgz", + "integrity": "sha512-CO8cdpBB2yqzEf8v895L+GNKYJiEq8eKlHU38af3snQBQ+sdAIUepjMSguOIJC7ICbzm0ZI+Af2If4vIJrtmOg==", + "dev": true, + "dependencies": { + "@colors/colors": "^1.6.0", + "@dabh/diagnostics": "^2.0.2", + "async": "^3.2.3", + "is-stream": "^2.0.0", + "logform": "^2.6.0", + "one-time": "^1.0.0", + "readable-stream": "^3.4.0", + "safe-stable-stringify": "^2.3.1", + "stack-trace": "0.0.x", + "triple-beam": "^1.3.0", + "winston-transport": "^4.7.0" + }, + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/winston-transport": { + "version": "4.7.1", + "resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.7.1.tgz", + "integrity": "sha512-wQCXXVgfv/wUPOfb2x0ruxzwkcZfxcktz6JIMUaPLmcNhO4bZTwA/WtDWK74xV3F2dKu8YadrFv0qhwYjVEwhA==", + "dev": true, + "dependencies": { + "logform": "^2.6.1", + "readable-stream": "^3.6.2", + "triple-beam": "^1.3.0" + }, + "engines": { + "node": ">= 12.0.0" + } + }, + "node_modules/winston-transport/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/winston-transport/node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/winston/node_modules/is-stream": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", + "dev": true, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/winston/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/winston/node_modules/string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, "node_modules/wipe-node-cache": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/wipe-node-cache/-/wipe-node-cache-2.1.2.tgz", @@ -20055,18 +20825,6 @@ "node": ">=10" } }, - "node_modules/yargs-unparser/node_modules/camelcase": { - "version": "6.3.0", - "resolved": "https://registry.npmjs.org/camelcase/-/camelcase-6.3.0.tgz", - "integrity": "sha512-Gmy6FhYlCY7uOElZUSbxo2UCDH8owEk996gkbrpsgGtrJLM3J7jGxl9Ic7Qwwj4ivOE5AWZWRMecDdF7hqGjFA==", - "dev": true, - "engines": { - "node": ">=10" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/yargs-unparser/node_modules/is-plain-obj": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/is-plain-obj/-/is-plain-obj-2.1.0.tgz", @@ -20123,6 +20881,15 @@ "funding": { "url": "https://github.com/sponsors/colinhacks" } + }, + "node_modules/zod-to-json-schema": { + "version": "3.23.3", + "resolved": "https://registry.npmjs.org/zod-to-json-schema/-/zod-to-json-schema-3.23.3.tgz", + "integrity": "sha512-TYWChTxKQbRJp5ST22o/Irt9KC5nj7CdBKYB/AosCRdj/wxEMvv4NNaj9XVUHDOIp53ZxArGhnw5HMZziPFjog==", + "dev": true, + "peerDependencies": { + "zod": "^3.23.3" + } } } } diff --git a/package.json b/package.json index 83e1250ae..61ea98fe2 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,7 @@ "update-grammar": "ts-node ./scripts/update-grammar.ts", "precompile": "npm run clean", "compile": "npm-run-all compile:*", - "compile:keyfile": "ts-node ./scripts/generate-keyfile.ts", + "compile:constants": "ts-node ./scripts/generate-constants.ts", "compile:resources": "npm run update-grammar", "compile:extension": "tsc -p ./", "compile:extension-bundles": "webpack --mode development", @@ -56,7 +56,7 @@ "test-webview": "mocha -r ts-node/register --file ./src/test/setup-webview.ts src/test/suite/views/webview-app/**/*.test.tsx", "ai-accuracy-tests": "mocha -r ts-node/register --file ./src/test/ai-accuracy-tests/test-setup.ts ./src/test/ai-accuracy-tests/ai-accuracy-tests.ts", "analyze-bundle": "webpack --mode production --analyze", - "vscode:prepublish": "npm run clean && npm run compile:keyfile && npm run compile:resources && webpack --mode production", + "vscode:prepublish": "npm run clean && npm run compile:constants && npm run compile:resources && webpack --mode production", "check": "npm run lint && npm run depcheck", "depcheck": "depcheck", "package": "cross-env NODE_OPTIONS='--require ./scripts/no-npm-list-fail.js' vsce package --githubBranch main", @@ -94,6 +94,11 @@ "name": "query", "isSticky": true, "description": "Ask how to write MongoDB queries or pipelines. For example, you can ask: \"Show me all the documents where the address contains the word street\"." + }, + { + "name": "docs", + "isSticky": true, + "description": "Ask MongoDB-related questions and find answers in the official documentation." } ] } @@ -1202,6 +1207,7 @@ "@mongodb-js/oidc-mock-provider": "^0.9.1", "@mongodb-js/oidc-plugin": "^0.4.0", "@mongodb-js/prettier-config-devtools": "^1.0.1", + "mongodb-rag-core": "^0.4.1", "@mongodb-js/sbom-tools": "^0.7.1", "@mongodb-js/signing-utils": "^0.3.5", "@mongosh/service-provider-core": "^2.2.15", diff --git a/scripts/generate-constants.ts b/scripts/generate-constants.ts new file mode 100644 index 000000000..19c738210 --- /dev/null +++ b/scripts/generate-constants.ts @@ -0,0 +1,31 @@ +#! /usr/bin/env ts-node + +import ora from 'ora'; +import fs from 'fs'; +import path from 'path'; +import { resolve } from 'path'; +import { config } from 'dotenv'; +import { promisify } from 'util'; + +const writeFile = promisify(fs.writeFile); +const ROOT_DIR = path.join(__dirname, '..'); +const ui = ora('Generate constants file').start(); + +config({ path: resolve(__dirname, '../.env') }); + +(async () => { + await writeFile( + `${ROOT_DIR}/constants.json`, + JSON.stringify( + { + segmentKey: process.env.SEGMENT_KEY, + docsChatbotBaseUri: process.env.MONGODB_DOCS_CHATBOT_BASE_URI, + }, + null, + 2 + ) + ); + ui.succeed('The constants file has been generated'); +})().catch((error) => { + ui.fail(`Failed to generate constants file: ${error.message}`); +}); diff --git a/scripts/generate-keyfile.ts b/scripts/generate-keyfile.ts deleted file mode 100644 index b7a6add73..000000000 --- a/scripts/generate-keyfile.ts +++ /dev/null @@ -1,28 +0,0 @@ -#! /usr/bin/env ts-node - -import ora from 'ora'; -import fs from 'fs'; -import path from 'path'; -import { resolve } from 'path'; -import { config } from 'dotenv'; -import { promisify } from 'util'; - -const writeFile = promisify(fs.writeFile); -const ROOT_DIR = path.join(__dirname, '..'); -const ui = ora('Generate constants keyfile').start(); - -config({ path: resolve(__dirname, '../.env') }); - -(async () => { - if (process.env.SEGMENT_KEY) { - await writeFile( - `${ROOT_DIR}/constants.json`, - JSON.stringify({ segmentKey: process.env.SEGMENT_KEY }, null, 2) - ); - ui.succeed('Generated segment constants file'); - } else { - throw new Error('The Segment key is missing in environment variables'); - } -})().catch((error) => { - ui.fail(`Failed to generate segment constants file: ${error.message}`); -}); diff --git a/src/mdbExtensionController.ts b/src/mdbExtensionController.ts index a72aee212..98bbf9e7d 100644 --- a/src/mdbExtensionController.ts +++ b/src/mdbExtensionController.ts @@ -155,6 +155,7 @@ export default class MDBExtensionController implements vscode.Disposable { this._telemetryService.activateSegmentAnalytics(); this._participantController.createParticipant(this._context); + await this._participantController.createDocsChatbot(this._context); await this._connectionController.loadSavedConnections(); await this._languageServerController.startLanguageServer(); diff --git a/src/participant/chatMetadata.ts b/src/participant/chatMetadata.ts index 29a196c36..dc31d28ae 100644 --- a/src/participant/chatMetadata.ts +++ b/src/participant/chatMetadata.ts @@ -4,6 +4,7 @@ import { v4 as uuidv4 } from 'uuid'; export type ChatMetadata = { databaseName?: string; collectionName?: string; + docsChatbotConversationId?: string; }; export class ChatMetadataStore { diff --git a/src/participant/constants.ts b/src/participant/constants.ts index 39f560998..b867f0738 100644 --- a/src/participant/constants.ts +++ b/src/participant/constants.ts @@ -83,3 +83,12 @@ export function queryRequestChatResult( ): ChatResult { return createChatResult('query', history); } + +export function docsRequestChatResult(chatId: string): ChatResult { + return { + metadata: { + chatId, + intent: 'docs', + }, + }; +} diff --git a/src/participant/docsChatbotAIService.ts b/src/participant/docsChatbotAIService.ts new file mode 100644 index 000000000..d73de13f5 --- /dev/null +++ b/src/participant/docsChatbotAIService.ts @@ -0,0 +1,130 @@ +import type { Reference, VerifiedAnswer } from 'mongodb-rag-core'; + +const MONGODB_DOCS_CHATBOT_API_VERSION = 'v1'; + +// eslint-disable-next-line @typescript-eslint/no-var-requires +const { version } = require('../../package.json'); + +type Role = 'user' | 'assistant'; + +type ConversationData = { + _id: string; + createdAt: string; + messages: MessageData[]; + conversationId: string; +}; + +type MessageData = { + id: string; + role: Role; + content: string; + createdAt: string; + rating?: boolean; + references?: Reference[]; + suggestedPrompts?: string[]; + metadata?: AssistantMessageMetadata; +}; + +type AssistantMessageMetadata = { + [k: string]: unknown; + + /** + If the message came from the verified answers collection, contains the + metadata about the verified answer. + */ + verifiedAnswer?: { + _id: VerifiedAnswer['_id']; + created: string; + updated: string | undefined; + }; +}; + +export class DocsChatbotAIService { + _serverBaseUri?: string; + + constructor(serverBaseUri?: string) { + this._serverBaseUri = serverBaseUri; + } + + private getServerBaseUri(): string { + if (!this._serverBaseUri) { + throw new Error( + 'You must define a serverBaseUri for the DocsChatbotAIService' + ); + } + return this._serverBaseUri; + } + + private getUri(path: string): string { + const serverBaseUri = this.getServerBaseUri(); + return `${serverBaseUri}api/${MONGODB_DOCS_CHATBOT_API_VERSION}${path}`; + } + + async createConversation(): Promise { + const uri = this.getUri('/conversations'); + return this._fetch({ + uri, + method: 'POST', + }); + } + + async _fetch({ + uri, + method, + body, + headers, + }: { + uri: string; + method: string; + body?: string; + headers?: { [key: string]: string }; + }): Promise { + const resp = await fetch(uri, { + headers: { + origin: this.getServerBaseUri(), + 'User-Agent': `mongodb-vscode/${version}`, + ...headers, + }, + method, + ...(body && { body }), + }); + let conversation; + try { + conversation = await resp.json(); + } catch (error) { + throw new Error('[Docs chatbot] Internal server error'); + } + + if (resp.status === 400) { + throw new Error(`[Docs chatbot] Bad request: ${conversation.error}`); + } + if (resp.status === 429) { + throw new Error(`[Docs chatbot] Rate limited: ${conversation.error}`); + } + if (resp.status >= 500) { + throw new Error( + `[Docs chatbot] Internal server error: ${conversation.error}` + ); + } + return { + ...conversation, + conversationId: conversation._id, + }; + } + + async addMessage({ + conversationId, + message, + }: { + conversationId: string; + message: string; + }): Promise { + const uri = this.getUri(`/conversations/${conversationId}/messages`); + return await this._fetch({ + uri, + method: 'POST', + body: JSON.stringify({ message }), + headers: { 'Content-Type': 'application/json' }, + }); + } +} diff --git a/src/participant/markdown.ts b/src/participant/markdown.ts index 75ca9ed4b..df377b4a9 100644 --- a/src/participant/markdown.ts +++ b/src/participant/markdown.ts @@ -23,10 +23,10 @@ export function createMarkdownLink({ ) : undefined; const commandQueryString = data ? `?${encodedData}` : ''; - const connName = new vscode.MarkdownString( + const link = new vscode.MarkdownString( `- ${name}\n` ); - connName.supportHtml = true; - connName.isTrusted = { enabledCommands: [commandId] }; - return connName; + link.supportHtml = true; + link.isTrusted = { enabledCommands: [commandId] }; + return link; } diff --git a/src/participant/participant.ts b/src/participant/participant.ts index bd2f92580..4ccd57eff 100644 --- a/src/participant/participant.ts +++ b/src/participant/participant.ts @@ -1,6 +1,9 @@ import * as vscode from 'vscode'; import { getSimplifiedSchema } from 'mongodb-schema'; import type { Document } from 'bson'; +import { config } from 'dotenv'; +import path from 'path'; +import { promises as fs } from 'fs'; import { createLogger } from '../logging'; import type ConnectionController from '../connectionController'; @@ -17,6 +20,7 @@ import { genericRequestChatResult, namespaceRequestChatResult, queryRequestChatResult, + docsRequestChatResult, } from './constants'; import { QueryPrompt } from './prompts/query'; import { COL_NAME_ID, DB_NAME_ID, NamespacePrompt } from './prompts/namespace'; @@ -25,16 +29,20 @@ import { getSimplifiedSampleDocuments } from './sampleDocuments'; import { getCopilotModel } from './model'; import { createMarkdownLink } from './markdown'; import { ChatMetadataStore } from './chatMetadata'; +import { DocsChatbotAIService } from './docsChatbotAIService'; import { chatResultFeedbackKindToTelemetryValue, TelemetryEventTypes, } from '../telemetry/telemetryService'; import type TelemetryService from '../telemetry/telemetryService'; +import type { Reference } from 'mongodb-rag-core'; const log = createLogger('participant'); const NUM_DOCUMENTS_TO_SAMPLE = 3; +const MONGODB_DOCS_LINK = 'https://www.mongodb.com/docs/'; + interface NamespaceQuickPicks { label: string; data: string; @@ -73,6 +81,7 @@ export default class ParticipantController { _connectionController: ConnectionController; _storageController: StorageController; _chatMetadataStore: ChatMetadataStore; + _docsChatbotAIService?: DocsChatbotAIService; _telemetryService: TelemetryService; constructor({ @@ -90,6 +99,37 @@ export default class ParticipantController { this._telemetryService = telemetryService; } + // To integrate with the MongoDB documentation chatbot, + // set the MONGODB_DOCS_CHATBOT_BASE_URI environment variable when running the extension from a branch. + // This variable is automatically injected during the .vsix build process via GitHub Actions. + async _readDocsChatbotBaseUri( + context: vscode.ExtensionContext + ): Promise { + config({ path: path.join(context.extensionPath, '.env') }); + + try { + const docsChatbotBaseUriFileLocation = path.join( + context.extensionPath, + './constants.json' + ); + // eslint-disable-next-line no-sync + const constantsFile = await fs.readFile( + docsChatbotBaseUriFileLocation, + 'utf8' + ); + const { docsChatbotBaseUri } = JSON.parse(constantsFile) as { + docsChatbotBaseUri?: string; + }; + return docsChatbotBaseUri; + } catch (error) { + log.error( + 'Failed to read docsChatbotBaseUri from the constants file', + error + ); + return; + } + } + createParticipant(context: vscode.ExtensionContext): vscode.ChatParticipant { // Chat participants appear as top-level options in the chat input // when you type `@`, and can contribute sub-commands in the chat input @@ -110,6 +150,11 @@ export default class ParticipantController { return this._participant; } + async createDocsChatbot(context: vscode.ExtensionContext): Promise { + const docsChatbotBaseUri = await this._readDocsChatbotBaseUri(context); + this._docsChatbotAIService = new DocsChatbotAIService(docsChatbotBaseUri); + } + getParticipant(): vscode.ChatParticipant | undefined { return this._participant; } @@ -175,6 +220,31 @@ export default class ParticipantController { return responseContent; } + _streamRunnableContentActions({ + responseContent, + stream, + }: { + responseContent: string; + stream: vscode.ChatResponseStream; + }): void { + const runnableContent = getRunnableContentFromString(responseContent); + if (runnableContent) { + const commandArgs: RunParticipantQueryCommandArgs = { + runnableContent, + }; + stream.button({ + command: EXTENSION_COMMANDS.RUN_PARTICIPANT_QUERY, + title: vscode.l10n.t('▶️ Run'), + arguments: [commandArgs], + }); + stream.button({ + command: EXTENSION_COMMANDS.OPEN_PARTICIPANT_QUERY_IN_PLAYGROUND, + title: vscode.l10n.t('Open in playground'), + arguments: [commandArgs], + }); + } + } + // @MongoDB what is mongodb? async handleGenericRequest( request: vscode.ChatRequest, @@ -198,22 +268,10 @@ export default class ParticipantController { }); stream.markdown(responseContent); - const runnableContent = getRunnableContentFromString(responseContent); - if (runnableContent) { - const commandArgs: RunParticipantQueryCommandArgs = { - runnableContent, - }; - stream.button({ - command: EXTENSION_COMMANDS.RUN_PARTICIPANT_QUERY, - title: vscode.l10n.t('▶️ Run'), - arguments: [commandArgs], - }); - stream.button({ - command: EXTENSION_COMMANDS.OPEN_PARTICIPANT_QUERY_IN_PLAYGROUND, - title: vscode.l10n.t('Open in playground'), - arguments: [commandArgs], - }); - } + this._streamRunnableContentActions({ + responseContent, + stream, + }); return genericRequestChatResult(context.history); } @@ -730,24 +788,148 @@ export default class ParticipantController { stream.markdown(responseContent); - const runnableContent = getRunnableContentFromString(responseContent); - if (runnableContent) { - const commandArgs: RunParticipantQueryCommandArgs = { - runnableContent, - }; - stream.button({ - command: EXTENSION_COMMANDS.RUN_PARTICIPANT_QUERY, - title: vscode.l10n.t('▶️ Run'), - arguments: [commandArgs], + this._streamRunnableContentActions({ + responseContent, + stream, + }); + + return queryRequestChatResult(context.history); + } + + async _handleDocsRequestWithChatbot({ + docsChatbotAIService, + prompt, + chatId, + }: { + docsChatbotAIService: DocsChatbotAIService; + prompt: string; + chatId: string; + }): Promise<{ + responseContent: string; + responseReferences?: Reference[]; + }> { + let { docsChatbotConversationId } = + this._chatMetadataStore.getChatMetadata(chatId) ?? {}; + if (!docsChatbotConversationId) { + const conversation = await docsChatbotAIService.createConversation(); + docsChatbotConversationId = conversation._id; + this._chatMetadataStore.setChatMetadata(chatId, { + docsChatbotConversationId, }); - stream.button({ - command: EXTENSION_COMMANDS.OPEN_PARTICIPANT_QUERY_IN_PLAYGROUND, - title: vscode.l10n.t('Open in playground'), - arguments: [commandArgs], + } + + const response = await docsChatbotAIService.addMessage({ + message: prompt, + conversationId: docsChatbotConversationId, + }); + + return { + responseContent: response.content, + responseReferences: response.references, + }; + } + + async _handleDocsRequestWithCopilot( + ...args: [ + vscode.ChatRequest, + vscode.ChatContext, + vscode.ChatResponseStream, + vscode.CancellationToken + ] + ): Promise<{ + responseContent: string; + responseReferences?: Reference[]; + }> { + const [request, context, stream, token] = args; + const messages = GenericPrompt.buildMessages({ + request, + context, + }); + + const abortController = new AbortController(); + token.onCancellationRequested(() => { + abortController.abort(); + }); + const responseContent = await this.getChatResponseContent({ + messages, + stream, + token, + }); + const responseReferences = [ + { + url: MONGODB_DOCS_LINK, + title: 'View MongoDB documentation', + }, + ]; + + return { + responseContent, + responseReferences, + }; + } + + async handleDocsRequest( + ...args: [ + vscode.ChatRequest, + vscode.ChatContext, + vscode.ChatResponseStream, + vscode.CancellationToken + ] + ): Promise { + const [request, context, stream, token] = args; + const abortController = new AbortController(); + token.onCancellationRequested(() => { + abortController.abort(); + }); + + const chatId = ChatMetadataStore.getChatIdFromHistoryOrNewChatId( + context.history + ); + + let docsChatbotHasThrownError = false; + let docsResult: { + responseContent?: string; + responseReferences?: Reference[]; + } = {}; + + if (this._docsChatbotAIService) { + try { + docsResult = await this._handleDocsRequestWithChatbot({ + docsChatbotAIService: this._docsChatbotAIService, + prompt: request.prompt, + chatId, + }); + } catch (error) { + // If the docs chatbot API is not available, fall back to Copilot’s LLM and include + // the MongoDB documentation link for users to go to our documentation site directly. + docsChatbotHasThrownError = true; + log.error(error); + } + } + + if (!this._docsChatbotAIService || docsChatbotHasThrownError) { + docsResult = await this._handleDocsRequestWithCopilot(...args); + } + + if (docsResult.responseContent) { + stream.markdown(docsResult.responseContent); + this._streamRunnableContentActions({ + responseContent: docsResult.responseContent, + stream, }); } - return queryRequestChatResult(context.history); + if (docsResult.responseReferences) { + for (const ref of docsResult.responseReferences) { + const link = new vscode.MarkdownString( + `- ${ref.title}\n` + ); + link.supportHtml = true; + stream.markdown(link); + } + } + + return docsRequestChatResult(chatId); } async chatHandler( @@ -792,7 +974,7 @@ export default class ParticipantController { if (request.command === 'query') { return await this.handleQueryRequest(...args); } else if (request.command === 'docs') { - // TODO(VSCODE-570): Implement this. + return await this.handleDocsRequest(...args); } else if (request.command === 'schema') { // TODO(VSCODE-571): Implement this. } diff --git a/src/telemetry/telemetryService.ts b/src/telemetry/telemetryService.ts index f7ff48731..41fe33b4c 100644 --- a/src/telemetry/telemetryService.ts +++ b/src/telemetry/telemetryService.ts @@ -187,13 +187,12 @@ export default class TelemetryService { ); // eslint-disable-next-line no-sync const constantsFile = fs.readFileSync(segmentKeyFileLocation, 'utf8'); - const constants = JSON.parse(constantsFile) as { segmentKey: string }; - - log.info('SegmentKey was found', { type: typeof constants.segmentKey }); - - return constants.segmentKey; + const { segmentKey } = JSON.parse(constantsFile) as { + segmentKey?: string; + }; + return segmentKey; } catch (error) { - log.error('SegmentKey was not found', error); + log.error('Failed to read segmentKey from the constants file', error); return; } } @@ -275,7 +274,7 @@ export default class TelemetryService { async _getConnectionTelemetryProperties( dataService: DataService, connectionType: ConnectionTypes - ) { + ): Promise { return await getConnectionTelemetryProperties(dataService, connectionType); } @@ -323,7 +322,9 @@ export default class TelemetryService { return 'other'; } - getTelemetryUserIdentity() { + getTelemetryUserIdentity(): { + anonymousId: string; + } { return { anonymousId: this._segmentAnonymousId, }; @@ -388,7 +389,7 @@ export default class TelemetryService { trackSavedConnectionsLoaded( savedConnectionsLoadedProps: SavedConnectionsLoadedProperties - ) { + ): void { this.track( TelemetryEventTypes.SAVED_CONNECTIONS_LOADED, savedConnectionsLoadedProps @@ -397,7 +398,7 @@ export default class TelemetryService { trackKeytarSecretsMigrationFailed( keytarSecretsMigrationFailedProps: KeytarSecretsMigrationFailedProperties - ) { + ): void { this.track( TelemetryEventTypes.KEYTAR_SECRETS_MIGRATION_FAILED, keytarSecretsMigrationFailedProps diff --git a/src/test/suite/participant/docsChatbotAIService.test.ts b/src/test/suite/participant/docsChatbotAIService.test.ts new file mode 100644 index 000000000..6860c7a0b --- /dev/null +++ b/src/test/suite/participant/docsChatbotAIService.test.ts @@ -0,0 +1,108 @@ +import { beforeEach, afterEach } from 'mocha'; +import { expect } from 'chai'; +import sinon from 'sinon'; + +import { DocsChatbotAIService } from '../../../participant/docsChatbotAIService'; + +suite('DocsChatbotAIService Test Suite', function () { + const initialFetch = global.fetch; + + afterEach(function () { + global.fetch = initialFetch; + sinon.restore(); + }); + + suite('when serverBaseUri is missing', function () { + test('DocsChatbotAIService constructor does not throw', () => { + const docsChatbotAIService = new DocsChatbotAIService(); + expect(docsChatbotAIService._serverBaseUri).to.be.undefined; + }); + + test('createConversation throws if serverBaseUri is not set', async () => { + const docsChatbotAIService = new DocsChatbotAIService(); + try { + await docsChatbotAIService.createConversation(); + expect.fail('It must fail with missing serverBaseUri'); + } catch (error) { + expect((error as Error).message).to.include( + 'You must define a serverBaseUri for the DocsChatbotAIService' + ); + } + }); + }); + + suite('when serverBaseUri is present', function () { + const serverBaseUri = 'https://example.com/'; + let docsChatbotAIService: DocsChatbotAIService; + + beforeEach(() => { + docsChatbotAIService = new DocsChatbotAIService(serverBaseUri); + }); + + test('creates conversations', async () => { + const fetchStub = sinon.stub().resolves({ + status: 200, + ok: true, + json: () => + Promise.resolve({ + _id: '650b4b260f975ef031016c8a', + messages: [], + }), + }); + global.fetch = fetchStub; + const conversation = await docsChatbotAIService.createConversation(); + expect(conversation._id).to.be.eql('650b4b260f975ef031016c8a'); + }); + + test('throws on server errors', async () => { + const fetchStub = sinon.stub().resolves({ + status: 500, + ok: false, + statusText: 'Server error', + json: sinon.stub().rejects(new Error('invalid json')), + }); + global.fetch = fetchStub; + + try { + await docsChatbotAIService.createConversation(); + expect.fail('It must fail with the server error'); + } catch (error) { + expect((error as Error).message).to.include('Internal server error'); + } + }); + + test('throws on bad requests', async () => { + const fetchStub = sinon.stub().resolves({ + status: 400, + ok: false, + statusText: 'Client error', + json: sinon.stub().resolves({}), + }); + global.fetch = fetchStub; + + try { + await docsChatbotAIService.createConversation(); + expect.fail('It must fail with the bad request error'); + } catch (error) { + expect((error as Error).message).to.include('Bad request'); + } + }); + + test('throws on a rate limit', async () => { + const fetchStub = sinon.stub().resolves({ + status: 429, + ok: false, + statusText: 'Model error', + json: sinon.stub().resolves({}), + }); + global.fetch = fetchStub; + + try { + await docsChatbotAIService.createConversation(); + expect.fail('It must fail with the rate limited error'); + } catch (error) { + expect((error as Error).message).to.include('Rate limited'); + } + }); + }); +}); diff --git a/src/test/suite/participant/participant.test.ts b/src/test/suite/participant/participant.test.ts index 7ea3849c3..354a95a95 100644 --- a/src/test/suite/participant/participant.test.ts +++ b/src/test/suite/participant/participant.test.ts @@ -1055,6 +1055,62 @@ suite('Participant Controller Test Suite', function () { }); }); }); + + suite('docs command', function () { + const initialFetch = global.fetch; + let fetchStub; + + afterEach(function () { + global.fetch = initialFetch; + sinon.restore(); + }); + + beforeEach(function () { + fetchStub = sinon.stub().resolves({ + status: 200, + ok: true, + json: () => + Promise.resolve({ + _id: '650b4b260f975ef031016c8a', + messages: [], + }), + }); + global.fetch = fetchStub; + sendRequestStub.onCall(0).resolves({ + text: ['connection info'], + }); + }); + + test('uses docs chatbot when it is available', async function () { + sinon.replace( + testParticipantController, + '_readDocsChatbotBaseUri', + sinon.stub().resolves('url') + ); + await testParticipantController.createDocsChatbot( + extensionContextStub + ); + const chatRequestMock = { + prompt: 'how to connect to mongodb', + command: 'docs', + references: [], + }; + await invokeChatHandler(chatRequestMock); + expect(fetchStub).to.have.been.called; + expect(sendRequestStub).to.have.not.been.called; + }); + + test('falls back to the copilot model when docs chatbot api is not available', async function () { + const chatRequestMock = { + prompt: 'how to connect to mongodb', + command: 'docs', + references: [], + }; + await invokeChatHandler(chatRequestMock); + expect(fetchStub).to.have.not.been.called; + expect(sendRequestStub).to.have.been.called; + }); + }); }); });