From f374b14083520b59ddc6438ae5f5a4592d62da4d Mon Sep 17 00:00:00 2001 From: Sergey Petushkov Date: Wed, 22 Mar 2023 18:20:55 +0100 Subject: [PATCH 1/2] feat(mongodb-constants): add filter method --- package-lock.json | 40 ++++ packages/mongodb-constants/package.json | 4 + packages/mongodb-constants/src/filter.spec.ts | 146 +++++++++++++ packages/mongodb-constants/src/filter.ts | 201 ++++++++++++++++++ packages/mongodb-constants/src/index.ts | 1 + 5 files changed, 392 insertions(+) create mode 100644 packages/mongodb-constants/src/filter.spec.ts create mode 100644 packages/mongodb-constants/src/filter.ts diff --git a/package-lock.json b/package-lock.json index a379fe3c..5adef3a3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3488,6 +3488,12 @@ "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.0.tgz", "integrity": "sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA==" }, + "node_modules/@types/semver": { + "version": "7.3.13", + "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.3.13.tgz", + "integrity": "sha512-21cFJr9z3g5dW8B0CVI9g2O9beqaThGQ6ZFBqHfwhzLDKUxaqTIy3vnfah/UPkfOiF2pLq+tGz+W8RyCskuslw==", + "dev": true + }, "node_modules/@types/sinon": { "version": "10.0.13", "resolved": "https://registry.npmjs.org/@types/sinon/-/sinon-10.0.13.tgz", @@ -13218,6 +13224,9 @@ "name": "@mongodb-js/mongodb-constants", "version": "0.2.2", "license": "Apache-2.0", + "dependencies": { + "semver": "^7.3.8" + }, "devDependencies": { "@mongodb-js/eslint-config-devtools": "0.9.3", "@mongodb-js/mocha-config-compass": "^0.10.0", @@ -13225,6 +13234,7 @@ "@mongodb-js/tsconfig-compass": "^0.6.0", "@types/chai": "^4.2.21", "@types/mocha": "^9.0.0", + "@types/semver": "^7.3.13", "@types/sinon-chai": "^3.2.5", "acorn": "^8.8.0", "chai": "^4.3.6", @@ -13250,6 +13260,20 @@ "node": ">=0.4.0" } }, + "packages/mongodb-constants/node_modules/semver": { + "version": "7.3.8", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz", + "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==", + "dependencies": { + "lru-cache": "^6.0.0" + }, + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, "scripts": { "version": "0.1.17", "license": "SSPL", @@ -15526,6 +15550,7 @@ "@mongodb-js/tsconfig-compass": "^0.6.0", "@types/chai": "^4.2.21", "@types/mocha": "^9.0.0", + "@types/semver": "*", "@types/sinon-chai": "^3.2.5", "acorn": "^8.8.0", "chai": "^4.3.6", @@ -15535,6 +15560,7 @@ "mocha": "^8.4.0", "nyc": "^15.1.0", "prettier": "2.3.2", + "semver": "^7.3.8", "sinon": "^9.2.3", "typescript": "^4.3.5" }, @@ -15544,6 +15570,14 @@ "resolved": "https://registry.npmjs.org/acorn/-/acorn-8.8.0.tgz", "integrity": "sha512-QOxyigPVrpZ2GXT+PFyZTl6TtOFc5egxHIP9IlQ+RbupQuX4RkT/Bee4/kQuC02Xkzg84JcT7oLYtDIQxp+v7w==", "dev": true + }, + "semver": { + "version": "7.3.8", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.3.8.tgz", + "integrity": "sha512-NB1ctGL5rlHrPJtFDVIVzTyQylMLu9N9VICA6HSFJo8MCGVTMW6gfpicwKmmK/dAjTOrqu5l63JJOpDSrAis3A==", + "requires": { + "lru-cache": "^6.0.0" + } } } }, @@ -15931,6 +15965,12 @@ "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.0.tgz", "integrity": "sha512-//oorEZjL6sbPcKUaCdIGlIUeH26mgzimjBB77G6XRgnDl/L5wOnpyBGRe/Mmf5CVW3PwEBE1NjiMZ/ssFh4wA==" }, + "@types/semver": { + "version": "7.3.13", + "resolved": "https://registry.npmjs.org/@types/semver/-/semver-7.3.13.tgz", + "integrity": "sha512-21cFJr9z3g5dW8B0CVI9g2O9beqaThGQ6ZFBqHfwhzLDKUxaqTIy3vnfah/UPkfOiF2pLq+tGz+W8RyCskuslw==", + "dev": true + }, "@types/sinon": { "version": "10.0.13", "resolved": "https://registry.npmjs.org/@types/sinon/-/sinon-10.0.13.tgz", diff --git a/packages/mongodb-constants/package.json b/packages/mongodb-constants/package.json index 7b8e7acb..a4d0de99 100644 --- a/packages/mongodb-constants/package.json +++ b/packages/mongodb-constants/package.json @@ -52,6 +52,7 @@ "@mongodb-js/tsconfig-compass": "^0.6.0", "@types/chai": "^4.2.21", "@types/mocha": "^9.0.0", + "@types/semver": "^7.3.13", "@types/sinon-chai": "^3.2.5", "acorn": "^8.8.0", "chai": "^4.3.6", @@ -63,5 +64,8 @@ "prettier": "2.3.2", "sinon": "^9.2.3", "typescript": "^4.3.5" + }, + "dependencies": { + "semver": "^7.3.8" } } diff --git a/packages/mongodb-constants/src/filter.spec.ts b/packages/mongodb-constants/src/filter.spec.ts new file mode 100644 index 00000000..8259e349 --- /dev/null +++ b/packages/mongodb-constants/src/filter.spec.ts @@ -0,0 +1,146 @@ +import { expect } from 'chai'; +import type { Completion, Meta } from './filter'; +import { wrapField, filter } from './filter'; + +describe('completer', function () { + const simpleConstants: Completion[] = [ + { value: 'foo', version: '0.0.0', meta: 'stage' }, + { value: 'Foo', version: '0.0.0', meta: 'stage' }, + { value: 'bar', version: '1.0.0', meta: 'accumulator' }, + { value: 'buz', version: '2.0.0', meta: 'expr:array' }, + { value: 'barbar', version: '2.0.0', meta: 'expr:bool' }, + ]; + + function getFilteredValues(...args: Parameters): string[] { + return filter(args[0], args[1] ?? simpleConstants).map( + (completion) => completion.value + ); + } + + it('should return results filtered by server version', function () { + expect(getFilteredValues({ serverVersion: '1.0.0' })).to.deep.eq([ + 'foo', + 'Foo', + 'bar', + ]); + expect(getFilteredValues({ serverVersion: '0.0.1-alpha0' })).to.deep.eq([ + 'foo', + 'Foo', + ]); + }); + + it('should return results filtered by meta', function () { + expect(getFilteredValues({ meta: ['stage', 'accumulator'] })).to.deep.eq([ + 'foo', + 'Foo', + 'bar', + ]); + expect(getFilteredValues({ meta: ['expr:*'] })).to.deep.eq([ + 'buz', + 'barbar', + ]); + }); + + describe('stage filter', function () { + const stageConstants: Completion[] = [ + { + value: '$a', + version: '0.0.0', + meta: 'stage', + env: ['adl'], + namespace: ['database'], + apiVersions: [], + }, + { + value: '$b', + version: '0.0.0', + meta: 'stage', + env: ['on-prem'], + namespace: ['collection'], + apiVersions: [], + }, + { + value: '$c', + version: '0.0.0', + meta: 'stage', + env: ['atlas'], + namespace: ['timeseries'], + apiVersions: [1], + }, + ]; + + it('should return all constants when stage filters are not provided', function () { + expect(getFilteredValues({}, stageConstants)).to.deep.eq([ + '$a', + '$b', + '$c', + ]); + }); + + it('should filter stages by env', function () { + expect( + getFilteredValues({ stage: { env: ['adl', 'atlas'] } }, stageConstants) + ).to.deep.eq(['$a', '$c']); + }); + + it('should filter stages by namespace', function () { + expect( + getFilteredValues( + { stage: { namespace: 'collection' } }, + stageConstants + ) + ).to.deep.eq(['$b']); + }); + + it('should filter stages by apiVersion', function () { + expect( + getFilteredValues({ stage: { apiVersion: 1 } }, stageConstants) + ).to.deep.eq(['$c']); + }); + }); + + it('should keep field description when provided', function () { + const completions = filter( + { + meta: ['field:identifier'], + fields: [ + { name: 'foo', description: 'ObjectId' }, + { name: 'bar', description: 'Int32' }, + ], + }, + [] + ).map((completion) => { + return { + value: completion.value, + description: completion.description, + }; + }); + expect(completions).to.deep.eq([ + { value: 'foo', description: 'ObjectId' }, + { value: 'bar', description: 'Int32' }, + ]); + }); + + describe('wrapField', function () { + it('should leave identifier as-is if its roughly valid', function () { + expect(wrapField('foo')).to.eq('foo'); + expect(wrapField('bar_buz')).to.eq('bar_buz'); + expect(wrapField('$something')).to.eq('$something'); + expect(wrapField('_or_other')).to.eq('_or_other'); + expect(wrapField('number1')).to.eq('number1'); + }); + + it("should wrap field in quotes when it's rougly not a valid js identifier", function () { + expect(wrapField('123foobar')).to.eq('"123foobar"'); + expect(wrapField('bar@buz')).to.eq('"bar@buz"'); + expect(wrapField('foo bar')).to.eq('"foo bar"'); + expect(wrapField('with.a.dot')).to.eq('"with.a.dot"'); + expect(wrapField('bla; process.exit(1); var foo')).to.eq( + '"bla; process.exit(1); var foo"' + ); + expect(wrapField('quotes"in"the"middle')).to.eq( + '"quotes\\"in\\"the\\"middle"' + ); + }); + }); +}); diff --git a/packages/mongodb-constants/src/filter.ts b/packages/mongodb-constants/src/filter.ts new file mode 100644 index 00000000..983bee8e --- /dev/null +++ b/packages/mongodb-constants/src/filter.ts @@ -0,0 +1,201 @@ +import { gte } from 'semver'; +import { ACCUMULATORS } from './accumulators'; +import { BSON_TYPE_ALIASES } from './bson-type-aliases'; +import { BSON_TYPES } from './bson-types'; +import { CONVERSION_OPERATORS } from './conversion-operators'; +import { EXPRESSION_OPERATORS } from './expression-operators'; +import { JSON_SCHEMA } from './json-schema'; +import { QUERY_OPERATORS } from './query-operators'; +import { STAGE_OPERATORS } from './stage-operators'; + +const ALL_CONSTANTS = [ + ...ACCUMULATORS, + ...BSON_TYPES, + ...BSON_TYPE_ALIASES, + ...CONVERSION_OPERATORS, + ...EXPRESSION_OPERATORS, + ...JSON_SCHEMA, + ...QUERY_OPERATORS, + ...STAGE_OPERATORS, +]; + +export type Meta = + | typeof ALL_CONSTANTS[number]['meta'] + | 'field:identifier' + | 'field:reference'; + +/** + * Our completions are a mix of ace autocompleter types and some custom values + * added on top, this interface provides a type definition for all required + * properties that completer is using + * + * @internal + */ +export type Completion = { + value: string; + version: string; + meta: Meta; + description?: string; + comment?: string; + snippet?: string; + score?: number; + env?: string[]; + namespace?: string[]; + apiVersions?: number[]; + outputStage?: boolean; + fullScan?: boolean; + firstStage?: boolean; + geospatial?: boolean; +}; + +export type StageFilterOptions = { + env?: string | string[]; + namespace?: string | string[]; + apiVersion?: number | number[]; +}; + +export type FilterOptions = { + /** + * Current server version (default is 999.999.999) + */ + serverVersion?: string; + /** + * Additional fields that are part of the document schema to add to + * autocomplete as identifiers and identifier references + */ + fields?: (string | { name: string; description?: string })[]; + /** + * Filter completions by completion category + */ + meta?: (Meta | 'field:*' | 'accumulator:*' | 'expr:*')[]; + /** + * Stage-only filters + */ + stage?: StageFilterOptions; +}; + +function matchesMeta(filter: string[], meta: string) { + const metaParts = meta.split(':'); + return filter.some((metaFilter) => { + const filterParts = metaFilter.split(':'); + return ( + filterParts.length === metaParts.length && + filterParts.every((part, index) => { + return part === '*' || part === metaParts[index]; + }) + ); + }); +} + +function isIn( + val: T | T[] | undefined, + set: T[] | undefined +): boolean { + // Do not filter when either value or match set is not provided + if (typeof val === 'undefined' || typeof set === 'undefined') { + return true; + } + // Otherwise check that value intersects with the match set + val = Array.isArray(val) ? val : [val]; + return val.some((v) => set.includes(v)); +} + +export function createConstantFilter({ + meta: filterMeta, + serverVersion = '999.999.999', + stage: filterStage = {}, +}: Pick = {}): ( + completion: Completion +) => boolean { + const currentServerVersion = + /^(?\d+?\.\d+?\.\d+?)/.exec(serverVersion)?.groups?.version ?? + serverVersion; + return ({ version: minServerVersion, meta, env, namespace, apiVersions }) => { + return ( + gte(currentServerVersion, minServerVersion) && + isIn(filterStage.env, env) && + isIn(filterStage.namespace, namespace) && + isIn(filterStage.apiVersion, apiVersions) && + (!filterMeta || matchesMeta(filterMeta, meta)) + ); + }; +} + +function isValidIdentifier(identifier: string) { + // Quick check for common case first + if (/[.\s"'()[\];={}:]/.test(identifier)) { + return false; + } + try { + // Everything else we check using eval as regex methods of checking are quite + // hard to do (see https://mathiasbynens.be/notes/javascript-identifiers-es6) + // eslint-disable-next-line @typescript-eslint/no-implied-eval + new Function(`"use strict";let _ = { ${identifier}: 0 };`); + return true; + } catch { + return false; + } +} + +/** + * Helper method to conditionally wrap value if it's not a valid identifier + */ +export function wrapField(field: string, force = false): string { + return force || !isValidIdentifier(field) + ? `"${field.replace(/["\\]/g, '\\$&')}"` + : field; +} + +function normalizeField( + field: string | { name: string; description?: string } +) { + return typeof field === 'string' + ? { value: field } + : { + value: field.name, + description: field.description, + }; +} + +/** + * Convenience method to filter list of mongodb constants based on constants + * values + * + * @param options filter options + * @param constants list of constants to filter, for testing purposes only + * @returns filtered constants + */ +export function filter( + options: FilterOptions = {}, + constants: Completion[] = ALL_CONSTANTS as Completion[] +): Completion[] { + const { serverVersion = '999.999.999', fields = [], meta, stage } = options; + const completionsFilter = createConstantFilter({ + serverVersion, + meta, + stage, + }); + const completionsWithFields = constants.concat( + fields.flatMap((field) => { + const { value, description } = normalizeField(field); + return [ + { + value: value, + meta: 'field:identifier', + version: '0.0.0', + description, + }, + { + value: `$${value}`, + meta: 'field:reference', + version: '0.0.0', + description, + }, + ]; + }) + ); + + return completionsWithFields.filter((completion) => { + return completionsFilter(completion); + }); +} diff --git a/packages/mongodb-constants/src/index.ts b/packages/mongodb-constants/src/index.ts index 6e27f0bb..ad90788e 100644 --- a/packages/mongodb-constants/src/index.ts +++ b/packages/mongodb-constants/src/index.ts @@ -8,3 +8,4 @@ export * from './json-schema'; export * from './ns'; export * from './query-operators'; export * from './stage-operators'; +export { filter, wrapField } from './filter'; From 1e5545f1b58a047b5b4e067f02e216f8d689ac5f Mon Sep 17 00:00:00 2001 From: Sergey Petushkov Date: Thu, 23 Mar 2023 14:34:12 +0100 Subject: [PATCH 2/2] chore(constants): rename to getFilteredCompletions --- packages/mongodb-constants/src/filter.spec.ts | 22 ++++++++++++++----- packages/mongodb-constants/src/filter.ts | 17 ++++++++++---- packages/mongodb-constants/src/index.spec.ts | 2 ++ packages/mongodb-constants/src/index.ts | 2 +- 4 files changed, 33 insertions(+), 10 deletions(-) diff --git a/packages/mongodb-constants/src/filter.spec.ts b/packages/mongodb-constants/src/filter.spec.ts index 8259e349..469b798d 100644 --- a/packages/mongodb-constants/src/filter.spec.ts +++ b/packages/mongodb-constants/src/filter.spec.ts @@ -1,6 +1,6 @@ import { expect } from 'chai'; -import type { Completion, Meta } from './filter'; -import { wrapField, filter } from './filter'; +import type { Completion } from './filter'; +import { wrapField, getFilteredCompletions } from './filter'; describe('completer', function () { const simpleConstants: Completion[] = [ @@ -11,8 +11,10 @@ describe('completer', function () { { value: 'barbar', version: '2.0.0', meta: 'expr:bool' }, ]; - function getFilteredValues(...args: Parameters): string[] { - return filter(args[0], args[1] ?? simpleConstants).map( + function getFilteredValues( + ...args: Parameters + ): string[] { + return getFilteredCompletions(args[0], args[1] ?? simpleConstants).map( (completion) => completion.value ); } @@ -29,6 +31,16 @@ describe('completer', function () { ]); }); + it('should ignore version when version is not valid', function () { + expect(getFilteredValues({ serverVersion: '1' })).to.deep.eq([ + 'foo', + 'Foo', + 'bar', + 'buz', + 'barbar', + ]); + }); + it('should return results filtered by meta', function () { expect(getFilteredValues({ meta: ['stage', 'accumulator'] })).to.deep.eq([ 'foo', @@ -100,7 +112,7 @@ describe('completer', function () { }); it('should keep field description when provided', function () { - const completions = filter( + const completions = getFilteredCompletions( { meta: ['field:identifier'], fields: [ diff --git a/packages/mongodb-constants/src/filter.ts b/packages/mongodb-constants/src/filter.ts index 983bee8e..43ba09f6 100644 --- a/packages/mongodb-constants/src/filter.ts +++ b/packages/mongodb-constants/src/filter.ts @@ -19,6 +19,8 @@ const ALL_CONSTANTS = [ ...STAGE_OPERATORS, ]; +const DEFAULT_SERVER_VERSION = '999.999.999'; + export type Meta = | typeof ALL_CONSTANTS[number]['meta'] | 'field:identifier' @@ -102,14 +104,16 @@ function isIn( export function createConstantFilter({ meta: filterMeta, - serverVersion = '999.999.999', + serverVersion = DEFAULT_SERVER_VERSION, stage: filterStage = {}, }: Pick = {}): ( completion: Completion ) => boolean { const currentServerVersion = /^(?\d+?\.\d+?\.\d+?)/.exec(serverVersion)?.groups?.version ?? - serverVersion; + // Fallback to default server version if provided version doesn't match + // regex so that semver doesn't throw when checking + DEFAULT_SERVER_VERSION; return ({ version: minServerVersion, meta, env, namespace, apiVersions }) => { return ( gte(currentServerVersion, minServerVersion) && @@ -165,11 +169,16 @@ function normalizeField( * @param constants list of constants to filter, for testing purposes only * @returns filtered constants */ -export function filter( +export function getFilteredCompletions( options: FilterOptions = {}, constants: Completion[] = ALL_CONSTANTS as Completion[] ): Completion[] { - const { serverVersion = '999.999.999', fields = [], meta, stage } = options; + const { + serverVersion = DEFAULT_SERVER_VERSION, + fields = [], + meta, + stage, + } = options; const completionsFilter = createConstantFilter({ serverVersion, meta, diff --git a/packages/mongodb-constants/src/index.spec.ts b/packages/mongodb-constants/src/index.spec.ts index e080cc11..0b332923 100644 --- a/packages/mongodb-constants/src/index.spec.ts +++ b/packages/mongodb-constants/src/index.spec.ts @@ -4,6 +4,8 @@ import * as constants from './index'; describe('constants', function () { it('should export all constants', function () { expect(Object.keys(constants)).to.deep.eq([ + 'getFilteredCompletions', + 'wrapField', 'ACCUMULATORS', 'BSON_TYPE_ALIASES', 'BSON_TYPES', diff --git a/packages/mongodb-constants/src/index.ts b/packages/mongodb-constants/src/index.ts index ad90788e..d42e8cfb 100644 --- a/packages/mongodb-constants/src/index.ts +++ b/packages/mongodb-constants/src/index.ts @@ -8,4 +8,4 @@ export * from './json-schema'; export * from './ns'; export * from './query-operators'; export * from './stage-operators'; -export { filter, wrapField } from './filter'; +export { getFilteredCompletions, wrapField } from './filter';