From 5a522eeaee34e775a6c2a9159c7ba4d0f16aebfe Mon Sep 17 00:00:00 2001 From: Avior Date: Wed, 24 Jul 2024 15:48:28 +0200 Subject: [PATCH 1/6] feat: Add advanced filtering capabilities Signed-off-by: Avior --- .bruno/cards/advanced-query.bru | 25 ++ server/src/V2/Components/Card.ts | 64 ++-- server/src/V2/Components/Serie.ts | 64 ++-- server/src/V2/Components/Set.ts | 60 ++- server/src/V2/endpoints/jsonEndpoints.ts | 94 ++--- server/src/libs/QueryEngine/filter.ts | 461 +++++++++++++++++++++++ server/src/libs/QueryEngine/parsers.ts | 197 ++++++++++ server/src/util.ts | 2 +- 8 files changed, 814 insertions(+), 153 deletions(-) create mode 100644 .bruno/cards/advanced-query.bru create mode 100644 server/src/libs/QueryEngine/filter.ts create mode 100644 server/src/libs/QueryEngine/parsers.ts diff --git a/.bruno/cards/advanced-query.bru b/.bruno/cards/advanced-query.bru new file mode 100644 index 000000000..be0306014 --- /dev/null +++ b/.bruno/cards/advanced-query.bru @@ -0,0 +1,25 @@ +meta { + name: Advanced Query + type: http + seq: 1 +} + +get { + url: {{BASE_URL}}/v2/en/cards?name=eq:Pikachu&hp=gte:60&hp=lt:70&localId=5&localId=not:tg&id=neq:cel25-5 + body: none + auth: none +} + +params:query { + name: eq:Pikachu + hp: gte:60 + hp: lt:70 + localId: 5 + localId: not:tg + id: neq:cel25-5 +} + +assert { + res.status: eq 200 + res.body: length 14 +} diff --git a/server/src/V2/Components/Card.ts b/server/src/V2/Components/Card.ts index 442a5ad92..b7d83c6b8 100644 --- a/server/src/V2/Components/Card.ts +++ b/server/src/V2/Components/Card.ts @@ -1,48 +1,47 @@ import { objectLoop } from '@dzeio/object-util' -import { CardResume, Card as SDKCard, SupportedLanguages } from '@tcgdex/sdk' -import { Query } from '../../interfaces' -import { handlePagination, handleSort, handleValidation } from '../../util' -import Set from './Set' +import type { CardResume, Card as SDKCard, SupportedLanguages } from '@tcgdex/sdk' +import { executeQuery, type Query } from '../../libs/QueryEngine/filter' +import TCGSet from './Set' +import de from '../../../generated/de/cards.json' import en from '../../../generated/en/cards.json' -import fr from '../../../generated/fr/cards.json' import es from '../../../generated/es/cards.json' +import fr from '../../../generated/fr/cards.json' +import id from '../../../generated/id/cards.json' import it from '../../../generated/it/cards.json' -import pt from '../../../generated/pt/cards.json' -import ptbr from '../../../generated/pt-br/cards.json' -import ptpt from '../../../generated/pt-pt/cards.json' -import de from '../../../generated/de/cards.json' +import ja from '../../../generated/ja/cards.json' +import ko from '../../../generated/ko/cards.json' import nl from '../../../generated/nl/cards.json' import pl from '../../../generated/pl/cards.json' +import ptbr from '../../../generated/pt-br/cards.json' +import ptpt from '../../../generated/pt-pt/cards.json' +import pt from '../../../generated/pt/cards.json' import ru from '../../../generated/ru/cards.json' -import ja from '../../../generated/ja/cards.json' -import ko from '../../../generated/ko/cards.json' -import zhtw from '../../../generated/zh-tw/cards.json' -import id from '../../../generated/id/cards.json' import th from '../../../generated/th/cards.json' import zhcn from '../../../generated/zh-cn/cards.json' +import zhtw from '../../../generated/zh-tw/cards.json' const cards = { - 'en': en, - 'fr': fr, - 'es': es, - 'it': it, - 'pt': pt, + en: en, + fr: fr, + es: es, + it: it, + pt: pt, 'pt-br': ptbr, 'pt-pt': ptpt, - 'de': de, - 'nl': nl, - 'pl': pl, - 'ru': ru, - 'ja': ja, - 'ko': ko, + de: de, + nl: nl, + pl: pl, + ru: ru, + ja: ja, + ko: ko, 'zh-tw': zhtw, - 'id': id, - 'th': th, + id: id, + th: th, 'zh-cn': zhcn, } as const -type LocalCard = Omit & {set: () => Set} +type LocalCard = Omit & {set: () => TCGSet} interface variants { normal?: boolean; @@ -93,8 +92,8 @@ export default class Card implements LocalCard { }) } - public set(): Set { - return Set.findOne(this.lang, {filters: { id: this.card.set.id }}) as Set + public set(): TCGSet { + return TCGSet.findOne(this.lang, { id: this.card.set.id }) as TCGSet } public static getAll(lang: SupportedLanguages): Array { @@ -102,16 +101,15 @@ export default class Card implements LocalCard { } public static find(lang: SupportedLanguages, query: Query) { - return handlePagination(handleSort(handleValidation(this.getAll(lang), query), query), query) - .map((it) => new Card(lang, it)) + return executeQuery(Card.getAll(lang), query).data.map((it) => new Card(lang, it)) } public static findOne(lang: SupportedLanguages, query: Query) { - const res = handleSort(handleValidation(this.getAll(lang), query), query) + const res = Card.find(lang, query) if (res.length === 0) { return undefined } - return new Card(lang, res[0]) + return res[0] } public resume(): CardResume { diff --git a/server/src/V2/Components/Serie.ts b/server/src/V2/Components/Serie.ts index edead5415..cb8d713f5 100644 --- a/server/src/V2/Components/Serie.ts +++ b/server/src/V2/Components/Serie.ts @@ -1,49 +1,48 @@ import { objectLoop } from '@dzeio/object-util' -import { Serie as SDKSerie, SerieResume, SupportedLanguages } from '@tcgdex/sdk' -import { Query } from '../../interfaces' -import { handlePagination, handleSort, handleValidation } from '../../util' -import Set from './Set' +import type { Serie as SDKSerie, SerieResume, SupportedLanguages } from '@tcgdex/sdk' +import { executeQuery, type Query } from '../../libs/QueryEngine/filter' +import TCGSet from './Set' +import de from '../../../generated/de/series.json' import en from '../../../generated/en/series.json' -import fr from '../../../generated/fr/series.json' import es from '../../../generated/es/series.json' +import fr from '../../../generated/fr/series.json' +import id from '../../../generated/id/series.json' import it from '../../../generated/it/series.json' -import pt from '../../../generated/pt/series.json' -import ptbr from '../../../generated/pt-br/series.json' -import ptpt from '../../../generated/pt-pt/series.json' -import de from '../../../generated/de/series.json' +import ja from '../../../generated/ja/series.json' +import ko from '../../../generated/ko/series.json' import nl from '../../../generated/nl/series.json' import pl from '../../../generated/pl/series.json' +import ptbr from '../../../generated/pt-br/series.json' +import ptpt from '../../../generated/pt-pt/series.json' +import pt from '../../../generated/pt/series.json' import ru from '../../../generated/ru/series.json' -import ja from '../../../generated/ja/series.json' -import ko from '../../../generated/ko/series.json' -import zhtw from '../../../generated/zh-tw/series.json' -import id from '../../../generated/id/series.json' import th from '../../../generated/th/series.json' import zhcn from '../../../generated/zh-cn/series.json' +import zhtw from '../../../generated/zh-tw/series.json' const series = { - 'en': en, - 'fr': fr, - 'es': es, - 'it': it, - 'pt': pt, + en: en, + fr: fr, + es: es, + it: it, + pt: pt, 'pt-br': ptbr, 'pt-pt': ptpt, - 'de': de, - 'nl': nl, - 'pl': pl, - 'ru': ru, - 'ja': ja, - 'ko': ko, + de: de, + nl: nl, + pl: pl, + ru: ru, + ja: ja, + ko: ko, 'zh-tw': zhtw, - 'id': id, - 'th': th, + id: id, + th: th, 'zh-cn': zhcn, } as const -type LocalSerie = Omit & {sets: () => Array} +type LocalSerie = Omit & {sets: () => Array} export default class Serie implements LocalSerie { @@ -63,8 +62,8 @@ export default class Serie implements LocalSerie { }) } - public sets(): Array { - return this.serie.sets.map((s) => Set.findOne(this.lang, {filters: { id: s.id }}) as Set) + public sets(): Array { + return this.serie.sets.map((s) => TCGSet.findOne(this.lang, { id: s.id }) as TCGSet) } public static getAll(lang: SupportedLanguages): Array { @@ -72,16 +71,15 @@ export default class Serie implements LocalSerie { } public static find(lang: SupportedLanguages, query: Query) { - return handlePagination(handleSort(handleValidation(this.getAll(lang), query), query), query) - .map((it) => new Serie(lang, it)) + return executeQuery(Serie.getAll(lang), query).data.map((it) => new Serie(lang, it)) } public static findOne(lang: SupportedLanguages, query: Query) { - const res = handleValidation(this.getAll(lang), query) + const res = Serie.find(lang, query) if (res.length === 0) { return undefined } - return new Serie(lang, res[0]) + return res[0] } public resume(): SerieResume { diff --git a/server/src/V2/Components/Set.ts b/server/src/V2/Components/Set.ts index 5f7de8745..9efb3f9e9 100644 --- a/server/src/V2/Components/Set.ts +++ b/server/src/V2/Components/Set.ts @@ -1,45 +1,44 @@ import { objectLoop } from '@dzeio/object-util' -import { Set as SDKSet, SetResume, SupportedLanguages } from '@tcgdex/sdk' -import { Query } from '../../interfaces' -import { handlePagination, handleSort, handleValidation } from '../../util' +import type { Set as SDKSet, SetResume, SupportedLanguages } from '@tcgdex/sdk' +import { executeQuery, type Query } from '../../libs/QueryEngine/filter' import Card from './Card' import Serie from './Serie' +import de from '../../../generated/de/sets.json' import en from '../../../generated/en/sets.json' -import fr from '../../../generated/fr/sets.json' import es from '../../../generated/es/sets.json' +import fr from '../../../generated/fr/sets.json' +import id from '../../../generated/id/sets.json' import it from '../../../generated/it/sets.json' -import pt from '../../../generated/pt/sets.json' -import ptbr from '../../../generated/pt-br/sets.json' -import ptpt from '../../../generated/pt-pt/sets.json' -import de from '../../../generated/de/sets.json' +import ja from '../../../generated/ja/sets.json' +import ko from '../../../generated/ko/sets.json' import nl from '../../../generated/nl/sets.json' import pl from '../../../generated/pl/sets.json' +import ptbr from '../../../generated/pt-br/sets.json' +import ptpt from '../../../generated/pt-pt/sets.json' +import pt from '../../../generated/pt/sets.json' import ru from '../../../generated/ru/sets.json' -import ja from '../../../generated/ja/sets.json' -import ko from '../../../generated/ko/sets.json' -import zhtw from '../../../generated/zh-tw/sets.json' -import id from '../../../generated/id/sets.json' import th from '../../../generated/th/sets.json' import zhcn from '../../../generated/zh-cn/sets.json' +import zhtw from '../../../generated/zh-tw/sets.json' const sets = { - 'en': en, - 'fr': fr, - 'es': es, - 'it': it, - 'pt': pt, + en: en, + fr: fr, + es: es, + it: it, + pt: pt, 'pt-br': ptbr, 'pt-pt': ptpt, - 'de': de, - 'nl': nl, - 'pl': pl, - 'ru': ru, - 'ja': ja, - 'ko': ko, + de: de, + nl: nl, + pl: pl, + ru: ru, + ja: ja, + ko: ko, 'zh-tw': zhtw, - 'id': id, - 'th': th, + id: id, + th: th, 'zh-cn': zhcn, } as const @@ -77,11 +76,11 @@ export default class Set implements LocalSet { symbol?: string | undefined public serie(): Serie { - return Serie.findOne(this.lang, {filters: { id: this.set.serie.id }}) as Serie + return Serie.findOne(this.lang, { id: this.set.serie.id }) as Serie } public cards(): Array { - return this.set.cards.map((s) => Card.findOne(this.lang, { filters: { id: s.id }}) as Card) + return this.set.cards.map((s) => Card.findOne(this.lang, { id: s.id }) as Card) } public static getAll(lang: SupportedLanguages): Array { @@ -89,16 +88,15 @@ export default class Set implements LocalSet { } public static find(lang: SupportedLanguages, query: Query) { - return handlePagination(handleSort(handleValidation(this.getAll(lang), query), query), query) - .map((it) => new Set(lang, it)) + return executeQuery(Set.getAll(lang), query).data.map((it) => new Set(lang, it)) } public static findOne(lang: SupportedLanguages, query: Query) { - const res = handleValidation(this.getAll(lang), query) + const res = Set.find(lang, query) if (res.length === 0) { return undefined } - return new Set(lang, res[0]) + return res[0] } public resume(): SetResume { diff --git a/server/src/V2/endpoints/jsonEndpoints.ts b/server/src/V2/endpoints/jsonEndpoints.ts index 1594ff63b..0df44172e 100644 --- a/server/src/V2/endpoints/jsonEndpoints.ts +++ b/server/src/V2/endpoints/jsonEndpoints.ts @@ -1,40 +1,41 @@ -import { objectKeys, objectLoop } from '@dzeio/object-util' -import { Card as SDKCard } from '@tcgdex/sdk' +import { objectKeys } from '@dzeio/object-util' +import type { Card as SDKCard } from '@tcgdex/sdk' import apicache from 'apicache' -import express, { Request } from 'express' -import { Query } from '../../interfaces' +import express, { type Request } from 'express' +import type { Query } from '../../libs/QueryEngine/filter' +import { recordToQuery } from '../../libs/QueryEngine/parsers' import { betterSorter, checkLanguage, sendError, unique } from '../../util' import Card from '../Components/Card' import Serie from '../Components/Serie' -import Set from '../Components/Set' +import TCGSet from '../Components/Set' type CustomRequest = Request & { /** * disable caching */ DO_NOT_CACHE?: boolean - advQuery?: Query + advQuery?: Query } const server = express.Router() const endpointToField: Record = { - "categories": 'category', + categories: 'category', 'energy-types': 'energyType', - "hp": 'hp', - 'illustrators': 'illustrator', - "rarities": 'rarity', + hp: 'hp', + illustrators: 'illustrator', + rarities: 'rarity', 'regulation-marks': 'regulationMark', - "retreats": 'retreat', - "stages": "stage", - "suffixes": "suffix", + retreats: 'retreat', + stages: "stage", + suffixes: "suffix", "trainer-types": "trainerType", // fields that need special care 'dex-ids': 'dexId', - "sets": "set", - "types": "types", - "variants": "variants", + sets: "set", + types: "types", + variants: "variants", } server @@ -65,27 +66,7 @@ server return } - const items: Query = { - filters: undefined, - sort: undefined, - pagination: undefined - } - - objectLoop(req.query as Record>, (value: string | Array, key: string) => { - if (!key.includes(':')) { - key = 'filters:' + key - } - const [cat, item] = key.split(':', 2) as ['filters', string] - if (!items[cat]) { - items[cat] = {} - } - const finalValue = Array.isArray(value) ? value.map((it) => isNaN(parseInt(it)) ? it : parseInt(it)) : isNaN(parseInt(value)) ? value : parseInt(value) - // @ts-expect-error normal behavior - items[cat][item] = finalValue - - }) - - req.advQuery = items + req.advQuery = recordToQuery(req.query as Record>) next() }) @@ -97,24 +78,27 @@ server const { lang, what } = req.params if (!checkLanguage(lang)) { - return sendError('LanguageNotFoundError', res, lang) + sendError('LanguageNotFoundError', res, lang) + return } + // biome-ignore lint/style/noNonNullAssertion: const query: Query = req.advQuery! - let data: Array = [] + let data: Array = [] switch (what.toLowerCase()) { case 'card': data = Card.find(lang, query) break case 'set': - data = Set.find(lang, query) + data = TCGSet.find(lang, query) break case 'serie': data = Serie.find(lang, query) break default: - return sendError('EndpointNotFoundError', res, what) + sendError('EndpointNotFoundError', res, what) + return } const item = Math.min(data.length - 1, Math.max(0, Math.round(Math.random() * data.length))) req.DO_NOT_CACHE = true @@ -129,17 +113,18 @@ server .get('/:lang/:endpoint', (req: CustomRequest, res): void => { let { lang, endpoint } = req.params - const query: Query = req.advQuery! + const query: Query = req.advQuery ?? {} if (endpoint.endsWith('.json')) { endpoint = endpoint.replace('.json', '') } if (!checkLanguage(lang)) { - return sendError('LanguageNotFoundError', res, lang) + sendError('LanguageNotFoundError', res, lang) + return } - let result: any + let result: unknown switch (endpoint) { case 'cards': @@ -149,7 +134,7 @@ server break case 'sets': - result = Set + result = TCGSet .find(lang, query) .map((c) => c.resume()) break @@ -165,7 +150,6 @@ server case "rarities": case "regulation-marks": case "retreats": - case "series": case "stages": case "suffixes": case "trainer-types": @@ -220,26 +204,26 @@ server return sendError('LanguageNotFoundError', res, lang) } - let result: any | undefined + let result: unknown switch (endpoint) { case 'cards': - result = Card.findOne(lang, { filters: { id }, strict: true })?.full() + result = Card.findOne(lang, { id })?.full() if (!result) { - result = Card.findOne(lang, { filters: { name: id }, strict: true })?.full() + result = Card.findOne(lang, { name: id })?.full() } break case 'sets': - result = Set.findOne(lang, { filters: { id }, strict: true })?.full() + result = TCGSet.findOne(lang, { id })?.full() if (!result) { - result = Set.findOne(lang, {filters: { name: id }, strict: true })?.full() + result = TCGSet.findOne(lang, { name: id })?.full() } break case 'series': - result = Serie.findOne(lang, { filters: { id }, strict: true })?.full() + result = Serie.findOne(lang, { id })?.full() if (!result) { - result = Serie.findOne(lang, { filters: { name: id }, strict: true })?.full() + result = Serie.findOne(lang, { name: id })?.full() } break default: @@ -274,12 +258,12 @@ server return sendError('LanguageNotFoundError', res, lang) } - let result: any | undefined + let result: unknown switch (endpoint) { case 'sets': result = Card - .findOne(lang, { filters: { localId: subid, set: id }, strict: true})?.full() + .findOne(lang, { localId: subid, 'set.id': id })?.full() break } if (!result) { diff --git a/server/src/libs/QueryEngine/filter.ts b/server/src/libs/QueryEngine/filter.ts new file mode 100644 index 000000000..275478cb4 --- /dev/null +++ b/server/src/libs/QueryEngine/filter.ts @@ -0,0 +1,461 @@ +import { objectGet, objectKeys, objectLoop, objectSize } from '@dzeio/object-util' +import { isNull } from '../../util' + +interface QueryRootFilters { + /** + * one of the results should be true to be true + */ + $or?: Array> + /** + * every results should be false to be true + */ + $nor?: Array> + /** + * (default) make sure every sub queries return true + */ + $and?: Array> + /** + * at least one result must be false + */ + $nand?: Array> + /** + * invert the result from the following query + */ + $not?: QueryList + + /************** + * PAGINATION * + **************/ + + /** + * define a precise offset of the data you fetched + */ + $offset?: number + /** + * limit the number of elements returned from the dataset + */ + $limit?: number + + /** + * instead of using a precise offset, use a page system + */ + $page?: number + + /********** + * Sorting * + **********/ + + /** + * sort the data the way you want with each keys being priorized + * + * ex: + * {a: Sort.DESC, b: Sort.ASC} + * + * will sort first by a and if equal will sort by b + */ + $sort?: SortInterface +} + +/** + * Logical operators that can be used to filter data + */ +export type QueryLogicalOperator = { + /** + * one of the results should be true to be true + */ + $or: Array> +} | { + /** + * every results should be false to be true + */ + $nor: Array> +} | { + /** + * at least one result must be false + */ + $nand: Array> +} | { + /** + * (default) make sure every sub queries return true + */ + $and: Array> +} | { + /** + * invert the result from the following query + */ + $not: QueryValues +} + +/** + * differents comparisons operators that can be used to filter data + */ +export type QueryComparisonOperator = { + /** + * the remote source value must be absolutelly equal to the proposed value + */ + $eq: Value | null +} | { + /** + * the remote source value must be greater than the proposed value + */ + $gt: number | Date +} | { + /** + * the remote source value must be lesser than the proposed value + */ + $lt: number | Date +} | { + /** + * the remote source value must be greater or equal than the proposed value + */ + $gte: number | Date +} | { + /** + * the remote source value must be lesser or equal than the proposed value + */ + $lte: number | Date +} | { + /** + * the remote source value must be one of the proposed values + */ + $in: Array +} | { + /** + * laxist validation of the remote value + * + * for strings: remote contains value while not following casing like ($lax) `pou` === `Pouet` (remote) + * for numbers: it does a string conversion first + */ + $lax: Value | null +} | { + /** + * (for arrays only) specify a needed length of the array + */ + $len: number | { $gt: number } +} + +export type QueryList = { + [Key in keyof Obj]?: QueryValues +} + +/** + * Differents values the element can take + * if null it will check if it is NULL on the remote + * if array it will check oneOf + * if RegExp it will check if regexp match + */ +export type QueryValues = Value | + null | + Array | + RegExp | + QueryComparisonOperator | + QueryLogicalOperator + +/** + * The query element that allows you to query different elements + */ +export type Query = QueryList & QueryRootFilters + + +// biome-ignore lint/style/useEnumInitializers: +export enum Sort { + /** + * Sort the values from the lowest to the largest + */ + ASC, + /** + * Sort the values form the largest to the lowest + */ + DESC +} + +/** + * sorting interface with priority + */ +export type SortInterface = { + [Key in keyof Obj]?: Sort +} + + + +export declare type AllowedValues = string | number | bigint | boolean | null | undefined + +interface FilterResult { + data: Array + rows: number + pagination?: { + page: number + pageCount: number + totalRows: number + } | undefined +} + +/** + * + * @param data the original data + * @param query the query to filter against + * @param options additionnal execution options + * @returns the filtered/ordered/paginated {@link data} + */ +export function executeQuery>(data: Array, query: Query, options?: { debug?: boolean }): FilterResult { + if (options?.debug) { + console.log('Query', query) + } + // filter + let filtered = data.filter((it) => { + const res = objectLoop(query, (value, key) => { + if (key === '$or') { + for (const sub of value as Array>) { + const final = filterEntry(sub, it) + // eslint-disable-next-line max-depth + if (final) { + return true + } + } + return false + } + if ((key as string).startsWith('$')) { + return true + } + return filterEntry(query, it) + }) + + return res + }) + if (options?.debug) { + console.log('postFilters', filtered) + } + + // sort + if (query.$sort && objectSize(query.$sort) >= 1) { + // temp until better solution is found + // get the first key + const firstKey = objectKeys(query.$sort)[0] + // biome-ignore lint/style/noNonNullAssertion: item is not null + const first = query.$sort[firstKey]! + + // forst by the first key + filtered = filtered.sort((objA, objB) => { + const a = objA[firstKey] + const b = objB[firstKey] + const ascend = first !== Sort.DESC // it is Ascend by default, so compare against it + if (typeof a === 'number' && typeof b === 'number') { + if (ascend) { + return b - a + } + return a - b + } + if (a instanceof Date && b instanceof Date) { + if (ascend) { + return a.getTime() - b.getTime() + } + return b.getTime() - a.getTime() + } + if (typeof a === 'string' && typeof b === 'string') { + if (ascend) { + return a.localeCompare(b) + } + return b.localeCompare(a) + + } + if (ascend) { + return a > b ? 1 : -1 + } + return a > b ? -1 : 1 + }) + } + if (options?.debug) { + console.log('postSort', filtered) + } + + // length of the query assuming a single page + const unpaginatedLength = filtered.length + let page: number | null = null + let pageCount: number | null = null + // limit + if (!isNull(query.$offset) || !isNull(query.$limit) || !isNull(query.$page)) { + let limit = query.$limit ?? -1 + if (!isNull(query.$page) && isNull(query.$offset) && isNull(query.$limit)) { + console.warn('using $page NEED a $limit too, setting limit to `10`') + limit = 10 + } + const offset = query.$offset ?? (query.$page ? query.$page * limit : 0) + filtered = filtered.slice(offset, limit >= 0 ? offset + limit : undefined) + page = Math.trunc(offset / limit) + pageCount = Math.ceil(unpaginatedLength / limit) + } + if (options?.debug) { + console.log('postLimit', filtered) + } + + return { + data: filtered, + rows: filtered.length, + pagination: (!isNull(page) && !isNull(pageCount)) ? { + page: page, + pageCount: pageCount, + totalRows: unpaginatedLength + } : undefined + } +} + +/** + * + * @param query the query of the entry + * @param item the implementation of the item + * @returns if it should be kept or not + */ +export function filterEntry(query: QueryList, item: T): boolean { + // eslint-disable-next-line complexity + const res = objectLoop(query as any, (queryValue, key: keyof typeof query) => { + /** + * TODO: handle $keys + */ + if ((key as string).startsWith('$')) { + return true + } + + let value: unknown = undefined + + // handle deeply nested items + if ((key as string).includes('.')) { + value = objectGet(item, key as string) + } + + // handle if nested item does not exists + if (typeof value === 'undefined') { + value = item[key] + } + + return filterValue(value, queryValue) + }) + + return res +} + +/** + * indicate if a value should be kept by an ENTIRE query + * + * @param value the value to filter + * @param query the full query + * @returns if the query should keep the value or not + */ +function filterValue(value: unknown, query: QueryValues) { + if (typeof query !== 'object' || query === null || query instanceof RegExp || Array.isArray(query)) { + return filterItem(value, query) + } + + // loop through each keys of the query + // eslint-disable-next-line arrow-body-style + return objectLoop(query, (querySubValue: unknown, queryKey: string) => { + return filterItem(value, {[queryKey]: querySubValue } as QueryValues) + }) +} + +/** + * + * @param value the value to check + * @param query a SINGLE query to check against + * @returns if the value should be kept or not + */ +// eslint-disable-next-line complexity +function filterItem(value: any, query: QueryValues): boolean { + /** + * check if the value is null + */ + if (query === null) { + return typeof value === 'undefined' || value === null + } + + if (query instanceof RegExp) { + return query.test(typeof value === 'string' ? value : value.toString()) + } + + /** + * ?!? + */ + if (value === null || typeof value === 'undefined') { + return false + } + + /** + * strict value check by default + */ + if (!(typeof query === 'object')) { + return query === value + } + + /** + * Array checking and $in + */ + if (Array.isArray(query) || '$in' in query) { + const arr = Array.isArray(query) ? query : query.$in as Array + return arr.includes(value) + } + + if ('$inc' in query) { + if (typeof value === 'number' && typeof query.$inc === 'number') { + return value === query.$inc + } + return (value.toString() as string).toLowerCase().includes(query.$inc!.toString()!.toLowerCase()) + } + + if ('$eq' in query) { + return query.$eq === value + } + + /** + * numbers specific cases for numbers + */ + if ('$gt' in query) { + value = value instanceof Date ? value.getTime() : value + const comparedValue = query.$gt instanceof Date ? query.$gt.getTime() : query.$gt + return typeof value === 'number' && typeof comparedValue === 'number' && value > comparedValue + } + + if ('$lt' in query) { + value = value instanceof Date ? value.getTime() : value + const comparedValue = query.$lt instanceof Date ? query.$lt.getTime() : query.$lt + return typeof value === 'number' && typeof comparedValue === 'number' && value < comparedValue + } + + if ('$gte' in query) { + value = value instanceof Date ? value.getTime() : value + const comparedValue = query.$gte instanceof Date ? query.$gte.getTime() : query.$gte + return typeof value === 'number' && typeof comparedValue === 'number' && value >= comparedValue + } + + if ('$lte' in query) { + value = value instanceof Date ? value.getTime() : value + const comparedValue = query.$lte instanceof Date ? query.$lte.getTime() : query.$lte + return typeof value === 'number' && typeof comparedValue === 'number' && value <= comparedValue + } + + if ('$len' in query && Array.isArray(value)) { + return value.length === query.$len + } + + /** + * Logical Operators + */ + if ('$or' in query && Array.isArray(query.$or)) { + return !!query.$or.find((it) => filterValue(value, it as QueryValues)) + } + if ('$and' in query && Array.isArray(query.$and)) { + return !query.$and.find((it) => !filterValue(value, it as QueryValues)) + } + + if ('$not' in query) { + return !filterValue(value, query.$not as QueryValues) + } + + if ('$nor' in query && Array.isArray(query.$nor)) { + return !query.$nor.find((it) => filterValue(value, it as QueryValues)) + } + + if ('$nand' in query && Array.isArray(query.$nand)) { + return !!query.$nand.find((it) => !filterValue(value, it as QueryValues)) + } + + return false +} diff --git a/server/src/libs/QueryEngine/parsers.ts b/server/src/libs/QueryEngine/parsers.ts new file mode 100644 index 000000000..bba4f7b3f --- /dev/null +++ b/server/src/libs/QueryEngine/parsers.ts @@ -0,0 +1,197 @@ +import { isObject, objectLoop } from '@dzeio/object-util' +import { Sort, type Query, type QueryValues } from './filter' + +/** + * List of allowed prefixes + */ +const prefixes = [ + 'like', + 'not', + 'notlike', + 'eq', + 'neq', + 'gte', + 'gt', + 'lt', + 'lte', + 'null', + 'notnull' +] as const + +type Prefix = typeof prefixes[number] + +type ParamValue = string | number | boolean + +/** + * indicate if the string is a prefix or not + * + * @param str the string to check + * @returns {boolean} true if it's a prefix, else false + */ +function isPrefix(str: string): str is Prefix { + return prefixes.includes(str as Prefix) +} + +/** + * Parse a {@link URL.searchParams} object into a {@link Query} + * + * @param searchParams the searchparams object to parse + * @param skip keys that are skipped by the transformer + * + * @returns the searchParams into a Query object + */ +export function parseSearchParams(searchParams: URLSearchParams, skip: Array = []): Query { + const query: Query> = {} + skip.push('sort:field', 'sort:order') + + const sortField = searchParams.get('sort:field') + if (sortField) { + const order = searchParams.get('sort:order') ?? 'ASC' + + query.$sort = { [sortField]: order === 'ASC' ? Sort.ASC : Sort.DESC } + } + for (const [key, value] of searchParams) { + + if (key === 'pagination:page') { + query.$page = Number.parseInt(value) + continue + } + + if (key === 'pagination:itemsPerPage') { + query.$limit = Number.parseInt(value) + continue + } + + if (skip.includes(key)) { + continue + } + + const params = parseParam(key, value) + if (!query[key]) { + query[key] = params + } else { + if (isObject(params)) { + objectLoop(params, (v, k) => { + query[key][k] = v + return + }) + } else { + query[key] = params + } + } + + } + + console.log(query) + return query as Query +} + +/** + * parse a simple {@link Record} object into a {@link Query} + * + * @param searchParams the searchparams object to parse + * @param skip keys that are skipped by the transformer + * + * @returns the searchParams into a Query object + */ +export function recordToQuery(input: Record>, skip: Array = []): Query { + const query: Query> = {} + skip.push('sort:field', 'sort:order') + + const sortField = input['sort:field'] as string + if (sortField) { + const order = input['sort:order'] ?? 'ASC' + + query.$sort = { [sortField]: order === 'ASC' ? Sort.ASC : Sort.DESC } + } + + objectLoop(input, (value: string | Array, key) => { + + if (key === 'pagination:page') { + query.$page = Number.parseInt(value as string) + return + } + + if (key === 'pagination:itemsPerPage') { + query.$limit = Number.parseInt(value as string) + return + } + + if (skip.includes(key)) { + return + } + + if (!Array.isArray(value)) { + value = [value] + } + + for (const it of value) { + const params = parseParam(key, it) + if (!query[key]) { + query[key] = params + } else { + if (isObject(params)) { + objectLoop(params, (v, k) => { + query[key][k] = v + return + }) + } else { + query[key] = params + } + } + } + + }) + + console.log(query) + return query as Query +} + +/** + * Parse a single element + * + * @param _key currently unused, kept for future compatibility + * @param value the value to parse + * + * @returns the parsed {@link Query} element to be added + */ +function parseParam(key: string, value: string): QueryValues { + const colonLocation = value.indexOf(':') + let filter: Prefix = 'like' + let compared: string | number = value + if (colonLocation >= 2) { // 2 because the smallest prefix is two characters long + const prefix = value.slice(0, colonLocation) + if (isPrefix(prefix)) { + filter = prefix + compared = value.slice(colonLocation + 1) + } + } + + if (/^\d+\.?\d*$/g.test(compared)) { + compared = Number.parseFloat(compared) + } + + switch (filter) { + case 'not': + case 'notlike': + return { $not: { $inc: compared }} + case 'eq': + return compared + case 'neq': + return { $not: compared } + case 'gte': + return { $gte: compared } + case 'gt': + return { $gt: compared } + case 'lt': + return { $lt: compared } + case 'lte': + return { $lte: compared } + case 'null': + return null + case 'notnull': + return { $not: null } + default: + return { $inc: compared } + } +} diff --git a/server/src/util.ts b/server/src/util.ts index 1db8fce0b..d177f05fd 100644 --- a/server/src/util.ts +++ b/server/src/util.ts @@ -282,6 +282,6 @@ function objectGet(obj: object, path: Array): any | un * @param value the value the check * @returns if the value is undefined or null or not */ -function isNull(value: any): value is (undefined | null) { +export function isNull(value: unknown): value is (undefined | null) { return typeof value === 'undefined' || value === null } From 06428cf6aa093564910b693b85a62662b122c7c1 Mon Sep 17 00:00:00 2001 From: Avior Date: Wed, 21 Aug 2024 13:54:46 +0200 Subject: [PATCH 2/6] feat: Also support GraphQL Signed-off-by: Avior --- server/src/V2/graphql/resolver.ts | 54 ++++++++++++++++++------------- 1 file changed, 32 insertions(+), 22 deletions(-) diff --git a/server/src/V2/graphql/resolver.ts b/server/src/V2/graphql/resolver.ts index 2c05f2a91..961762a8b 100644 --- a/server/src/V2/graphql/resolver.ts +++ b/server/src/V2/graphql/resolver.ts @@ -1,44 +1,54 @@ -import { SupportedLanguages } from '@tcgdex/sdk' -import { Query } from '../../interfaces' +import type { SupportedLanguages } from '@tcgdex/sdk' +import { type Query, Sort } from '../../libs/QueryEngine/filter' +import { recordToQuery } from '../../libs/QueryEngine/parsers' import { checkLanguage } from '../../util' import Card from '../Components/Card' import Serie from '../Components/Serie' import Set from '../Components/Set' -const middleware = (fn: (lang: SupportedLanguages, query: Query) => any) => ( - data: Query, +// TODO: make a better way to find the language +function getLang(e: any): SupportedLanguages { + // get the locale directive + const langArgument = e?.fieldNodes?.[0]?.directives?.[0]?.arguments?.[0]?.value + + if (!langArgument) { + return 'en' + } + + if (langArgument.kind === 'Variable') { + return e.variableValues[langArgument.name.value] + } + return langArgument.value +} + +const middleware = (fn: (lang: SupportedLanguages, query: Query) => any) => ( + data: Record, _: any, e: any ) => { // get the locale directive - const langArgument = e?.fieldNodes?.[0]?.directives?.[0]?.arguments?.[0]?.value + const lang = getLang(e) - // Deprecated code handling - // @ts-expect-error count is deprectaed in the frontend - if (data.pagination?.count) { - // @ts-expect-error count is deprectaed in the frontend - data.pagination.itemsPerPage = data.pagination.count - } + const query = recordToQuery(data.filters) - // if there is no locale directive - if (!langArgument) { - return fn('en', data) + // Deprecated code handling + if (data.pagination) { + query.$page = data.pagination.page ?? 1 + query.$limit = data.pagination.itemsPerPage ?? 100 } - // set default locale directive value - let lang = 'en' - // handle variable for directive value - if (langArgument.kind === 'Variable') { - lang = e.variableValues[langArgument.name.value] - } else { - lang = langArgument.value + if (data.sort) { + query.$sort = { + [data.sort.field]: data.sort.order === 'DESC' ? Sort.DESC : Sort.ASC + } } if (!checkLanguage(lang)) { return undefined } - return fn(lang, data) + + return fn(lang, query) } export default { From afc65a1a1a214afe03f903481e89efe258276e58 Mon Sep 17 00:00:00 2001 From: Avior Date: Wed, 25 Sep 2024 23:47:03 +0200 Subject: [PATCH 3/6] feat: Add more testing & validations --- server/bun.lockb | Bin 87589 -> 87286 bytes server/src/V2/endpoints/jsonEndpoints.ts | 4 +++- server/src/V2/graphql/index.ts | 4 ++-- server/src/V2/graphql/resolver.ts | 2 +- server/src/libs/QueryEngine/filter.ts | 6 ++++-- server/src/libs/QueryEngine/parsers.ts | 8 +++----- server/src/status.ts | 4 ++-- server/tsconfig.json | 3 ++- 8 files changed, 17 insertions(+), 14 deletions(-) diff --git a/server/bun.lockb b/server/bun.lockb index 21dd12259926dbc9ace489c2cc8b928a65e28966..4f557055812bc98281b6a73c8ba84b9136178475 100755 GIT binary patch delta 7684 zcmaJ`dt4P&*PcCakURc&Ks*p~Ml)|sr1oZng^7ZI$VCnyXbN6PNzKbr z$AhAxQdyZ9X_%U2`KD=^U0y}Aq>DnP-9Aga@jZKHj{d%XzVq{8&9krTS$pl-<4ofl zroVTYR!3WYKWYiCnwP!)pPG436}>4Vk)c3t|%_aw%djB%#uo8H^fU2x(Y&dX=QPCRGwY<4cr5~Ag>@hDr=hXlTmLk z%`7dmXWRRMcSC(3tS78JyD*E7<6`hjDCFa1XGhJ*E3uar*mH>=<)o;V{S#mS=hK)w~Ewq!Z8@&ZX~wkT`mk6xZ+SmzQrZC{ily4891%#xDKO2G^cQPnki|ElUa z#=ymSg<7Zzz`3aGl9Hm5sO$>Ae!7EGQO-j#9+t;^gwZb%mOC^6mWMn9mWS3CmQUnv z)c+l>2kq?vx=;mzJ%lQudV)nCI47?(s;pQLz5?e1_8h3&JbRSgo+}7Wo{WxsqS691 z#;-6^Zx;*8BUBPK(_U042sgmFqnXqAN+RB~G$UZE6!X2~ABB18n8}azrxsIKUPH9r z-xvLQpx=xbK?sJu4vYM&&Kj)GI|-IY{s)wEf!L2lv&yrD`dC50VpbC@4`2-}_h$^+ z@%B0Tjum=RrdxmW!Xf%BvthX!TY?_8cxrI#X+8$V`)nMl$EG;5G&d?gyK)AdaqDTx zKp!5VzKMFPxv+euCyjEyBz;Ub^6e5M-g$@~?$U4K`@{6ON2cf#6<I%aeH@>iMAS zVZ&jU+w}H}!Fhe8tWSc8G}rbw9q5w7ng&SCJ`^69D9K0;M5dZiM0cxH1cm{?8hos5 zH?;)HVuu$62HDt9N)M8y0*u2O2-J$^uY*BEwaA-GKY~4?bKGJ5o7m6whu_QAar4N#&W!U47V<==w zLLv*J^Mh=rZsB~jF7{8BhT+OcYeZ}=7;g)gBCXPK-UpAALzAVgxEiL7gIp0W^rgT!n-q$p!4Mm4 zm7WFTf?T!jxCKm4HB2S7fjxpc?RFpy>aUB#t-@lJW`K>=>UbKy3C73N(op&pY`DgF z%Rx9gj0E>XKQMj0Tf?lTV_=W-&@n$UC&*HUMfb+Uv%$0#YzSQpP8Pql(D?)#3n%kX zS(=O^M$dV^J}-gkG3QBeoLYv;OeV8c7Ux9rUB~v4!zwd3YO%_uWNh_Esot8*YA8KX zX77+AQ5IVV(fLFhiy(87Y|6p`6Q?!VK#n9?Iu6cb=t7x;tdc8I!I&EkCV~y+ZKXn7 z#cQovhgzjQU`F72Vxvb>V6shGgcD_|)}$fSD!CxdpU@fHE~S=aSvrie(I^wOyQ>RM zA|6E-zA~oA!K{>FPBtCF)eOpzhq=e;o?&_Xt2CzSfw;Vil_&qBzu^1Nt%p2Fhk1lVz~sF?~**UFh` zR580tBSm;G}!A8=!qBI&?ASBt&KYqo+L ze){Y*dV01O9R^cKDoG>v66h;YSSlR=YX%FkD{MUF*`d!4eP9#GTnc@q&{wLkf71!D z<6u!`3Y$cgWzbg!ePC&1DTlst=qp!PI<5Io2#&F>Hxb5mN8FZIkamY^v#35=M|Pm>CZ#o z^Uw#DPttton-6{S6;?n8z?#8A7AUNU@)khf0_X#qP39`-tAf5Nh1ux@*m1C^YK4_i zWi|9wLm${2vMhwYh0wQ9;b{kU4lHGn!sb!!BIsKLePHt`X)*LIhQ7rLtD<(WHn7ws z3Lb+TOQ3HF^es{7_a)i1yGFrnw4(;fYM|@|g)OCBFF@G~P_|TIwUoXT%9cVI*m9DV zLD@1WTc)s;bO5XwETmRpt0=D)%4(quYz>)Tgt8Z*>_vqUod7!y7PVYq>#1@%lr4uc zFqJGTplk({tx))5CfGT!l$8qGM71lSY$cR|)lnb$+#dgxoPuqHYIb{s5fgTnSvfAJ|7U zZWHuvg1$`(J4PK~H^DMCE9?{6wHf+0LtnkZPEdM1^wmQj*h!MMK;IVV+oG_~=m1zV zSjbj|ou<64(6<%(z|N3)8}x01zHJKoicWwX2a9S@*f&(!0DTS62X>Y$+o5ke^levI z3$=iq154SVunSbX1NwGAAJ|1o+6jF-p>L}To#y9t)@io&im?Rw>`D-GW7+7!O`S?$98gHZiHe&p*NY}ELh+(+V&?TQ;a zMZEed{j@*a-4AD=k6ti=CV0F0e2t^d0|no~@`|R|y}ydu08`%^GN`N1WkLJ7w+rfQ ze>~p&c-caOe{pT&pYfY0cZ>Xd{zs)RIDa4k~-9v^31M&g>sVG6)IaNQXI*U0-!1)TMqKL>ER(}5gd z29RsmJlIbB@?mEJ#Rl)}lzDMyXAugv04gvWmo;Iu2YB6^0B?5y z*az@1yal`i>;;;Dx3z1y%`cjPcY*x?&jdcvL4Z5M2Ra1s37t5J>(js~-~{j?@B#1< za1=NW{0I0LI0k$I@IIdcp8=l(XMitw*uDWwz*m49@FmaW*HrjdIph8yPVN5_ zV8Sy>XNNh#`Kol#r{Da#`wPEOB8!cV8yp>ng!0ti5M3NO;@gW8Z|xG<;OK}i z{$3;>D>mASu{u2I`t3M8{6^d{GjB@16VC#v2-p26vooF!-3eqv=oUg9Edfc4+`-)5R zMK&cidN5b#d`T@&`Lq1i$gOV}HMnKD(A`dRkn?5tsLF@y4t_YwORG})T@l;7$I-#r51`9PK$Xjb(_c*J*cs{u()97`|l2~ z-unWEZ~I+jIcEJW=Wkx>3K!NZC=VIrfm=X{DgScY@YOf@LJj4U3nA(e7Z%Ud!<_@aTV6KK@+j2C!&^k7MqT9!p=;D#uITofdXbxC)z^eVMUCWU zk~+$Sh&6=jiS|}*aOa`4FI$W;VzIGRGO<)PQ*Gs@OpS44@xjhl-w_vLru3f@aRtp2 zFeRG*9j-2Lg9zt4>5>I8b*C>44@V20ZdhDjb*~#H4pC3S1UcVnmmi(D{93~y?mQ1Q zoKI9c(ICP368umx_M^snV&z99@I=#Q@dbn=X>_tgy&w48FXzu z8pOc|j6FwPhz6`i-Nem0H9}%NToZ6NR8ONI!TBZNY2QEFGK84cp`Y`}o$z4y%!TsL{IiaDCvv=6EXeQ;VegGD30NaDD)&NxWtM;MaZP11~42 zhb5LOeq~WTJ&^7_RoMeQd#dRkNMT>~Lk}cKfAt18v8tsjOre_5m6^pGk!nR(R>%BR zKTo!aHK=<$Q8`Ba(UXmmp27+1gN&V_#(JUc4>9UvUdRFGYyF4zJeF}KbwnxLFcNc; zT7`xzOZ^Py|7mA5aK7i?F|)j^^4>Ec7$`1U+wY{ZZWyRu4eExd_rxZ_9kaUP*Jo;9 z@kv|qV8Z_D;%;nTg7fwL2Z4Rv4n8;alGXq>TRcM`GbMBl6MvX8@%r^*)L;UhuQSvF zZ#e1vieT>Eeb$qGyuUWOVlft~4QRmP)q~vF)Ni~InPaNtgZ;8mP4i)qY>K+rheZTA zzb2#~xo+D0-L(N2gJ%VncCLEF2a~uZ=^rDuytejgkAN?0&>#^F;GduRmk*2g+m8l3 z(l>9-%RTz^x<-+0Rg-*Kgx>{J;AE@LO8@BS7Vm8)k^M(4K!yKZW5`&KKdufO?e8kG z_9S(?FDt}X3$=T9HY36L`Cfw-QlnEi$`;0=_1cP;mu0JG>l%(?;iQ*|2nt)ZqBL) zy^Q9O>PbK3<9;>XpT&DL`!UU%n|^AsKPwJAX{_>+#Dr6d0 zh^UyF@APtgrDf&y)=|r-G&TE`l@Bs2uib7w()+EMbG+SuF7}-@-`;EQy}q^g>^;r= zw$rk0k7Z?K*K;?%-#4%BvSy!pa&GdqBJ0`usptQ8EWgdrz~`F8?Tr6)?MNZ)<`e!_Hc?tw*2&Kg(_AU}O(L`LqU(omZ@;4WBH zIWtNNGdZ+1ARKV|#EF@Cg$2@Ps%#Ky8xbT)El_$Bmcxt;HfkjA6coZ36*Va5*e=4KW1W&)Xq4XxGh<0=Z}$5g zI49#WEGO*@tgj@MIp=mTEAl+nJ1bs02IWY!(@~I~pPybT4FzY>$IT9$`#c@zIdY8T zZ3SnUnfdv-`4O2VDV@v!b(C{*7Q^x^lo;*GBxf!zIHK{eJPRXWS${jr=R+|Mw4|%q@f1g4L{T2fx|#hA@mNPeL_tB8Bzbv0+VMDr*>FborET5K23fG2 zqWp-d1-Utr6dq3NEg?27!t6gMH#e_0+$UvDo?Hq$j_NF-WfvpOMuX89>&{u-Ly|DT z&W=4LsTFJ$ERVJv7WsDCU|CguW#9Z{l{!{XKEJPn; zwwsRGFN|;$BE4KpxsCkGj`T9e-T}*F6~&qvoa(VNvm6tV0Lf91S%T#)Jw!S8eGYPX zO7hYRvm&NtmR6%ZXTBKI!85{ZEib*GU=7NdkLhE!I{?cuE;Gx0oGEeUzp7JlJWZqxq?;M&yP+ACn`gbX>KH_ z&eyIi1`9X37}mJwr zNC{KrJ1xm4QYBZIYFQs3N&P9cRf7C$0QYW*#)6Hc)X)TF0p5!gnOQjEShyBT1ne8jG70mC9< zC5ZY0DIHZMB+x9w+#q?w!A$=;c&sLOM^$`7N+(t6irv?owYm_yNGBJ{4xx;_WnKoB z-@u~D+9E;e()Le!#8h&1R>dxIcUG0_sPRl=XS*B{L@8YomC4wuSwKTW)CO{QQN=$< z>8dJyg3TOQjJaG4_8j~g7z^i9FrG1_BGj(P*jjK~cqS=8S7KFwxja!M9YA7JXsZ;pKgK+ z&ss*l4uG+50~*%dt~_QZFpH7PN7zBlS>=WJTzhY3hlSWJ%fX(d3e5c{xZ)W#7NhLb zflfyzT9#wi_f+#0s?BvD@PFPV##8;@Y&hzM8XfV(ZvitG1qQcV1WV$jW(n*lNdsA| zG7eYIdi^b1zt^cr{)F<35emsC2+Nlk~)MA{e*BaB`m za`#r{eck!q6E`W@u8O|ova6PS+~+BjYEKZyN$I1CTa?^Kl>;NFyicMSMeaVTr3z5WBgb7=p!%l-y5M-b7gfJbb7QtLz7lM`PotYLCr_XI>#r z=rl01XP%i8V8%@+H;ALt{S%d!Aeo1f2$@kffN|3%Mu`1zX)+cw8<#xcjaiy}IG)P= z5|z6s5z{o0K&8{tsLOOKeFfH+x=v4{i(nPgHKEcau%+2n>YJ^J ze&otdqdqxSx()UW#pR^YO|a@5O$?xVuywgsO3l^8AgatwqrrJr`U5PPlJnB&cd)&A zn$YMT*p3-i8aqQ1skCzj^yNcez9ych(fQC<0DWM?NGX6muiYGMVAUJ7L{C<9wXiVMoXvR#@WItrFm4rS$~BK^fSb zWtylYH(1GXC|jJ^&UM)hFpRzlxOO}s&sE1_={^nvZ9uFZc5$&ePDYxXyOdr1KUvzebt)yh;~*(-$v-$sEKnl zdL#60fMk;Zcyd+G;y=`&h{@FlAmut?a=)b8q(1H4aL7|Blag)EVFPhH>H|GyD9Kc zJ2|BnZ9McMCHXe=X^Lac2M0u}-nAPK;b|amf0R#?8b zexmr>oof5k|K7l7)K6yFq`O=eaUJXma2a?PI057WIY2fr8OQ|ioFjFoirZ((CgLgs z7z>O6UItzQMgw?Mknq2X(lY=KN2wo>1|$JE0;Ly$kpK=;X%K*?0cjYJ4B#l{=K+2& zm2mv?QxQJ_NqEZP2MP&~P5cy4#?SQckf0(fwoqkJkGgY#hA zIgV{3U>uB32E^IP2R}cy@-s6(TuK`7JTL^~OQ{~h1zh<8FdX0jyvn@_d5}>6D@zAB zTYnCe0~-&pKWAYaz>2vI`+Mt|PXKyid}$Jp1xx`P9y=BGG5)5*P6K9mIG6EX9{46; z9gq*q24(?;Kmkw$lmaEdOrRLxG7kJTU@fo)prOA6mMz1D3s?#`0S;;|@Cq;oSORcL z7Xyocg}{7Z9;rZKuLE0v z%|H#X72vXMz;=Kez5(n2c6zS&!tMdM?oELG-U1E*oRI^-LEtaIUx8ZBbuG(13UUNE z3=9EypyL1s!yUa1@PJ;N#q}q^N5E;|ec(OdZ@@{w4SWEc0{#ws2ymM-z{kKj;8UOu zxWMVQ0G|VmfX{&Dz$M@Uz~$$Gi@+Bi{w3`9z_-9v;49!8pe^wKz}KE@=3M_Bz~$Ef zE@NB)ZUD`1;%D7m{NQhLKyC3qiiIUIIyy3{_Rif_mLTuT*M-kF$$5G~x^h2I?rEjp?neiCUvf9P8Fn_+r`Z8n%#DgfAh?Cso)9(~^Pp{z_r-YC zE8&)FC;xTT2q!kOC-yK}^&l0GIbT1B3HH8L&lvs2y#3c#^pQo+$QX9Q3wVlr82zUh zZM5d${3kr>k5nVtRBxDgI=}J9JUpMRdJrSyb^FFg(b)Ry9@*rm0Q&J!AnK1jw&BS# z6u(!f=CO^2KW;1PDcj43fd~4E(r`vH?y&|J~Ns^i2^`5kc+Vp|4h>rEXRljF#f3Q{3 z=3iv7(PqB;^nR4+AxIy0CjD3B9a)cz=_PH}_sYU1M(7{N82zUH07i}s)_XP(&xt~P zP6J^J^1f-`c(MG(qm-2`Wibwlc-B7D4>UlttNJyVSnvDc=i0t_F#X3EU-(5KU`&el zjdPEFyFUEALyrqaO%&HG*Has!hl~2Wh6v-9Ue!>fiWD9)27vLOG+aY9d;P#NrHxkAjVK`uQd5#;3ly2p`TWe7@7~a9?-y zrj5~;_d7&{yrcACW!k5n*f8c$U2TjsjnOOE&->XS?8_eGI*$vxfjN#f@8dlEJvfND z`j`L1596A_3o}GrZ6fSJ-fs}U<-9gOz-N0049?3LgBxBpQLpcYV6j9$)kLJqKHc;H zMMMXAzZk^6`1hLwk32Qhh}l>PL-ce73w*HdRxq{W^gq~S=}|r+CfNHq;nKwBdlr10 z@mFJaFtQsSt~-6idZFn-O+^(}%i*SIHdnvZR18S0zzOV!M0!6URQ=L&_sJ`jRt)M{ zAl^?0vy!@h8xt|2siy{#JNDFbnu)+5??;Bl>h$ur+PyOg9(;qqqe`!7CTz0DQJ7R5 z*MCN7koVieg8iB2y2U+z)X>>8(vD4C?_kB;4AEy;k)Cn-GOIW!mg)VPgRRkDh6(b1 zG+5*}X8QDnVuR5*ig(@}`U*J2dcRKWYIfn#<7dwQX;pf^PYj(q{rHswwR_OiZmyjo z{j26k%|qSa7sH>^$NP%zVy(W+SA=1q?DfUC-j5Jz!#+V16^`+3|m^v_ygsomBST8bE-{Q<)0`Ii8_ zq@~DfD`OjNhFBN&i9IVynm=mS%B-&#c~}k*Q<~{3f`mmM8zh=Hh>eWX{X<09VMjYr G(d*x>?Dn7l diff --git a/server/src/V2/endpoints/jsonEndpoints.ts b/server/src/V2/endpoints/jsonEndpoints.ts index d9366c70e..a0534e60a 100644 --- a/server/src/V2/endpoints/jsonEndpoints.ts +++ b/server/src/V2/endpoints/jsonEndpoints.ts @@ -267,8 +267,10 @@ server switch (endpoint) { case 'sets': + // allow the dev to use a non prefixed value like `10` instead of `010` for newer sets result = Card - .findOne(lang, { localId: subid, 'set.id': id })?.full() + // @ts-expect-error normal behavior until the filtering is more fiable + .findOne(lang, { localId: { $or: [subid.padStart(3, '0'), subid]}, 'set.id': id })?.full() break } if (!result) { diff --git a/server/src/V2/graphql/index.ts b/server/src/V2/graphql/index.ts index 384a16b78..e54a177c5 100644 --- a/server/src/V2/graphql/index.ts +++ b/server/src/V2/graphql/index.ts @@ -1,6 +1,6 @@ import express from 'express' -import fs from 'fs' -import { buildSchema, GraphQLError } from 'graphql' +import fs from 'node:fs' +import { buildSchema, type GraphQLError } from 'graphql' import { createHandler } from 'graphql-http/lib/use/express' import { type ruruHTML as RuruHTML } from 'ruru/dist/server' /** @ts-expect-error typing is not correctly mapped (real type at ruru/dist/server.d.ts) */ diff --git a/server/src/V2/graphql/resolver.ts b/server/src/V2/graphql/resolver.ts index 961762a8b..6c095b9a4 100644 --- a/server/src/V2/graphql/resolver.ts +++ b/server/src/V2/graphql/resolver.ts @@ -29,7 +29,7 @@ const middleware = (fn: (lang: SupportedLanguages, query: Query) => any) // get the locale directive const lang = getLang(e) - const query = recordToQuery(data.filters) + const query = recordToQuery(data.filters ?? {}) // Deprecated code handling if (data.pagination) { diff --git a/server/src/libs/QueryEngine/filter.ts b/server/src/libs/QueryEngine/filter.ts index 275478cb4..9ead9e643 100644 --- a/server/src/libs/QueryEngine/filter.ts +++ b/server/src/libs/QueryEngine/filter.ts @@ -222,6 +222,7 @@ export function executeQuery>(data: A return res }) + if (options?.debug) { console.log('postFilters', filtered) } @@ -279,7 +280,8 @@ export function executeQuery>(data: A console.warn('using $page NEED a $limit too, setting limit to `10`') limit = 10 } - const offset = query.$offset ?? (query.$page ? query.$page * limit : 0) + // when using $page, they start at 1 and not 0 + const offset = query.$offset ?? (query.$page ? (query.$page - 1) * limit : 0) filtered = filtered.slice(offset, limit >= 0 ? offset + limit : undefined) page = Math.trunc(offset / limit) pageCount = Math.ceil(unpaginatedLength / limit) @@ -347,7 +349,7 @@ function filterValue(value: unknown, query: QueryValues // loop through each keys of the query // eslint-disable-next-line arrow-body-style - return objectLoop(query, (querySubValue: unknown, queryKey: string) => { + return objectLoop(query as any, (querySubValue: unknown, queryKey: string) => { return filterItem(value, {[queryKey]: querySubValue } as QueryValues) }) } diff --git a/server/src/libs/QueryEngine/parsers.ts b/server/src/libs/QueryEngine/parsers.ts index bba4f7b3f..bf0046b01 100644 --- a/server/src/libs/QueryEngine/parsers.ts +++ b/server/src/libs/QueryEngine/parsers.ts @@ -20,8 +20,6 @@ const prefixes = [ type Prefix = typeof prefixes[number] -type ParamValue = string | number | boolean - /** * indicate if the string is a prefix or not * @@ -72,7 +70,7 @@ export function parseSearchParams(searchParams: URLSe } else { if (isObject(params)) { objectLoop(params, (v, k) => { - query[key][k] = v + (query[key] as any)[k] = v return }) } else { @@ -132,7 +130,7 @@ export function recordToQuery(input: Record { - query[key][k] = v + (query[key] as any)[k] = v return }) } else { @@ -155,7 +153,7 @@ export function recordToQuery(input: Record { +function parseParam(_key: string, value: string): QueryValues { const colonLocation = value.indexOf(':') let filter: Prefix = 'like' let compared: string | number = value diff --git a/server/src/status.ts b/server/src/status.ts index e4e97e235..8554ee57f 100644 --- a/server/src/status.ts +++ b/server/src/status.ts @@ -346,7 +346,7 @@ export default express.Router() ${objectMap(setsData, (serie, serieId) => { // Loop through every series and name them - const name = Serie.findOne('en', { filters: { id: serieId }})?.name ?? Serie.findOne('ja' as any, { filters: { id: serieId }})?.name + const name = Serie.findOne('en', { id: serieId })?.name ?? Serie.findOne('ja' as any, { id: serieId })?.name return ` @@ -364,7 +364,7 @@ export default express.Router() // loop through every sets // find the set in the first available language (Should be English globally) - const setTotal = Set.findOne(data[0] as 'en', { filters: { id: setId }}) + const setTotal = Set.findOne(data[0] as 'en', { id: setId }) let str = '' + `` // let str = '' + `` diff --git a/server/tsconfig.json b/server/tsconfig.json index a3165dfaa..33878bf06 100644 --- a/server/tsconfig.json +++ b/server/tsconfig.json @@ -1,7 +1,8 @@ { "extends": "./node_modules/@dzeio/config/tsconfig.base.json", "compilerOptions": { - "outDir": "dist" + "outDir": "dist", + "esModuleInterop": true }, "include": ["src"] } From a5429162a86be37bb565caf9a8aff15066cf82e7 Mon Sep 17 00:00:00 2001 From: Avior Date: Wed, 25 Sep 2024 23:52:16 +0200 Subject: [PATCH 4/6] chore: missing files --- .bruno/sets/Advanced Query.bru | 21 +++++++++++++++++++++ .github/workflows/test.yml | 8 ++++++++ 2 files changed, 29 insertions(+) create mode 100644 .bruno/sets/Advanced Query.bru diff --git a/.bruno/sets/Advanced Query.bru b/.bruno/sets/Advanced Query.bru new file mode 100644 index 000000000..b07788021 --- /dev/null +++ b/.bruno/sets/Advanced Query.bru @@ -0,0 +1,21 @@ +meta { + name: Advanced Query + type: http + seq: 4 +} + +get { + url: {{BASE_URL}}/v2/en/sets?cardCount.official=gt:64&id=swsh + body: none + auth: none +} + +params:query { + cardCount.official:gt: 64 + id: swsh +} + +assert { + res.status: eq 200 + res.body: length 17 +} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2ce0d19f1..ef535b02f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -21,6 +21,7 @@ jobs: - name: Install deps run: | + bun install -g @usebruno/cli bun install --frozen-lockfile cd server bun install --frozen-lockfile @@ -31,3 +32,10 @@ jobs: bun run validate cd server bun run validate + + - name: Validate some requests + run: | + cd server + bun run start & + cd ../.bruno + bru run --env Developpement From af7af3d1964e1fa413b4bf43a251439c05d709ef Mon Sep 17 00:00:00 2001 From: Avior Date: Wed, 25 Sep 2024 23:57:36 +0200 Subject: [PATCH 5/6] fix: github Actions using IPV6 --- .bruno/environments/Developpement.bru | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bruno/environments/Developpement.bru b/.bruno/environments/Developpement.bru index fa6654ac8..c0a0a0e4e 100644 --- a/.bruno/environments/Developpement.bru +++ b/.bruno/environments/Developpement.bru @@ -1,3 +1,3 @@ vars { - BASE_URL: http://localhost:3000 + BASE_URL: http://127.0.0.1:3000 } From 156d43a53569225889973e93adb55ce594a5c08b Mon Sep 17 00:00:00 2001 From: Avior Date: Thu, 26 Sep 2024 00:03:04 +0200 Subject: [PATCH 6/6] fix: missing sleep time --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ef535b02f..1344c0c35 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -37,5 +37,6 @@ jobs: run: | cd server bun run start & + sleep 10 cd ../.bruno bru run --env Developpement

${name} (${serieId})

${setTotal?.name} (${setId})
${setTotal?.cardCount.total ?? 1} cards
${setId})