From 9d3bd3a6dc67ab42ad4af0064524129f81822972 Mon Sep 17 00:00:00 2001 From: Ahad Birang Date: Mon, 1 Aug 2022 16:48:48 +0430 Subject: [PATCH 1/9] feat: create index for path base search --- src/module.ts | 5 +++++ src/runtime/composables/query.ts | 9 ++++++-- src/runtime/server/api/cache.ts | 11 ++++++++-- src/runtime/server/api/findOne.ts | 34 +++++++++++++++++++++++++++++++ 4 files changed, 55 insertions(+), 4 deletions(-) create mode 100644 src/runtime/server/api/findOne.ts diff --git a/src/module.ts b/src/module.ts index 370f442ec..948e00634 100644 --- a/src/module.ts +++ b/src/module.ts @@ -262,6 +262,11 @@ export default defineNuxtModule({ route: `/api/${options.base}/query`, handler: resolveRuntimeModule('./server/api/query') }, + { + method: 'get', + route: `/api/${options.base}/find_one/:qid/**:slug`, + handler: resolveRuntimeModule('./server/api/findOne') + }, { method: 'get', route: `/api/${options.base}/cache`, diff --git a/src/runtime/composables/query.ts b/src/runtime/composables/query.ts index b2adb688b..39681194e 100644 --- a/src/runtime/composables/query.ts +++ b/src/runtime/composables/query.ts @@ -24,7 +24,12 @@ export const createQueryFetch = (path?: string) => (query: Qu const params = query.params() - const apiPath = withContentBase(process.dev ? '/query' : `/query/${hash(params)}`) + let apiPath + if (params.where?.length === 1 && Object.keys(params.where[0]).length === 1 && typeof params.where[0]._path === 'string') { + apiPath = withContentBase(`/find_one/${hash(params)}/${params.where[0]._path}`) + } else { + apiPath = withContentBase(process.dev ? '/query' : `/query/${hash(params)}`) + } // Prefetch the query if (!process.dev && process.server) { @@ -35,7 +40,7 @@ export const createQueryFetch = (path?: string) => (query: Qu }) } - return $fetch(apiPath as any, { + return $fetch(apiPath as any, { method: 'GET', responseType: 'json', params: { diff --git a/src/runtime/server/api/cache.ts b/src/runtime/server/api/cache.ts index 4e22ac63b..375f47fc2 100644 --- a/src/runtime/server/api/cache.ts +++ b/src/runtime/server/api/cache.ts @@ -1,11 +1,18 @@ import { defineEventHandler } from 'h3' -import { serverQueryContent } from '../storage' +import { cacheStorage, serverQueryContent } from '../storage' // This route is used to cache all the parsed content export default defineEventHandler(async (event) => { const now = Date.now() // Fetch all content - await serverQueryContent(event).find() + const data = await serverQueryContent(event).find() + + // Generate Index + const index = data.reduce((acc, item) => { + acc[item._path!] = item._id + return acc + }, {} as Record) + await cacheStorage.setItem('index.json', index) return { generatedAt: now, diff --git a/src/runtime/server/api/findOne.ts b/src/runtime/server/api/findOne.ts new file mode 100644 index 000000000..15f93c3f0 --- /dev/null +++ b/src/runtime/server/api/findOne.ts @@ -0,0 +1,34 @@ +import { createError, defineEventHandler } from 'h3' +import { withLeadingSlash } from 'ufo' +import { cacheStorage, getContent, serverQueryContent } from '../storage' + +// This route is used to cache all the parsed content +export default defineEventHandler(async (event) => { + let index = await cacheStorage.getItem('index.json') as Record + if (!index) { + // Fetch all content + const data = await serverQueryContent(event).find() + + index = data.reduce((acc, item) => { + acc[item._path!] = item._id + return acc + }, {} as Record) + + await cacheStorage.setItem('index.json', index) + } + + const slug = withLeadingSlash(event.context.params.slug) + + if (!index[slug]) { + throw createError({ + statusMessage: 'Document not found!', + statusCode: 404, + data: { + description: 'Could not find document for the given path.', + path: slug + } + }) + } + + return getContent(event, index[slug]) +}) From e83a16039a507cbe8520516f240b77550050d8c8 Mon Sep 17 00:00:00 2001 From: Ahad Birang Date: Thu, 4 Aug 2022 14:46:26 +0430 Subject: [PATCH 2/9] fix: use search index in query API --- src/module.ts | 5 ---- src/runtime/composables/query.ts | 7 +----- src/runtime/server/api/findOne.ts | 34 --------------------------- src/runtime/server/storage.ts | 38 ++++++++++++++++++++++++++++++- 4 files changed, 38 insertions(+), 46 deletions(-) delete mode 100644 src/runtime/server/api/findOne.ts diff --git a/src/module.ts b/src/module.ts index 948e00634..370f442ec 100644 --- a/src/module.ts +++ b/src/module.ts @@ -262,11 +262,6 @@ export default defineNuxtModule({ route: `/api/${options.base}/query`, handler: resolveRuntimeModule('./server/api/query') }, - { - method: 'get', - route: `/api/${options.base}/find_one/:qid/**:slug`, - handler: resolveRuntimeModule('./server/api/findOne') - }, { method: 'get', route: `/api/${options.base}/cache`, diff --git a/src/runtime/composables/query.ts b/src/runtime/composables/query.ts index 39681194e..d0d51f0d5 100644 --- a/src/runtime/composables/query.ts +++ b/src/runtime/composables/query.ts @@ -24,12 +24,7 @@ export const createQueryFetch = (path?: string) => (query: Qu const params = query.params() - let apiPath - if (params.where?.length === 1 && Object.keys(params.where[0]).length === 1 && typeof params.where[0]._path === 'string') { - apiPath = withContentBase(`/find_one/${hash(params)}/${params.where[0]._path}`) - } else { - apiPath = withContentBase(process.dev ? '/query' : `/query/${hash(params)}`) - } + const apiPath = withContentBase(process.dev ? '/query' : `/query/${hash(params)}`) // Prefetch the query if (!process.dev && process.server) { diff --git a/src/runtime/server/api/findOne.ts b/src/runtime/server/api/findOne.ts deleted file mode 100644 index 15f93c3f0..000000000 --- a/src/runtime/server/api/findOne.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { createError, defineEventHandler } from 'h3' -import { withLeadingSlash } from 'ufo' -import { cacheStorage, getContent, serverQueryContent } from '../storage' - -// This route is used to cache all the parsed content -export default defineEventHandler(async (event) => { - let index = await cacheStorage.getItem('index.json') as Record - if (!index) { - // Fetch all content - const data = await serverQueryContent(event).find() - - index = data.reduce((acc, item) => { - acc[item._path!] = item._id - return acc - }, {} as Record) - - await cacheStorage.setItem('index.json', index) - } - - const slug = withLeadingSlash(event.context.params.slug) - - if (!index[slug]) { - throw createError({ - statusMessage: 'Document not found!', - statusCode: 404, - data: { - description: 'Could not find document for the given path.', - path: slug - } - }) - } - - return getContent(event, index[slug]) -}) diff --git a/src/runtime/server/storage.ts b/src/runtime/server/storage.ts index c082c04b7..b7ca085e3 100644 --- a/src/runtime/server/storage.ts +++ b/src/runtime/server/storage.ts @@ -186,6 +186,23 @@ export async function parseContent (id: string, content: string, opts: ParseCont return result } +export async function getIndex (event: CompatibilityEvent) { + let index = await cacheStorage.getItem('index.json') as Record + if (!index) { + // Fetch all content + const data = await serverQueryContent(event).find() + + index = data.reduce((acc, item) => { + acc[item._path!] = item._id + return acc + }, {} as Record) + + await cacheStorage.setItem('index.json', index) + } + + return index +} + export const createServerQueryFetch = (event: CompatibilityEvent, path?: string) => (query: QueryBuilder) => { if (path) { if (query.params().first) { @@ -200,8 +217,27 @@ export const createServerQueryFetch = (event: CompatibilityEv query.sort({ _file: 1, $numeric: true }) } + async function getList () { + const params = query.params() + const path = params?.where?.find(wh => wh._path)?._path + + if (path) { + const index = await getIndex(event) + const keys = Object.keys(index) + .filter(key => (path as any).test ? (path as any).test(path) : key === String(path)) + .map(key => index[key]) + + if (keys.length) { + const contents = await Promise.all(keys.map(key => getContent(event, key))) + return contents + } + } + + return getContentsList(event) + } + return createPipelineFetcher( - () => getContentsList(event) as unknown as Promise + getList as unknown as () => Promise )(query) } From cca3890065d80445729ebe512f1f11b4b0b7a33e Mon Sep 17 00:00:00 2001 From: Ahad Birang Date: Thu, 4 Aug 2022 14:55:11 +0430 Subject: [PATCH 3/9] chore: remove fallback --- src/runtime/server/storage.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/runtime/server/storage.ts b/src/runtime/server/storage.ts index b7ca085e3..d3230b4b7 100644 --- a/src/runtime/server/storage.ts +++ b/src/runtime/server/storage.ts @@ -221,16 +221,15 @@ export const createServerQueryFetch = (event: CompatibilityEv const params = query.params() const path = params?.where?.find(wh => wh._path)?._path - if (path) { + // Index is available for string and RegExp paths + if ((typeof path === 'string' || path instanceof RegExp)) { const index = await getIndex(event) const keys = Object.keys(index) .filter(key => (path as any).test ? (path as any).test(path) : key === String(path)) .map(key => index[key]) - if (keys.length) { - const contents = await Promise.all(keys.map(key => getContent(event, key))) - return contents - } + const contents = await Promise.all(keys.map(key => getContent(event, key))) + return contents } return getContentsList(event) From 99ecea4cbf6e9589aa2d9a7352a999519081a612 Mon Sep 17 00:00:00 2001 From: Ahad Birang Date: Thu, 4 Aug 2022 15:22:25 +0430 Subject: [PATCH 4/9] chore: refactor --- src/runtime/server/content-index.ts | 39 ++++++++++++++++++++++++++++ src/runtime/server/storage.ts | 40 ++--------------------------- src/runtime/types.d.ts | 2 +- 3 files changed, 42 insertions(+), 39 deletions(-) create mode 100644 src/runtime/server/content-index.ts diff --git a/src/runtime/server/content-index.ts b/src/runtime/server/content-index.ts new file mode 100644 index 000000000..7d0bf3291 --- /dev/null +++ b/src/runtime/server/content-index.ts @@ -0,0 +1,39 @@ +import type { CompatibilityEvent } from 'h3' +import type { ParsedContent, QueryBuilder } from '../types' +import { cacheStorage, getContent, getContentsList, serverQueryContent } from './storage' + +let contentIndex: Record +export async function getContentIndex (event: CompatibilityEvent) { + contentIndex = contentIndex || await cacheStorage.getItem('content-index.json') as Record + if (!contentIndex) { + // Fetch all content + const data = await serverQueryContent(event).find() + + contentIndex = data.reduce((acc, item) => { + acc[item._path!] = item._id + return acc + }, {} as Record) + + await cacheStorage.setItem('content-index.json', contentIndex) + } + + return contentIndex +} + +export async function getIndexedContentsList (event: CompatibilityEvent, query: QueryBuilder): Promise { + const params = query.params() + const path = params?.where?.find(wh => wh._path)?._path + + // Index is available for string and RegExp paths + if ((typeof path === 'string' || path instanceof RegExp)) { + const index = await getContentIndex(event) + const keys = Object.keys(index) + .filter(key => (path as any).test ? (path as any).test(path) : key === String(path)) + .map(key => index[key]) + + const contents = await Promise.all(keys.map(key => getContent(event, key))) + return contents as unknown as Promise + } + + return getContentsList(event) as unknown as Promise +} diff --git a/src/runtime/server/storage.ts b/src/runtime/server/storage.ts index d3230b4b7..24afbfd66 100644 --- a/src/runtime/server/storage.ts +++ b/src/runtime/server/storage.ts @@ -10,6 +10,7 @@ import { createPipelineFetcher } from '../query/match/pipeline' import { transformContent } from '../transformers' import type { ModuleOptions } from '../../module' import { getPreview, isPreview } from './preview' +import { getIndexedContentsList } from './content-index' // eslint-disable-next-line import/named // @ts-ignore import { useNitroApp, useRuntimeConfig, useStorage } from '#imports' @@ -186,23 +187,6 @@ export async function parseContent (id: string, content: string, opts: ParseCont return result } -export async function getIndex (event: CompatibilityEvent) { - let index = await cacheStorage.getItem('index.json') as Record - if (!index) { - // Fetch all content - const data = await serverQueryContent(event).find() - - index = data.reduce((acc, item) => { - acc[item._path!] = item._id - return acc - }, {} as Record) - - await cacheStorage.setItem('index.json', index) - } - - return index -} - export const createServerQueryFetch = (event: CompatibilityEvent, path?: string) => (query: QueryBuilder) => { if (path) { if (query.params().first) { @@ -217,27 +201,7 @@ export const createServerQueryFetch = (event: CompatibilityEv query.sort({ _file: 1, $numeric: true }) } - async function getList () { - const params = query.params() - const path = params?.where?.find(wh => wh._path)?._path - - // Index is available for string and RegExp paths - if ((typeof path === 'string' || path instanceof RegExp)) { - const index = await getIndex(event) - const keys = Object.keys(index) - .filter(key => (path as any).test ? (path as any).test(path) : key === String(path)) - .map(key => index[key]) - - const contents = await Promise.all(keys.map(key => getContent(event, key))) - return contents - } - - return getContentsList(event) - } - - return createPipelineFetcher( - getList as unknown as () => Promise - )(query) + return createPipelineFetcher(() => getIndexedContentsList(event, query))(query) } /** diff --git a/src/runtime/types.d.ts b/src/runtime/types.d.ts index a8df4b855..a4c62063e 100644 --- a/src/runtime/types.d.ts +++ b/src/runtime/types.d.ts @@ -471,7 +471,7 @@ export interface QueryBuilder { /** * Fetch sorround contents */ - findSurround(query: string | object, options?: Partial<{ before: number; after: number }>): Promise> + findSurround(query: string | QueryBuilderWhere, options?: Partial<{ before: number; after: number }>): Promise> /** * Filter contents based on locale From 050cdef1868ecadc725526e1116804c047ef4b40 Mon Sep 17 00:00:00 2001 From: Ahad Birang Date: Thu, 4 Aug 2022 15:47:18 +0430 Subject: [PATCH 5/9] fix: regex path --- src/runtime/server/content-index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runtime/server/content-index.ts b/src/runtime/server/content-index.ts index 7d0bf3291..e8e897043 100644 --- a/src/runtime/server/content-index.ts +++ b/src/runtime/server/content-index.ts @@ -28,7 +28,7 @@ export async function getIndexedContentsList (event: Compatib if ((typeof path === 'string' || path instanceof RegExp)) { const index = await getContentIndex(event) const keys = Object.keys(index) - .filter(key => (path as any).test ? (path as any).test(path) : key === String(path)) + .filter(key => (path as any).test ? (path as any).test(key) : key === String(path)) .map(key => index[key]) const contents = await Promise.all(keys.map(key => getContent(event, key))) From 83a95740ff2638c62f05fecad4e15cc774ae301b Mon Sep 17 00:00:00 2001 From: Ahad Birang Date: Thu, 4 Aug 2022 16:16:53 +0430 Subject: [PATCH 6/9] chore: clear index on file change --- src/module.ts | 6 +++++- src/runtime/server/content-index.ts | 3 +-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/module.ts b/src/module.ts index 370f442ec..dbcd0ad5d 100644 --- a/src/module.ts +++ b/src/module.ts @@ -530,12 +530,16 @@ export default defineNuxtModule({ nitro.options.runtimeConfig.public.content.wsUrl = url.replace('http', 'ws') // Watch contents - await nitro.storage.watch((event: WatchEvent, key: string) => { + await nitro.storage.watch(async (event: WatchEvent, key: string) => { // Ignore events that are not related to content if (!key.startsWith(MOUNT_PREFIX)) { return } key = key.substring(MOUNT_PREFIX.length) + + // Remove content Index + await nitro.storage.removeItem('cache:content:content-index.json') + // Broadcast a message to the server to refresh the page ws.broadcast({ event, key }) }) diff --git a/src/runtime/server/content-index.ts b/src/runtime/server/content-index.ts index e8e897043..59c4a78d2 100644 --- a/src/runtime/server/content-index.ts +++ b/src/runtime/server/content-index.ts @@ -2,9 +2,8 @@ import type { CompatibilityEvent } from 'h3' import type { ParsedContent, QueryBuilder } from '../types' import { cacheStorage, getContent, getContentsList, serverQueryContent } from './storage' -let contentIndex: Record export async function getContentIndex (event: CompatibilityEvent) { - contentIndex = contentIndex || await cacheStorage.getItem('content-index.json') as Record + let contentIndex = await cacheStorage.getItem('content-index.json') as Record if (!contentIndex) { // Fetch all content const data = await serverQueryContent(event).find() From 25ea6a88c2ecda04f1f57194e8980399446c930a Mon Sep 17 00:00:00 2001 From: Ahad Birang Date: Thu, 4 Aug 2022 17:17:59 +0430 Subject: [PATCH 7/9] fix: disable index in preview mode --- src/runtime/server/content-index.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/runtime/server/content-index.ts b/src/runtime/server/content-index.ts index 59c4a78d2..2c3f91518 100644 --- a/src/runtime/server/content-index.ts +++ b/src/runtime/server/content-index.ts @@ -1,5 +1,6 @@ import type { CompatibilityEvent } from 'h3' import type { ParsedContent, QueryBuilder } from '../types' +import { isPreview } from './preview' import { cacheStorage, getContent, getContentsList, serverQueryContent } from './storage' export async function getContentIndex (event: CompatibilityEvent) { @@ -24,7 +25,7 @@ export async function getIndexedContentsList (event: Compatib const path = params?.where?.find(wh => wh._path)?._path // Index is available for string and RegExp paths - if ((typeof path === 'string' || path instanceof RegExp)) { + if (!isPreview(event) && (typeof path === 'string' || path instanceof RegExp)) { const index = await getContentIndex(event) const keys = Object.keys(index) .filter(key => (path as any).test ? (path as any).test(key) : key === String(path)) From 2bf5cb7ba068ecc8f94722b8c4603d20a01861c4 Mon Sep 17 00:00:00 2001 From: Ahad Birang Date: Thu, 4 Aug 2022 17:18:24 +0430 Subject: [PATCH 8/9] feat: create navigation cache --- src/runtime/server/api/cache.ts | 12 ++++++------ src/runtime/server/api/navigation.ts | 10 +++++++++- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/runtime/server/api/cache.ts b/src/runtime/server/api/cache.ts index 375f47fc2..589c4dd08 100644 --- a/src/runtime/server/api/cache.ts +++ b/src/runtime/server/api/cache.ts @@ -1,18 +1,18 @@ import { defineEventHandler } from 'h3' +import { getContentIndex } from '../content-index' import { cacheStorage, serverQueryContent } from '../storage' // This route is used to cache all the parsed content export default defineEventHandler(async (event) => { const now = Date.now() // Fetch all content - const data = await serverQueryContent(event).find() + await serverQueryContent(event).find() // Generate Index - const index = data.reduce((acc, item) => { - acc[item._path!] = item._id - return acc - }, {} as Record) - await cacheStorage.setItem('index.json', index) + await getContentIndex(event) + + const navigation = await $fetch('/api/_content/navigation') + await cacheStorage.setItem('content-navigation.json', navigation) return { generatedAt: now, diff --git a/src/runtime/server/api/navigation.ts b/src/runtime/server/api/navigation.ts index 6bd04fc55..ef2a57c2c 100644 --- a/src/runtime/server/api/navigation.ts +++ b/src/runtime/server/api/navigation.ts @@ -1,12 +1,20 @@ import { defineEventHandler } from 'h3' -import { serverQueryContent } from '../storage' +import { cacheStorage, serverQueryContent } from '../storage' import { createNav } from '../navigation' import { ParsedContentMeta } from '../../types' import { getContentQuery } from '../../utils/query' +import { isPreview } from '../preview' export default defineEventHandler(async (event) => { const query = getContentQuery(event) + if (!isPreview(event) && Object.keys(query).length === 0) { + const cache = cacheStorage.getItem('content-navigation.json') + if (cache) { + return cache + } + } + const contents = await serverQueryContent(event, query) .where({ /** From 95f714f3c4a51a0d73eb44a0f7f2651eb1d0c229 Mon Sep 17 00:00:00 2001 From: Ahad Birang Date: Thu, 4 Aug 2022 17:32:48 +0430 Subject: [PATCH 9/9] chore: comments --- src/runtime/server/api/navigation.ts | 1 + src/runtime/server/content-index.ts | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/runtime/server/api/navigation.ts b/src/runtime/server/api/navigation.ts index ef2a57c2c..7bc4bb7a7 100644 --- a/src/runtime/server/api/navigation.ts +++ b/src/runtime/server/api/navigation.ts @@ -8,6 +8,7 @@ import { isPreview } from '../preview' export default defineEventHandler(async (event) => { const query = getContentQuery(event) + // Read from cache if not preview and there is no query if (!isPreview(event) && Object.keys(query).length === 0) { const cache = cacheStorage.getItem('content-navigation.json') if (cache) { diff --git a/src/runtime/server/content-index.ts b/src/runtime/server/content-index.ts index 2c3f91518..897ef9b99 100644 --- a/src/runtime/server/content-index.ts +++ b/src/runtime/server/content-index.ts @@ -24,7 +24,7 @@ export async function getIndexedContentsList (event: Compatib const params = query.params() const path = params?.where?.find(wh => wh._path)?._path - // Index is available for string and RegExp paths + // Read from Index is not preview and path is string or RegExp if (!isPreview(event) && (typeof path === 'string' || path instanceof RegExp)) { const index = await getContentIndex(event) const keys = Object.keys(index)