diff --git a/docs/markdown.md b/docs/markdown.md index 87a03b8e8..5f8d6402d 100644 --- a/docs/markdown.md +++ b/docs/markdown.md @@ -18,7 +18,7 @@ The front matter supports the following options: - **title** - the page title; defaults to the (first) first-level heading of the page, if any - **index** - whether to index this page if [search](./search) is enabled; defaults to true for listed pages - **keywords** - additional words to index for [search](./search); boosted at the same weight as the title -- **draft** - whether to skip this page during build; drafts are also not listed in the default sidebar +- **draft** - whether to skip this page during build; drafts are also not listed in the default sidebar nor searchable - **sql** - table definitions for [SQL code blocks](./sql) The front matter can also override the following [app-level configuration](./config) options: diff --git a/src/build.ts b/src/build.ts index de844cb56..99232e0bb 100644 --- a/src/build.ts +++ b/src/build.ts @@ -71,6 +71,9 @@ export async function build( const addStylesheet = (path: string, s: string) => stylesheets.add(/^\w+:/.test(s) ? s : resolvePath(path, s)); // Load pages, building a list of additional assets as we go. + let assetCount = 0; + let pageCount = 0; + const pagePaths = new Set(); for await (const path of config.paths()) { effects.output.write(`${faint("load")} ${path} `); const start = performance.now(); @@ -86,9 +89,18 @@ export async function build( for (const s of resolvers.stylesheets) addStylesheet(path, s); effects.output.write(`${faint("in")} ${(elapsed >= 100 ? yellow : faint)(`${elapsed}ms`)}\n`); outputs.set(path, {type: "module", resolvers}); + ++assetCount; continue; } } + const file = loaders.find(path); + if (file) { + effects.output.write(`${faint("copy")} ${join(root, path)} ${faint("→")} `); + const sourcePath = join(root, await file.load({useStale: true}, effects)); + await effects.copyFile(sourcePath, path); + ++assetCount; + continue; + } const page = await loaders.loadPage(path, options, effects); if (page.data.draft) { effects.logger.log(faint("(skipped)")); @@ -102,13 +114,16 @@ export async function build( for (const i of resolvers.globalImports) addGlobalImport(path, resolvers.resolveImport(i)); for (const s of resolvers.stylesheets) addStylesheet(path, s); effects.output.write(`${faint("in")} ${(elapsed >= 100 ? yellow : faint)(`${elapsed}ms`)}\n`); + pagePaths.add(path); outputs.set(path, {type: "page", page, resolvers}); + ++pageCount; } // Check that there’s at least one output. - const outputCount = outputs.size; + const outputCount = pageCount + assetCount; if (!outputCount) throw new CliError(`Nothing to build: no pages found in your ${root} directory.`); - effects.logger.log(`${faint("built")} ${outputCount} ${faint(`page${outputCount === 1 ? "" : "s"} in`)} ${root}`); + if (pageCount) effects.logger.log(`${faint("built")} ${pageCount} ${faint(`page${pageCount === 1 ? "" : "s"} in`)} ${root}`); // prettier-ignore + if (assetCount) effects.logger.log(`${faint("built")} ${assetCount} ${faint(`asset${assetCount === 1 ? "" : "s"} in`)} ${root}`); // prettier-ignore // For cache-breaking we rename most assets to include content hashes. const aliases = new Map(); @@ -117,7 +132,7 @@ export async function build( // Add the search bundle and data, if needed. if (config.search) { globalImports.add("/_observablehq/search.js").add("/_observablehq/minisearch.json"); - const contents = await searchIndex(config, effects); + const contents = await searchIndex(config, pagePaths, effects); effects.output.write(`${faint("index →")} `); const cachePath = join(cacheRoot, "_observablehq", "minisearch.json"); await prepareOutput(cachePath); @@ -349,7 +364,7 @@ export async function build( } effects.logger.log(""); - Telemetry.record({event: "build", step: "finish", pageCount: outputCount}); + Telemetry.record({event: "build", step: "finish", pageCount}); } function applyHash(path: string, hash: string): string { diff --git a/src/search.ts b/src/search.ts index fe2b008dc..9b7173742 100644 --- a/src/search.ts +++ b/src/search.ts @@ -29,7 +29,11 @@ const indexOptions = { type MiniSearchResult = Omit & {id: string; keywords: string}; -export async function searchIndex(config: Config, effects = defaultEffects): Promise { +export async function searchIndex( + config: Config, + paths: Iterable | AsyncIterable = getDefaultSearchPaths(config), + effects = defaultEffects +): Promise { const {pages, search, normalizePath} = config; if (!search) return "{}"; const cached = indexCache.get(pages); @@ -37,7 +41,7 @@ export async function searchIndex(config: Config, effects = defaultEffects): Pro // Index the pages const index = new MiniSearch(indexOptions); - for await (const result of indexPages(config, effects)) index.add(normalizeResult(result, normalizePath)); + for await (const result of indexPages(config, paths, effects)) index.add(normalizeResult(result, normalizePath)); if (search.index) for await (const result of search.index()) index.add(normalizeResult(result, normalizePath)); // Pass the serializable index options to the client. @@ -57,8 +61,12 @@ export async function searchIndex(config: Config, effects = defaultEffects): Pro return json; } -async function* indexPages(config: Config, effects: SearchIndexEffects): AsyncIterable { - const {root, pages, loaders} = config; +async function* indexPages( + config: Config, + paths: Iterable | AsyncIterable, + effects: SearchIndexEffects +): AsyncIterable { + const {pages, loaders} = config; // Get all the listed pages (which are indexed by default) const pagePaths = new Set(["/index"]); @@ -67,8 +75,7 @@ async function* indexPages(config: Config, effects: SearchIndexEffects): AsyncIt if ("pages" in p) for (const {path} of p.pages) pagePaths.add(path); } - for await (const path of config.paths()) { - if (path.endsWith(".js") && findModule(root, path)) continue; + for await (const path of paths) { const {body, title, data} = await loaders.loadPage(path, {...config, path}); // Skip pages that opt-out of indexing, and skip unlisted pages unless @@ -97,6 +104,15 @@ async function* indexPages(config: Config, effects: SearchIndexEffects): AsyncIt } } +async function* getDefaultSearchPaths(config: Config): AsyncGenerator { + const {root, loaders} = config; + for await (const path of config.paths()) { + if (path.endsWith(".js") && findModule(root, path)) continue; // ignore modules + if (loaders.find(path)) continue; // ignore assets + yield path; + } +} + function normalizeResult( {path, keywords, ...rest}: SearchResult, normalizePath: Config["normalizePath"] diff --git a/test/files-test.ts b/test/files-test.ts index bf7f0d481..3ac7e1f12 100644 --- a/test/files-test.ts +++ b/test/files-test.ts @@ -55,10 +55,10 @@ describe("visitFiles(root)", () => { "files.md", "observable logo small.png", "observable logo.png", - "unknown-mime-extension.really", "subsection/additional-styles.css", "subsection/file-sub.csv", - "subsection/subfiles.md" + "subsection/subfiles.md", + "unknown-mime-extension.really" ]); }); it("handles circular symlinks, visiting files only once", function () { @@ -66,7 +66,7 @@ describe("visitFiles(root)", () => { assert.deepStrictEqual(collect(visitFiles("test/input/circular-files")), ["a/a.txt", "b/b.txt"]); }); it("ignores .observablehq at any level", function () { - assert.deepStrictEqual(collect(visitFiles("test/files")), ["visible.txt", "sub/visible.txt"]); + assert.deepStrictEqual(collect(visitFiles("test/files")), ["sub/visible.txt", "visible.txt"]); }); }); @@ -74,7 +74,7 @@ describe("visitFiles(root, test)", () => { it("skips directories and files that don’t pass the specified test", () => { assert.deepStrictEqual( collect(visitFiles("test/input/build/params", (name) => isParameterized(name) || extname(name) !== "")), - ["observablehq.config.js", "[dir]/index.md", "[dir]/loaded.md.js"] + ["[dir]/index.md", "[dir]/loaded.md.js", "[name]-icon.svg.js", "observablehq.config.js"] ); assert.deepStrictEqual(collect(visitFiles("test/input/build/params", (name) => !isParameterized(name))), [ "observablehq.config.js" @@ -88,5 +88,5 @@ function collect(generator: Generator): string[] { if (value.startsWith(".observablehq/cache/")) continue; values.push(value); } - return values; + return values.sort(); } diff --git a/test/input/build/params/[name]-icon.svg.js b/test/input/build/params/[name]-icon.svg.js new file mode 100644 index 000000000..848609954 --- /dev/null +++ b/test/input/build/params/[name]-icon.svg.js @@ -0,0 +1,12 @@ +import {parseArgs} from "node:util"; + +const { + values: {name} +} = parseArgs({ + options: {name: {type: "string"}} +}); + +process.stdout.write(` + ${name} + +`); diff --git a/test/input/build/params/observablehq.config.js b/test/input/build/params/observablehq.config.js index 0a180653d..e4364b6ed 100644 --- a/test/input/build/params/observablehq.config.js +++ b/test/input/build/params/observablehq.config.js @@ -1,5 +1,5 @@ export default { async *dynamicPaths() { - yield* ["/bar/index", "/bar/loaded", "/foo/bar", "/foo/index"]; + yield* ["/bar/index", "/bar/loaded", "/foo/bar", "/foo/index", "/observable-icon.svg"]; } }; diff --git a/test/output/build/params/observable-icon.svg b/test/output/build/params/observable-icon.svg new file mode 100644 index 000000000..25697c298 --- /dev/null +++ b/test/output/build/params/observable-icon.svg @@ -0,0 +1,3 @@ + + observable +