refactor the route cache and other build internals (#2503)

* refactor dev to use vite server * refactor the route cache and other build internals * use debug package for debug logs (#2504) Co-authored-by: Matthew Phillips <[email protected]>
withastro · Jan 31, 2022 · de9fadb · de9fadb
1 parent d7149f9
commit de9fadb
Show file tree

Hide file tree

Showing 14 changed files with 190 additions and 196 deletions.
diff --git a/packages/astro/package.json b/packages/astro/package.json
@@ -69,6 +69,7 @@
     "@proload/core": "^0.2.1",
     "@proload/plugin-tsm": "^0.1.0",
     "@types/babel__core": "^7.1.15",
+    "@types/debug": "^4.1.7",
     "@web/parse5-utils": "^1.3.0",
     "astring": "^1.7.5",
     "ci-info": "^3.2.0",

diff --git a/packages/astro/src/@types/astro.ts b/packages/astro/src/@types/astro.ts
@@ -341,8 +341,6 @@ export interface RouteData {
 	type: 'page';
 }
 
-export type RouteCache = Record<string, GetStaticPathsResultKeyed>;
-
 export type RuntimeMode = 'development' | 'production';
 
 /**
@@ -385,7 +383,7 @@ export interface RSS {
 	}[];
 }
 
-export type RSSFunction = (args: RSS) => void;
+export type RSSFunction = (args: RSS) => RSSResult;
 
 export type FeedResult = { url: string; content?: string };
 export type RSSResult = { xml: FeedResult; xsl?: FeedResult };

diff --git a/packages/astro/src/cli/index.ts b/packages/astro/src/cli/index.ts
@@ -90,12 +90,8 @@ export async function cli(args: string[]) {
 	try {
 		config = await loadConfig({ cwd: projectRoot, flags });
 	} catch (err) {
-		if (err instanceof z.ZodError) {
-			console.error(formatConfigError(err));
-		} else {
-			console.error(colors.red((err as any).toString() || err));
-		}
-		process.exit(1);
+		throwAndExit(err);
+		return;
 	}
 
 	switch (cmd) {
@@ -143,6 +139,13 @@ export async function cli(args: string[]) {
 
 /** Display error and exit */
 function throwAndExit(err: any) {
-	console.error(colors.red(err.toString() || err));
+	if (err instanceof z.ZodError) {
+		console.error(formatConfigError(err));
+	} else if (err.stack) {
+		const [mainMsg, ...stackMsg] = err.stack.split('\n');
+		console.error(colors.red(mainMsg) + '\n' + colors.dim(stackMsg.join('\n')));
+	} else {
+		console.error(colors.red(err.toString() || err));
+	}
 	process.exit(1);
 }
diff --git a/packages/astro/src/core/build/index.ts b/packages/astro/src/core/build/index.ts
@@ -1,4 +1,4 @@
-import type { AstroConfig, ManifestData, RouteCache } from '../../@types/astro';
+import type { AstroConfig, ManifestData } from '../../@types/astro';
 import type { LogOptions } from '../logger';
 
 import fs from 'fs';
@@ -13,6 +13,7 @@ import { generateSitemap } from '../ssr/sitemap.js';
 import { collectPagesData } from './page-data.js';
 import { build as scanBasedBuild } from './scan-based-build.js';
 import { staticBuild } from './static-build.js';
+import { RouteCache } from '../ssr/route-cache.js';
 
 export interface BuildOptions {
 	mode?: string;
@@ -35,7 +36,7 @@ class AstroBuilder {
 	private logging: LogOptions;
 	private mode = 'production';
 	private origin: string;
-	private routeCache: RouteCache = {};
+	private routeCache: RouteCache;
 	private manifest: ManifestData;
 	private viteServer?: ViteDevServer;
 	private viteConfig?: ViteConfigWithSSR;
@@ -49,6 +50,7 @@ class AstroBuilder {
 		this.config = config;
 		const port = config.devOptions.port; // no need to save this (don’t rely on port in builder)
 		this.logging = options.logging;
+		this.routeCache = new RouteCache(this.logging);
 		this.origin = config.buildOptions.site ? new URL(config.buildOptions.site).origin : `http://localhost:${port}`;
 		this.manifest = createRouteManifest({ config }, this.logging);
 	}
@@ -74,7 +76,7 @@ class AstroBuilder {
 		this.viteConfig = viteConfig;
 		const viteServer = await vite.createServer(viteConfig);
 		this.viteServer = viteServer;
-		debug(logging, 'build', timerMessage('Vite started', timer.viteStart));
+		debug('build', timerMessage('Vite started', timer.viteStart));
 
 		timer.loadStart = performance.now();
 		const { assets, allPages } = await collectPagesData({
@@ -92,13 +94,13 @@ class AstroBuilder {
 				// TODO: add better type inference to data.preload[1]
 				const frontmatter = (data.preload[1] as any).frontmatter;
 				if (Boolean(frontmatter.draft) && !this.config.buildOptions.drafts) {
-					debug(logging, 'build', timerMessage(`Skipping draft page ${page}`, timer.loadStart));
+					debug('build', timerMessage(`Skipping draft page ${page}`, timer.loadStart));
 					delete allPages[page];
 				}
 			}
 		});
 
-		debug(logging, 'build', timerMessage('All pages loaded', timer.loadStart));
+		debug('build', timerMessage('All pages loaded', timer.loadStart));
 
 		// The names of each pages
 		const pageNames: string[] = [];
@@ -130,7 +132,7 @@ class AstroBuilder {
 				viteServer: this.viteServer,
 			});
 		}
-		debug(logging, 'build', timerMessage('Vite build finished', timer.buildStart));
+		debug('build', timerMessage('Vite build finished', timer.buildStart));
 
 		// Write any additionally generated assets to disk.
 		timer.assetsStart = performance.now();
@@ -141,7 +143,7 @@ class AstroBuilder {
 			fs.writeFileSync(filePath, assets[k], 'utf8');
 			delete assets[k]; // free up memory
 		});
-		debug(logging, 'build', timerMessage('Additional assets copied', timer.assetsStart));
+		debug('build', timerMessage('Additional assets copied', timer.assetsStart));
 
 		// Build your final sitemap.
 		timer.sitemapStart = performance.now();
@@ -151,7 +153,7 @@ class AstroBuilder {
 			await fs.promises.mkdir(new URL('./', sitemapPath), { recursive: true });
 			await fs.promises.writeFile(sitemapPath, sitemap, 'utf8');
 		}
-		debug(logging, 'build', timerMessage('Sitemap built', timer.sitemapStart));
+		debug('build', timerMessage('Sitemap built', timer.sitemapStart));
 
 		// You're done! Time to clean up.
 		await viteServer.close();
@@ -162,8 +164,6 @@ class AstroBuilder {
 
 	/** Stats */
 	private async printStats({ logging, timeStart, pageCount }: { logging: LogOptions; timeStart: number; pageCount: number }) {
-		/* eslint-disable no-console */
-		debug(logging, ''); // empty line for debug
 		const buildTime = performance.now() - timeStart;
 		const total = buildTime < 750 ? `${Math.round(buildTime)}ms` : `${(buildTime / 1000).toFixed(2)}s`;
 		const perPage = `${Math.round(buildTime / pageCount)}ms`;

diff --git a/packages/astro/src/core/build/page-data.ts b/packages/astro/src/core/build/page-data.ts
@@ -1,4 +1,4 @@
-import type { AstroConfig, ComponentInstance, GetStaticPathsResult, ManifestData, RouteCache, RouteData, RSSResult } from '../../@types/astro';
+import type { AstroConfig, ComponentInstance, ManifestData, RouteData, RSSResult } from '../../@types/astro';
 import type { AllPagesData } from './types';
 import type { LogOptions } from '../logger';
 import type { ViteDevServer } from '../vite.js';
@@ -7,10 +7,8 @@ import { fileURLToPath } from 'url';
 import * as colors from 'kleur/colors';
 import { debug } from '../logger.js';
 import { preload as ssrPreload } from '../ssr/index.js';
-import { validateGetStaticPathsModule, validateGetStaticPathsResult } from '../ssr/routing.js';
-import { generatePaginateFunction } from '../ssr/paginate.js';
 import { generateRssFunction } from '../ssr/rss.js';
-import { assignStaticPaths } from '../ssr/route-cache.js';
+import { callGetStaticPaths, RouteCache, RouteCacheEntry } from '../ssr/route-cache.js';
 
 export interface CollectPagesDataOptions {
 	astroConfig: AstroConfig;
@@ -57,62 +55,62 @@ export async function collectPagesData(opts: CollectPagesDataOptions): Promise<C
 					})
 						.then((routes) => {
 							const html = `${route.pathname}`.replace(/\/?$/, '/index.html');
-							debug(logging, 'build', `├── ${colors.bold(colors.green('✔'))} ${route.component} → ${colors.yellow(html)}`);
+							debug('build', `├── ${colors.bold(colors.green('✔'))} ${route.component} → ${colors.yellow(html)}`);
 							return routes;
 						})
 						.catch((err) => {
-							debug(logging, 'build', `├── ${colors.bold(colors.red('✘'))} ${route.component}`);
+							debug('build', `├── ${colors.bold(colors.red('✘'))} ${route.component}`);
 							throw err;
 						}),
 				};
 				return;
 			}
 			// dynamic route:
 			const result = await getStaticPathsForRoute(opts, route)
-				.then((routes) => {
-					const label = routes.paths.length === 1 ? 'page' : 'pages';
-					debug(logging, 'build', `├── ${colors.bold(colors.green('✔'))} ${route.component} → ${colors.magenta(`[${routes.paths.length} ${label}]`)}`);
-					return routes;
+				.then((_result) => {
+					const label = _result.staticPaths.length === 1 ? 'page' : 'pages';
+					debug('build', `├── ${colors.bold(colors.green('✔'))} ${route.component} → ${colors.magenta(`[${_result.staticPaths.length} ${label}]`)}`);
+					return _result;
 				})
 				.catch((err) => {
-					debug(logging, 'build', `├── ${colors.bold(colors.red('✗'))} ${route.component}`);
+					debug('build', `├── ${colors.bold(colors.red('✗'))} ${route.component}`);
 					throw err;
 				});
-			if (result.rss?.length) {
-				for (let i = 0; i < result.rss.length; i++) {
-					const rss = result.rss[i];
-					if (rss.xml) {
-						const { url, content } = rss.xml;
-						if (content) {
-							const rssFile = new URL(url.replace(/^\/?/, './'), astroConfig.dist);
-							if (assets[fileURLToPath(rssFile)]) {
-								throw new Error(`[getStaticPaths] RSS feed ${url} already exists.\nUse \`rss(data, {url: '...'})\` to choose a unique, custom URL. (${route.component})`);
-							}
-							assets[fileURLToPath(rssFile)] = content;
+			const rssFn = generateRssFunction(astroConfig.buildOptions.site, route);
+			for (const rssCallArg of result.rss) {
+				const rssResult = rssFn(rssCallArg);
+				if (rssResult.xml) {
+					const { url, content } = rssResult.xml;
+					if (content) {
+						const rssFile = new URL(url.replace(/^\/?/, './'), astroConfig.dist);
+						if (assets[fileURLToPath(rssFile)]) {
+							throw new Error(`[getStaticPaths] RSS feed ${url} already exists.\nUse \`rss(data, {url: '...'})\` to choose a unique, custom URL. (${route.component})`);
 						}
+						assets[fileURLToPath(rssFile)] = content;
 					}
-					if (rss.xsl?.content) {
-						const { url, content } = rss.xsl;
-						const stylesheetFile = new URL(url.replace(/^\/?/, './'), astroConfig.dist);
-						if (assets[fileURLToPath(stylesheetFile)]) {
-							throw new Error(
-								`[getStaticPaths] RSS feed stylesheet ${url} already exists.\nUse \`rss(data, {stylesheet: '...'})\` to choose a unique, custom URL. (${route.component})`
-							);
-						}
-						assets[fileURLToPath(stylesheetFile)] = content;
+				}
+				if (rssResult.xsl?.content) {
+					const { url, content } = rssResult.xsl;
+					const stylesheetFile = new URL(url.replace(/^\/?/, './'), astroConfig.dist);
+					if (assets[fileURLToPath(stylesheetFile)]) {
+						throw new Error(
+							`[getStaticPaths] RSS feed stylesheet ${url} already exists.\nUse \`rss(data, {stylesheet: '...'})\` to choose a unique, custom URL. (${route.component})`
+						);
 					}
+					assets[fileURLToPath(stylesheetFile)] = content;
 				}
 			}
+			const finalPaths = result.staticPaths.map((staticPath) => staticPath.params && route.generate(staticPath.params)).filter(Boolean);
 			allPages[route.component] = {
 				route,
-				paths: result.paths,
+				paths: finalPaths,
 				preload: await ssrPreload({
 					astroConfig,
 					filePath: new URL(`./${route.component}`, astroConfig.projectRoot),
 					logging,
 					mode: 'production',
 					origin,
-					pathname: result.paths[0],
+					pathname: finalPaths[0],
 					route,
 					routeCache,
 					viteServer,
@@ -124,18 +122,12 @@ export async function collectPagesData(opts: CollectPagesDataOptions): Promise<C
 	return { assets, allPages };
 }
 
-async function getStaticPathsForRoute(opts: CollectPagesDataOptions, route: RouteData): Promise<{ paths: string[]; rss?: RSSResult[] }> {
+async function getStaticPathsForRoute(opts: CollectPagesDataOptions, route: RouteData): Promise<RouteCacheEntry> {
 	const { astroConfig, logging, routeCache, viteServer } = opts;
 	if (!viteServer) throw new Error(`vite.createServer() not called!`);
 	const filePath = new URL(`./${route.component}`, astroConfig.projectRoot);
 	const mod = (await viteServer.ssrLoadModule(fileURLToPath(filePath))) as ComponentInstance;
-	validateGetStaticPathsModule(mod);
-	const rss = generateRssFunction(astroConfig.buildOptions.site, route);
-	await assignStaticPaths(routeCache, route, mod, rss.generator);
-	const staticPaths = routeCache[route.component];
-	validateGetStaticPathsResult(staticPaths, logging);
-	return {
-		paths: staticPaths.map((staticPath) => staticPath.params && route.generate(staticPath.params)).filter(Boolean),
-		rss: rss.rss,
-	};
+	const result = await callGetStaticPaths(mod, route, false, logging);
+	routeCache.set(route, result);
+	return result;
 }
diff --git a/packages/astro/src/core/build/scan-based-build.ts b/packages/astro/src/core/build/scan-based-build.ts
@@ -1,5 +1,5 @@
 import type { ViteDevServer } from '../vite.js';
-import type { AstroConfig, RouteCache } from '../../@types/astro';
+import type { AstroConfig } from '../../@types/astro';
 import type { AllPagesData } from './types';
 import type { LogOptions } from '../logger';
 import type { ViteConfigWithSSR } from '../create-vite.js';
@@ -9,6 +9,7 @@ import vite from '../vite.js';
 import { createBuildInternals } from '../../core/build/internal.js';
 import { rollupPluginAstroBuildHTML } from '../../vite-plugin-build-html/index.js';
 import { rollupPluginAstroBuildCSS } from '../../vite-plugin-build-css/index.js';
+import { RouteCache } from '../ssr/route-cache.js';
 
 export interface ScanBasedBuildOptions {
 	allPages: AllPagesData;

diff --git a/packages/astro/src/core/build/static-build.ts b/packages/astro/src/core/build/static-build.ts
@@ -1,6 +1,6 @@
 import type { OutputChunk, OutputAsset, PreRenderedChunk, RollupOutput } from 'rollup';
 import type { Plugin as VitePlugin, UserConfig } from '../vite';
-import type { AstroConfig, Renderer, RouteCache, SSRElement } from '../../@types/astro';
+import type { AstroConfig, Renderer, SSRElement } from '../../@types/astro';
 import type { AllPagesData } from './types';
 import type { LogOptions } from '../logger';
 import type { ViteConfigWithSSR } from '../create-vite';
@@ -22,6 +22,7 @@ import { createResult } from '../ssr/result.js';
 import { renderPage } from '../../runtime/server/index.js';
 import { prepareOutDir } from './fs.js';
 import { vitePluginHoistedScripts } from './vite-plugin-hoisted-scripts.js';
+import { RouteCache } from '../ssr/route-cache.js';
 
 export interface StaticBuildOptions {
 	allPages: AllPagesData;
@@ -182,7 +183,7 @@ async function ssrBuild(opts: StaticBuildOptions, internals: BuildInternals, inp
 		root: viteConfig.root,
 		envPrefix: 'PUBLIC_',
 		server: viteConfig.server,
-		base: astroConfig.buildOptions.site ? fileURLToPath(new URL(astroConfig.buildOptions.site)) : '/',
+		base: astroConfig.buildOptions.site ? new URL(astroConfig.buildOptions.site).pathname : '/',
 		ssr: viteConfig.ssr,
 	} as ViteConfigWithSSR);
 }
@@ -223,7 +224,7 @@ async function clientBuild(opts: StaticBuildOptions, internals: BuildInternals,
 		root: viteConfig.root,
 		envPrefix: 'PUBLIC_',
 		server: viteConfig.server,
-		base: astroConfig.buildOptions.site ? fileURLToPath(new URL(astroConfig.buildOptions.site)) : '/',
+		base: astroConfig.buildOptions.site ? new URL(astroConfig.buildOptions.site).pathname : '/',
 	});
 }
 
@@ -255,7 +256,7 @@ async function collectRenderers(opts: StaticBuildOptions): Promise<Renderer[]> {
 }
 
 async function generatePages(result: RollupOutput, opts: StaticBuildOptions, internals: BuildInternals, facadeIdToPageDataMap: Map<string, PageBuildData>) {
-	debug(opts.logging, 'generate', 'End build step, now generating');
+	debug('build', 'Finish build. Begin generating.');
 
 	// Get renderers to be shared for each page generation.
 	const renderers = await collectRenderers(opts);
@@ -330,15 +331,10 @@ async function generatePath(pathname: string, opts: StaticBuildOptions, gopts: G
 		const [params, pageProps] = await getParamsAndProps({
 			route: pageData.route,
 			routeCache,
-			logging,
 			pathname,
-			mod,
-			// Do not validate as validation already occurred for static routes
-			// and validation is relatively expensive.
-			validate: false,
 		});
 
-		debug(logging, 'generate', `Generating: ${pathname}`);
+		debug('build', `Generating: ${pathname}`);
 
 		const rootpath = new URL(astroConfig.buildOptions.site || 'http://localhost/').pathname;
 		const links = new Set<SSRElement>(