From 5f12c53a3c9f69218b3614fb587e124280425c3a Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Mon, 13 Mar 2023 15:57:17 +0100 Subject: [PATCH 1/6] feat(sitemap): support SSR generated routes --- packages/integrations/sitemap/package.json | 1 + packages/integrations/sitemap/src/index.ts | 71 ++++++++++++++++--- .../sitemap/src/utils/is-route-prerendered.ts | 8 +++ pnpm-lock.yaml | 12 +++- 4 files changed, 79 insertions(+), 13 deletions(-) create mode 100644 packages/integrations/sitemap/src/utils/is-route-prerendered.ts diff --git a/packages/integrations/sitemap/package.json b/packages/integrations/sitemap/package.json index afd959a62c92..3ea6b2381442 100644 --- a/packages/integrations/sitemap/package.json +++ b/packages/integrations/sitemap/package.json @@ -33,6 +33,7 @@ "test": "mocha --timeout 20000" }, "dependencies": { + "fast-glob": "^3.2.11", "sitemap": "^7.1.1", "zod": "^3.17.3" }, diff --git a/packages/integrations/sitemap/src/index.ts b/packages/integrations/sitemap/src/index.ts index e6e45ddd1dd6..ad73e24980d3 100644 --- a/packages/integrations/sitemap/src/index.ts +++ b/packages/integrations/sitemap/src/index.ts @@ -5,11 +5,14 @@ import { type LinkItem as LinkItemBase, type SitemapItemLoose, } from 'sitemap'; +import { resolve } from 'path'; import { fileURLToPath } from 'url'; +import fg from 'fast-glob'; import { ZodError } from 'zod'; import { generateSitemap } from './generate-sitemap.js'; import { Logger } from './utils/logger.js'; +import { isRoutePrerendered } from './utils/is-route-prerendered.js'; import { validateOptions } from './validate-options.js'; export type ChangeFreq = `${EnumChangefreq}`; @@ -51,18 +54,71 @@ const OUTFILE = 'sitemap-index.xml'; const createPlugin = (options?: SitemapOptions): AstroIntegration => { let config: AstroConfig; + const logger = new Logger(PKG_NAME); + return { name: PKG_NAME, hooks: { 'astro:config:done': async ({ config: cfg }) => { - config = cfg; + if (cfg.site) { + config = cfg; + } else { + // eslint-disable-next-line no-console + console.warn( + 'The Sitemap integration requires the `site` astro.config option. Skipping.' + ); + return; + } }, - 'astro:build:done': async ({ dir, pages }) => { - const logger = new Logger(PKG_NAME); + 'astro:build:start': async () => { + if (config.output !== 'server' || !config.site) { + return; + } + + const srcPath = fileURLToPath(config.srcDir); + const pagesPath = resolve(srcPath, 'pages'); + + const pageFiles = await fg(`${pagesPath}/**/*.{astro,ts,js}`); + + const routes = ( + await Promise.all( + pageFiles.map(async (filePath) => { + const isPrerendered = await isRoutePrerendered(filePath); + const index = filePath.indexOf('pages/') + 6; + const routeSegment = filePath + .substring(index) + .replace(/\.(astro|ts|js)/, '') + .replace(/index$/, ''); + + /** + * @TODO + * figure out how to run `getStaticPaths` here. + */ + const isDynamicRoute = routeSegment.endsWith(']'); + const shouldIndex = !isDynamicRoute && !isPrerendered; + + return shouldIndex ? `${config.site}/${routeSegment}` : undefined; + }) + ) + ).filter((route): route is string => Boolean(route)); + const opts = validateOptions(config.site, options); + + opts.customPages = opts.customPages + ? Array.from(new Set([...routes, ...opts.customPages])) + : routes; + options = opts; + + logger.info(`build is starting + ${JSON.stringify(opts.customPages, null, 2)}`); + }, + 'astro:build:done': async ({ dir, pages }) => { try { + if (!config.site) { + return; + } + const opts = validateOptions(config.site, options); const { filter, customPages, serialize, entryLimit } = opts; @@ -99,14 +155,7 @@ const createPlugin = (options?: SitemapOptions): AstroIntegration => { } if (pageUrls.length === 0) { - // offer suggestion for SSR users - if (config.output !== 'static') { - logger.warn( - `No pages found! We can only detect sitemap routes for "static" builds. Since you are using an SSR adapter, we recommend manually listing your sitemap routes using the "customPages" integration option.\n\nExample: \`sitemap({ customPages: ['https://example.com/route'] })\`` - ); - } else { - logger.warn(`No pages found!\n\`${OUTFILE}\` not created.`); - } + logger.warn(`No pages found!\n\`${OUTFILE}\` not created.`); return; } diff --git a/packages/integrations/sitemap/src/utils/is-route-prerendered.ts b/packages/integrations/sitemap/src/utils/is-route-prerendered.ts new file mode 100644 index 000000000000..468e09ba8489 --- /dev/null +++ b/packages/integrations/sitemap/src/utils/is-route-prerendered.ts @@ -0,0 +1,8 @@ +const REGEX_PRERENDER = /const prerender = true/g; +import { readFile } from 'fs/promises'; + +export async function isRoutePrerendered(filePath: string) { + const contents = await readFile(filePath, 'utf-8'); + + return REGEX_PRERENDER.test(contents); +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bf43c28474cd..f7a037b45791 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -668,7 +668,7 @@ importers: version: 3.0.1 typescript: specifier: '*' - version: 5.0.2 + version: 4.9.5 unist-util-visit: specifier: ^4.1.0 version: 4.1.0 @@ -4492,6 +4492,9 @@ importers: packages/integrations/sitemap: dependencies: + fast-glob: + specifier: ^3.2.11 + version: 3.2.12 sitemap: specifier: ^7.1.1 version: 7.1.1 @@ -15581,7 +15584,7 @@ packages: shiki: 0.10.1 shiki-twoslash: 3.1.0 tslib: 2.1.0 - typescript: 5.0.2 + typescript: 4.9.5 unist-util-visit: 2.0.3 transitivePeerDependencies: - supports-color @@ -16890,6 +16893,11 @@ packages: for-each: 0.3.3 is-typed-array: 1.1.10 + /typescript@4.9.5: + resolution: {integrity: sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==} + engines: {node: '>=4.2.0'} + hasBin: true + /typescript@5.0.2: resolution: {integrity: sha512-wVORMBGO/FAs/++blGNeAVdbNKtIh1rbBL2EyQ1+J9lClJ93KiiKe8PmFIVdXhHcyv44SL9oglmfeSsndo0jRw==} engines: {node: '>=12.20'} From 08a4108adf56254294d3494874c31db478dae272 Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Mon, 13 Mar 2023 16:15:50 +0100 Subject: [PATCH 2/6] feat(sitemap): add changeset for SSR support --- .changeset/chatty-dolls-visit.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/chatty-dolls-visit.md diff --git a/.changeset/chatty-dolls-visit.md b/.changeset/chatty-dolls-visit.md new file mode 100644 index 000000000000..6b9e53e88e72 --- /dev/null +++ b/.changeset/chatty-dolls-visit.md @@ -0,0 +1,5 @@ +--- +'@astrojs/sitemap': minor +--- + +Adds support to SSR routes to sitemap generation. From 6360bc27e26a07cdbcf369385301029fca2d5fb2 Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Mon, 13 Mar 2023 20:09:29 +0100 Subject: [PATCH 3/6] refactor: move logic to `astro:build:done` --- packages/integrations/sitemap/package.json | 1 - packages/integrations/sitemap/src/index.ts | 83 +++++-------------- .../sitemap/src/utils/is-route-prerendered.ts | 8 -- pnpm-lock.yaml | 3 - 4 files changed, 22 insertions(+), 73 deletions(-) delete mode 100644 packages/integrations/sitemap/src/utils/is-route-prerendered.ts diff --git a/packages/integrations/sitemap/package.json b/packages/integrations/sitemap/package.json index 3ea6b2381442..afd959a62c92 100644 --- a/packages/integrations/sitemap/package.json +++ b/packages/integrations/sitemap/package.json @@ -33,7 +33,6 @@ "test": "mocha --timeout 20000" }, "dependencies": { - "fast-glob": "^3.2.11", "sitemap": "^7.1.1", "zod": "^3.17.3" }, diff --git a/packages/integrations/sitemap/src/index.ts b/packages/integrations/sitemap/src/index.ts index ad73e24980d3..33e72fa23217 100644 --- a/packages/integrations/sitemap/src/index.ts +++ b/packages/integrations/sitemap/src/index.ts @@ -5,14 +5,11 @@ import { type LinkItem as LinkItemBase, type SitemapItemLoose, } from 'sitemap'; -import { resolve } from 'path'; import { fileURLToPath } from 'url'; -import fg from 'fast-glob'; import { ZodError } from 'zod'; import { generateSitemap } from './generate-sitemap.js'; import { Logger } from './utils/logger.js'; -import { isRoutePrerendered } from './utils/is-route-prerendered.js'; import { validateOptions } from './validate-options.js'; export type ChangeFreq = `${EnumChangefreq}`; @@ -61,61 +58,15 @@ const createPlugin = (options?: SitemapOptions): AstroIntegration => { hooks: { 'astro:config:done': async ({ config: cfg }) => { - if (cfg.site) { - config = cfg; - } else { - // eslint-disable-next-line no-console - console.warn( - 'The Sitemap integration requires the `site` astro.config option. Skipping.' - ); - return; - } - }, - - 'astro:build:start': async () => { - if (config.output !== 'server' || !config.site) { - return; - } - - const srcPath = fileURLToPath(config.srcDir); - const pagesPath = resolve(srcPath, 'pages'); - - const pageFiles = await fg(`${pagesPath}/**/*.{astro,ts,js}`); - - const routes = ( - await Promise.all( - pageFiles.map(async (filePath) => { - const isPrerendered = await isRoutePrerendered(filePath); - const index = filePath.indexOf('pages/') + 6; - const routeSegment = filePath - .substring(index) - .replace(/\.(astro|ts|js)/, '') - .replace(/index$/, ''); - - /** - * @TODO - * figure out how to run `getStaticPaths` here. - */ - const isDynamicRoute = routeSegment.endsWith(']'); - const shouldIndex = !isDynamicRoute && !isPrerendered; - - return shouldIndex ? `${config.site}/${routeSegment}` : undefined; - }) - ) - ).filter((route): route is string => Boolean(route)); - const opts = validateOptions(config.site, options); - - opts.customPages = opts.customPages - ? Array.from(new Set([...routes, ...opts.customPages])) - : routes; - options = opts; - - logger.info(`build is starting + ${JSON.stringify(opts.customPages, null, 2)}`); + config = cfg; }, - 'astro:build:done': async ({ dir, pages }) => { + 'astro:build:done': async ({ dir, routes }) => { try { if (!config.site) { + logger.warn( + 'The Sitemap integration requires the `site` astro.config option. Skipping.' + ); return; } @@ -134,12 +85,22 @@ const createPlugin = (options?: SitemapOptions): AstroIntegration => { return; } - let pageUrls = pages.map((p) => { - if (p.pathname !== '' && !finalSiteUrl.pathname.endsWith('/')) - finalSiteUrl.pathname += '/'; - const path = finalSiteUrl.pathname + p.pathname; - return new URL(path, finalSiteUrl).href; - }); + let pageUrls = routes.reduce((urls, r) => { + /** + * Dynamic URLs have entries with `undefined` pathnames + */ + if (r.pathname) { + /** + * remove the initial slash from relative pathname + * because `finalSiteUrl` always has trailing slash + */ + const path = finalSiteUrl.pathname + r.pathname.substring(1); + const newUrl = new URL(path, finalSiteUrl).href; + urls.push(newUrl); + } + + return urls; + }, []); try { if (filter) { @@ -151,7 +112,7 @@ const createPlugin = (options?: SitemapOptions): AstroIntegration => { } if (customPages) { - pageUrls = [...pageUrls, ...customPages]; + pageUrls = Array.from(new Set([...pageUrls, ...customPages])); } if (pageUrls.length === 0) { diff --git a/packages/integrations/sitemap/src/utils/is-route-prerendered.ts b/packages/integrations/sitemap/src/utils/is-route-prerendered.ts deleted file mode 100644 index 468e09ba8489..000000000000 --- a/packages/integrations/sitemap/src/utils/is-route-prerendered.ts +++ /dev/null @@ -1,8 +0,0 @@ -const REGEX_PRERENDER = /const prerender = true/g; -import { readFile } from 'fs/promises'; - -export async function isRoutePrerendered(filePath: string) { - const contents = await readFile(filePath, 'utf-8'); - - return REGEX_PRERENDER.test(contents); -} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f7a037b45791..66af6e8c6136 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -4492,9 +4492,6 @@ importers: packages/integrations/sitemap: dependencies: - fast-glob: - specifier: ^3.2.11 - version: 3.2.12 sitemap: specifier: ^7.1.1 version: 7.1.1 From 1589a7b36e5e113c43e53f9c2ca3f8a410cb7a4c Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Tue, 14 Mar 2023 21:22:09 +0100 Subject: [PATCH 4/6] generate route to obey `trailingSlash` setting --- packages/integrations/sitemap/src/index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/integrations/sitemap/src/index.ts b/packages/integrations/sitemap/src/index.ts index 33e72fa23217..f6ddbe7b37a6 100644 --- a/packages/integrations/sitemap/src/index.ts +++ b/packages/integrations/sitemap/src/index.ts @@ -94,7 +94,7 @@ const createPlugin = (options?: SitemapOptions): AstroIntegration => { * remove the initial slash from relative pathname * because `finalSiteUrl` always has trailing slash */ - const path = finalSiteUrl.pathname + r.pathname.substring(1); + const path = finalSiteUrl.pathname + r.generate(r.pathname).substring(1); const newUrl = new URL(path, finalSiteUrl).href; urls.push(newUrl); } From 77b0e1d2b9d75777565920a2b6b0aabad8a48867 Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Wed, 15 Mar 2023 10:38:11 +0100 Subject: [PATCH 5/6] add logic to respect "directory" build format --- packages/integrations/sitemap/src/index.ts | 12 ++++++++++-- pnpm-lock.yaml | 9 ++------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/packages/integrations/sitemap/src/index.ts b/packages/integrations/sitemap/src/index.ts index f6ddbe7b37a6..0814ae0e1a3d 100644 --- a/packages/integrations/sitemap/src/index.ts +++ b/packages/integrations/sitemap/src/index.ts @@ -95,8 +95,16 @@ const createPlugin = (options?: SitemapOptions): AstroIntegration => { * because `finalSiteUrl` always has trailing slash */ const path = finalSiteUrl.pathname + r.generate(r.pathname).substring(1); - const newUrl = new URL(path, finalSiteUrl).href; - urls.push(newUrl); + + let newUrl = new URL(path, finalSiteUrl).href; + + if (config.trailingSlash === 'never') { + urls.push(newUrl); + } else if (config.build.format === 'directory' && !newUrl.endsWith('/')) { + urls.push(newUrl + '/'); + } else { + urls.push(newUrl); + } } return urls; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 66af6e8c6136..bf43c28474cd 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -668,7 +668,7 @@ importers: version: 3.0.1 typescript: specifier: '*' - version: 4.9.5 + version: 5.0.2 unist-util-visit: specifier: ^4.1.0 version: 4.1.0 @@ -15581,7 +15581,7 @@ packages: shiki: 0.10.1 shiki-twoslash: 3.1.0 tslib: 2.1.0 - typescript: 4.9.5 + typescript: 5.0.2 unist-util-visit: 2.0.3 transitivePeerDependencies: - supports-color @@ -16890,11 +16890,6 @@ packages: for-each: 0.3.3 is-typed-array: 1.1.10 - /typescript@4.9.5: - resolution: {integrity: sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g==} - engines: {node: '>=4.2.0'} - hasBin: true - /typescript@5.0.2: resolution: {integrity: sha512-wVORMBGO/FAs/++blGNeAVdbNKtIh1rbBL2EyQ1+J9lClJ93KiiKe8PmFIVdXhHcyv44SL9oglmfeSsndo0jRw==} engines: {node: '>=12.20'} From 4e00db6b79d58c3a4320b9ab004ac3167a22f4a7 Mon Sep 17 00:00:00 2001 From: Atila Fassina Date: Sun, 16 Apr 2023 21:08:00 +0200 Subject: [PATCH 6/6] integration(sitemap): add unit test for ssr support --- packages/integrations/sitemap/package.json | 1 + .../test/fixtures/ssr/astro.config.mjs | 12 ++++++++++ .../sitemap/test/fixtures/ssr/package.json | 9 ++++++++ .../test/fixtures/ssr/src/pages/one.astro | 8 +++++++ .../test/fixtures/ssr/src/pages/two.astro | 8 +++++++ .../integrations/sitemap/test/ssr.test.js | 22 +++++++++++++++++++ pnpm-lock.yaml | 12 ++++++++++ 7 files changed, 72 insertions(+) create mode 100644 packages/integrations/sitemap/test/fixtures/ssr/astro.config.mjs create mode 100644 packages/integrations/sitemap/test/fixtures/ssr/package.json create mode 100644 packages/integrations/sitemap/test/fixtures/ssr/src/pages/one.astro create mode 100644 packages/integrations/sitemap/test/fixtures/ssr/src/pages/two.astro create mode 100644 packages/integrations/sitemap/test/ssr.test.js diff --git a/packages/integrations/sitemap/package.json b/packages/integrations/sitemap/package.json index afd959a62c92..7c465494ee39 100644 --- a/packages/integrations/sitemap/package.json +++ b/packages/integrations/sitemap/package.json @@ -37,6 +37,7 @@ "zod": "^3.17.3" }, "devDependencies": { + "@astrojs/node": "workspace:*", "astro": "workspace:*", "astro-scripts": "workspace:*", "chai": "^4.3.6", diff --git a/packages/integrations/sitemap/test/fixtures/ssr/astro.config.mjs b/packages/integrations/sitemap/test/fixtures/ssr/astro.config.mjs new file mode 100644 index 000000000000..d914d4357e9f --- /dev/null +++ b/packages/integrations/sitemap/test/fixtures/ssr/astro.config.mjs @@ -0,0 +1,12 @@ +import { defineConfig } from 'astro/config'; +import sitemap from '@astrojs/sitemap'; +import nodeServer from '@astrojs/node' + +export default defineConfig({ + integrations: [sitemap()], + site: 'http://example.com', + output: 'server', + adapter: nodeServer({ + mode: "standalone" + }) +}) diff --git a/packages/integrations/sitemap/test/fixtures/ssr/package.json b/packages/integrations/sitemap/test/fixtures/ssr/package.json new file mode 100644 index 000000000000..980e02e73898 --- /dev/null +++ b/packages/integrations/sitemap/test/fixtures/ssr/package.json @@ -0,0 +1,9 @@ +{ + "name": "@test/sitemap-trailing-slash", + "version": "0.0.0", + "private": true, + "dependencies": { + "astro": "workspace:*", + "@astrojs/sitemap": "workspace:*" + } +} diff --git a/packages/integrations/sitemap/test/fixtures/ssr/src/pages/one.astro b/packages/integrations/sitemap/test/fixtures/ssr/src/pages/one.astro new file mode 100644 index 000000000000..0c7fb90a735e --- /dev/null +++ b/packages/integrations/sitemap/test/fixtures/ssr/src/pages/one.astro @@ -0,0 +1,8 @@ + + + One + + +

One

+ + diff --git a/packages/integrations/sitemap/test/fixtures/ssr/src/pages/two.astro b/packages/integrations/sitemap/test/fixtures/ssr/src/pages/two.astro new file mode 100644 index 000000000000..e7ba9910e2a6 --- /dev/null +++ b/packages/integrations/sitemap/test/fixtures/ssr/src/pages/two.astro @@ -0,0 +1,8 @@ + + + Two + + +

Two

+ + diff --git a/packages/integrations/sitemap/test/ssr.test.js b/packages/integrations/sitemap/test/ssr.test.js new file mode 100644 index 000000000000..e6f8412d56fc --- /dev/null +++ b/packages/integrations/sitemap/test/ssr.test.js @@ -0,0 +1,22 @@ +import { loadFixture, readXML } from './test-utils.js'; +import { expect } from 'chai'; + +describe('SSR support', () => { + /** @type {import('./test-utils.js').Fixture} */ + let fixture; + + before(async () => { + fixture = await loadFixture({ + root: './fixtures/ssr/', + }); + await fixture.build(); + }); + + it('SSR pages require zero config', async () => { + const data = await readXML(fixture.readFile('/client/sitemap-0.xml')); + const urls = data.urlset.url; + + expect(urls[0].loc[0]).to.equal('http://example.com/one/'); + expect(urls[1].loc[0]).to.equal('http://example.com/two/'); + }); +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index bf43c28474cd..757197e0159d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -4499,6 +4499,9 @@ importers: specifier: ^3.17.3 version: 3.20.6 devDependencies: + '@astrojs/node': + specifier: workspace:* + version: link:../node astro: specifier: workspace:* version: link:../../astro @@ -4515,6 +4518,15 @@ importers: specifier: 0.5.0 version: 0.5.0 + packages/integrations/sitemap/test/fixtures/ssr: + dependencies: + '@astrojs/sitemap': + specifier: workspace:* + version: link:../../.. + astro: + specifier: workspace:* + version: link:../../../../../astro + packages/integrations/sitemap/test/fixtures/trailing-slash: dependencies: '@astrojs/sitemap':