chore: add sitemap and robots.txt

includes:
- docs-site sitemap, generated at build time
- root site sitemap, generated at request time via public topics
- sitemap index, pointing to these
- robots.txt pointing to the sitemap index

docs-site/next-sitemap.config.js

@@ -0,0 +1,9 @@
+/** @type {import('next-sitemap').IConfig} */
+// eslint-disable-next-line functional/immutable-data
+module.exports = {
+  siteUrl: "https://ameliorate.app/docs",
+  generateIndexSitemap: false, // docs sitemap is going to be pointed at by root site's sitemap index; also we shouldn't ever have more than 50k docs pages
+  changefreq: "daily", // docs don't seem like they need to be indexed more than once a day
+  priority: 0.5, // match the default of the official sitemaps protocol https://www.sitemaps.org/protocol.html
+  autoLastmod: false, // if true, it sets the lastmod for all pages to the time of each build, will be almost always incorrect, because docs don't change nearly as often as the project is deployed
+};

docs-site/package.json

@@ -6,12 +6,14 @@
     "dev": "next -p 3008",
     "//build": "delete .next to prevent netlify from caching and not updating sidebar when page titles change - thanks https://github.com/shuding/nextra/issues/1564#issuecomment-1688460307",
     "build": "npm run clean && next build",
+    "postbuild": "next-sitemap",
     "clean": "rm -rf .next",
     "start": "next start -p 3008"
   },
   "dependencies": {
     "next": "^14.1.1",
     "next-plausible": "^3.12.2",
+    "next-sitemap": "^4.2.3",
     "nextra": "^2.13.3",
     "nextra-theme-docs": "^2.13.3",
     "react": "^18.2.0",

docs-site/public/sitemap.xml

@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:news="http://www.google.com/schemas/sitemap-news/0.9" xmlns:xhtml="http://www.w3.org/1999/xhtml" xmlns:mobile="http://www.google.com/schemas/sitemap-mobile/1.0" xmlns:image="http://www.google.com/schemas/sitemap-image/1.1" xmlns:video="http://www.google.com/schemas/sitemap-video/1.1">
+<url><loc>https://ameliorate.app/docs</loc><changefreq>daily</changefreq><priority>0.5</priority></url>
+<url><loc>https://ameliorate.app/docs/comparisons</loc><changefreq>daily</changefreq><priority>0.5</priority></url>
+<url><loc>https://ameliorate.app/docs/diagramming-choices</loc><changefreq>daily</changefreq><priority>0.5</priority></url>
+<url><loc>https://ameliorate.app/docs/discourse-sessions</loc><changefreq>daily</changefreq><priority>0.5</priority></url>
+<url><loc>https://ameliorate.app/docs/faq</loc><changefreq>daily</changefreq><priority>0.5</priority></url>
+<url><loc>https://ameliorate.app/docs/features</loc><changefreq>daily</changefreq><priority>0.5</priority></url>
+<url><loc>https://ameliorate.app/docs/features/indicators</loc><changefreq>daily</changefreq><priority>0.5</priority></url>
+<url><loc>https://ameliorate.app/docs/features/more-actions</loc><changefreq>daily</changefreq><priority>0.5</priority></url>
+<url><loc>https://ameliorate.app/docs/features/views-and-filters</loc><changefreq>daily</changefreq><priority>0.5</priority></url>
+<url><loc>https://ameliorate.app/docs/getting-started</loc><changefreq>daily</changefreq><priority>0.5</priority></url>
+<url><loc>https://ameliorate.app/docs/getting-started/core-ideas</loc><changefreq>daily</changefreq><priority>0.5</priority></url>
+<url><loc>https://ameliorate.app/docs/getting-started/diagramming-basics</loc><changefreq>daily</changefreq><priority>0.5</priority></url>
+<url><loc>https://ameliorate.app/docs/getting-started/your-first-topic</loc><changefreq>daily</changefreq><priority>0.5</priority></url>
+<url><loc>https://ameliorate.app/docs/release-status</loc><changefreq>daily</changefreq><priority>0.5</priority></url>
+</urlset>

public/robots.txt

@@ -0,0 +1,4 @@
+User-Agent: *
+Allow: /
+
+Sitemap: https://ameliorate.app/sitemap.xml

public/sitemap.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<sitemapindex xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+  <sitemap>
+    <loc>https://ameliorate.app/sitemap-generated.xml</loc>
+  </sitemap>
+  <sitemap>
+    <loc>https://ameliorate.app/docs/sitemap.xml</loc>
+  </sitemap>
+</sitemapindex>

src/api/topic/topic.ts

@@ -0,0 +1,9 @@
+import { xprisma } from "@/db/extendedPrisma";
+
+export const getPublicTopics = async () => {
+  return await xprisma.topic.findMany({
+    where: {
+      visibility: "public",
+    },
+  });
+};

src/common/comment.ts

@@ -59,7 +59,9 @@ export const userCanDeleteComment = (
 };
 
 export const getLinkToComment = (comment: Comment, commentTopic: Topic) => {
+  // duplicate of getLinkToTopic because we want to add the comment param before returning the `href`
   const sourceUrl = new URL(`/${commentTopic.creatorName}/${commentTopic.title}`, getBaseUrl());
+
   sourceUrl.searchParams.set("comment", comment.id);
 
   return sourceUrl.href;

src/common/topic.ts

@@ -2,6 +2,7 @@ import { z } from "zod";
 
 import { reservedSecondLevelEndpointNames } from "@/common/reservedEndpointNames";
 import { userSchema } from "@/common/user";
+import { getBaseUrl } from "@/common/utils";
 
 // not sure how to guarantee that this matches the schema enum
 export const visibilityTypes = ["private", "unlisted", "public"] as const;
@@ -31,3 +32,9 @@ export const topicSchema = z.object({
 });
 
 export type Topic = z.infer<typeof topicSchema>;
+
+export const getLinkToTopic = (topic: Topic) => {
+  const sourceUrl = new URL(`/${topic.creatorName}/${topic.title}`, getBaseUrl());
+
+  return sourceUrl.href;
+};

src/pages/sitemap-generated.xml.page.ts

@@ -0,0 +1,53 @@
+/**
+ * Copied from example at https://nextjs.org/learn-pages-router/seo/crawling-and-indexing/xml-sitemaps
+ */
+
+import { GetServerSidePropsContext } from "next";
+
+import { getPublicTopics } from "@/api/topic/topic";
+import { getLinkToTopic } from "@/common/topic";
+import { getBaseUrl } from "@/common/utils";
+
+const getUrlXml = (loc: string, priority?: number, lastmod?: Date) => {
+  const priorityTag = priority ? `<priority>${priority}</priority>` : "";
+  const lastModTag = lastmod ? `<lastmod>${lastmod.toISOString()}</lastmod>` : "";
+
+  return `<url><loc>${loc}</loc><changefreq>daily</changefreq>${priorityTag}${lastModTag}</url>`;
+};
+
+const generateSiteMap = (publicTopics: Awaited<ReturnType<typeof getPublicTopics>>) => {
+  if (publicTopics.length > 49000)
+    throw new Error("Sitemap close to exceeding 50k URL limit, need to split up");
+
+  const baseUrl = getBaseUrl();
+
+  return `<?xml version="1.0" encoding="UTF-8"?>
+<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
+${getUrlXml(baseUrl, 1)}
+${getUrlXml(baseUrl + "/playground")}
+${publicTopics
+  .map((topic) => ({ loc: getLinkToTopic(topic), lastmod: topic.updatedAt }))
+  .toSorted((a, b) => (a.loc < b.loc ? -1 : 1)) // nicer to read if these are sorted
+  .map(({ loc, lastmod }) => getUrlXml(loc, undefined, lastmod))
+  .join("\n")}
+</urlset>`;
+};
+
+const SiteMap = () => {
+  // getServerSideProps will do the heavy lifting
+};
+
+export const getServerSideProps = async ({ res }: GetServerSidePropsContext) => {
+  const publicTopics = await getPublicTopics();
+  const sitemap = generateSiteMap(publicTopics);
+
+  res.setHeader("Content-Type", "text/xml");
+  res.write(sitemap);
+  res.end();
+
+  return {
+    props: {},
+  };
+};
+
+export default SiteMap;