[Docs Site] Add llms.txt (#19015)

package.json

@@ -52,6 +52,7 @@
 		"astro-icon": "^1.1.2",
 		"astro-live-code": "^0.0.4",
 		"date-fns": "^4.1.0",
+		"dedent": "^1.5.3",
 		"detype": "1.0.12",
 		"dompurify": "3.2.3",
 		"dot-prop": "^9.0.0",

src/pages/llms.txt.ts

@@ -0,0 +1,62 @@
+import type { APIRoute } from "astro";
+import { getCollection } from "astro:content";
+import dedent from "dedent";
+
+export const GET: APIRoute = async () => {
+	const products = await getCollection("products", (p) => {
+		return p.data.product.group?.toLowerCase() === "developer platform";
+	});
+
+	const docs = await getCollection("docs", (e) => {
+		return products.some((p) =>
+			e.slug.startsWith(p.data.product.url.slice(1, -1)),
+		);
+	});
+
+	const grouped = Object.entries(
+		Object.groupBy(docs, (e) => {
+			const product = products.find((p) =>
+				e.slug.startsWith(p.data.product.url.slice(1, -1)),
+			);
+
+			if (!product) throw new Error(`Unable to find product for ${e.slug}`);
+
+			return product.data.product.title;
+		}),
+	);
+
+	const markdown = dedent(`
+		# Cloudflare Developer Documentation
+
+		Easily build and deploy full-stack applications everywhere,
+		thanks to integrated compute, storage, and networking.
+
+		${grouped
+			.map(([product, entries]) => {
+				return dedent(`
+				## ${product}
+
+				${entries
+					?.map((e) => {
+						const line = `- [${e.data.title}](https://developers.cloudflare.com/${e.slug}/)`;
+
+						const description = e.data.description;
+
+						if (description) {
+							return line.concat(`: ${description}`);
+						}
+
+						return line;
+					})
+					.join("\n")}
+			`);
+			})
+			.join("\n\n")}
+	`);
+
+	return new Response(markdown, {
+		headers: {
+			"content-type": "text/plain",
+		},
+	});
+};