feat: llms.txt (sveltejs#979)

* wip

* Update +server.ts

* Update +server.ts

* wip

* wip

* Update +server.ts

* Update +server.ts

* wip

* cleanup

* wip

* refactor

* cleanupo

* Update +server.ts

* Update content.ts

* wip

* Create +server.ts

* minimize llms.txt

* Filter llms.txt

* clean up

* package

* chore: naming

* Update +server.ts

* Dynamic path names

* clean up

* fix

* under_score

* code style

* use real document titles, filter out empty files

* move llms.txt to llms-small.txt

* llms.txt index

* Update +server.ts

* Update +server.ts

* Fix index

* fix

* revert VERCEL_URL usage

* Update apps/svelte.dev/src/lib/server/content.ts

Co-authored-by: Rich Harris <[email protected]>

* move llm stuff into its own module

* revert whitespace changes

* snake_case

* tweak

* snake_case etc

* make ignores work

* simplify

* unused

* reduce indirection

* more

* move template into separate .md file

* add a section to /docs

* advent of svelte

---------

Co-authored-by: Simon Holthausen <[email protected]>
Co-authored-by: Rich Harris <[email protected]>

apps/svelte.dev/content/blog/2024-12-01-advent-of-svelte.md

@@ -99,9 +99,13 @@ As of today, you can also return things that _aren't_ built in to the language,
 - [docs](/docs/kit/hooks#Universal-hooks-transport)
 - [demo](https://stackblitz.com/edit/sveltejs-kit-template-default-b5zbxomg?file=src%2Fhooks.js)
 
-## Day 13
+## Day 13: rise of the robots
 
-Coming soon!
+For those of you using LLMs to help you write code — via Cursor or Copilot or Claude or Bolt or v0 or some other interface — we now publish the documentation in a selection of robot-friendly `llms.txt` files. This is experimental and will evolve over time, but by way of example here's a [snake game](/playground/0de3c1c1a31d47bdbb7c4aa3477a6b46) built by Sonnet 3.5 with no additional prompting.
+
+Thanks to [Didier Catz](https://x.com/didiercatz) and [Stanislav Khromov](https://bsky.app/profile/khromov.se) for building this!
+
+- [docs](/docs/llms)
 
 ## Day 14
 

apps/svelte.dev/package.json

@@ -67,6 +67,7 @@
 		"lightningcss": "^1.25.1",
 		"magic-string": "^0.30.11",
 		"marked": "^14.1.2",
+		"minimatch": "^10.0.1",
 		"prettier": "^3.3.2",
 		"prettier-plugin-svelte": "^3.2.4",
 		"satori": "^0.10.13",

apps/svelte.dev/src/lib/server/llms.ts

@@ -0,0 +1,134 @@
+import { minimatch } from 'minimatch';
+import { dev } from '$app/environment';
+import { index } from './content';
+
+interface GenerateLlmContentOptions {
+	ignore?: string[];
+	minimize?: Partial<MinimizeOptions>;
+	sections: Section[];
+}
+
+interface MinimizeOptions {
+	remove_legacy: boolean;
+	remove_note_blocks: boolean;
+	remove_details_blocks: boolean;
+	remove_playground_links: boolean;
+	remove_prettier_ignore: boolean;
+	normalize_whitespace: boolean;
+}
+
+interface Section {
+	slug: string;
+	title: string;
+}
+
+const defaults: MinimizeOptions = {
+	remove_legacy: false,
+	remove_note_blocks: false,
+	remove_details_blocks: false,
+	remove_playground_links: false,
+	remove_prettier_ignore: false,
+	normalize_whitespace: false
+};
+
+export function generate_llm_content(options: GenerateLlmContentOptions): string {
+	let content = '';
+
+	for (const section of options.sections) {
+		if (options.sections.length > 1) {
+			content += `# Start of ${section.title} documentation\n\n`;
+		}
+
+		for (const [path, document] of Object.entries(index)) {
+			if (!path.startsWith(`docs/${section.slug}`)) continue;
+
+			if (options.ignore?.some((pattern) => minimatch(path, pattern))) {
+				if (dev) console.log(`❌ Ignored by pattern: ${path}`);
+				continue;
+			}
+
+			const doc_content = options.minimize
+				? minimize_content(document.body, options.minimize)
+				: document.body;
+			if (doc_content.trim() === '') continue;
+
+			content += `\n# ${document.metadata.title}\n\n`;
+			content += doc_content;
+			content += '\n';
+		}
+	}
+
+	return content;
+}
+
+export const sections: Section[] = [
+	{ slug: 'svelte', title: 'Svelte' },
+	{ slug: 'kit', title: 'SvelteKit' },
+	{ slug: 'cli', title: 'the Svelte CLI' }
+];
+
+export function get_documentation_title(section: Section): string {
+	return `This is the developer documentation for ${section.title}.`;
+}
+
+function minimize_content(content: string, options?: Partial<MinimizeOptions>): string {
+	// Merge with defaults, but only for properties that are defined
+	const settings: MinimizeOptions = { ...defaults, ...options };
+
+	let minimized = content;
+
+	if (settings.remove_legacy) {
+		minimized = remove_quote_blocks(minimized, 'LEGACY');
+	}
+
+	if (settings.remove_note_blocks) {
+		minimized = remove_quote_blocks(minimized, 'NOTE');
+	}
+
+	if (settings.remove_details_blocks) {
+		minimized = remove_quote_blocks(minimized, 'DETAILS');
+	}
+
+	if (settings.remove_playground_links) {
+		// Replace playground URLs with /[link] but keep the original link text
+		minimized = minimized.replace(/\[([^\]]+)\]\(\/playground[^)]+\)/g, '[$1](/REMOVED)');
+	}
+
+	if (settings.remove_prettier_ignore) {
+		minimized = minimized
+			.split('\n')
+			.filter((line) => line.trim() !== '<!-- prettier-ignore -->')
+			.join('\n');
+	}
+
+	if (settings.normalize_whitespace) {
+		minimized = minimized.replace(/\s+/g, ' ');
+	}
+
+	minimized = minimized.trim();
+
+	return minimized;
+}
+
+function remove_quote_blocks(content: string, blockType: string): string {
+	return content
+		.split('\n')
+		.reduce((acc: string[], line: string, index: number, lines: string[]) => {
+			// If we find a block (with or without additional text), skip it and all subsequent blockquote lines
+			if (line.trim().startsWith(`> [!${blockType}]`)) {
+				// Skip all subsequent lines that are part of the blockquote
+				let i = index;
+				while (i < lines.length && (lines[i].startsWith('>') || lines[i].trim() === '')) {
+					i++;
+				}
+				// Update the index to skip all these lines
+				index = i - 1;
+				return acc;
+			}
+
+			// Only add the line if it's not being skipped
+			acc.push(line);
+			return acc;
+		}, [])
+		.join('\n');
+}

apps/svelte.dev/src/routes/docs/+page.svelte

@@ -90,6 +90,14 @@
 			</p>
 		</a>
 
+		<a href="/docs/llms">
+			<h2>I'm a Large Language Model (LLM)</h2>
+			<p>
+				If you're an artificial intelligence, or trying to teach one how to use Svelte, we offer the
+				documentation in plaintext format. Beep boop.
+			</p>
+		</a>
+
 		<a href="/chat" class="external">
 			<h2>Help! I'm stuck</h2>
 			<p>

apps/svelte.dev/src/routes/docs/[...path]/llms.txt/+server.ts

@@ -0,0 +1,29 @@
+import { error } from '@sveltejs/kit';
+import { generate_llm_content, get_documentation_title, sections } from '$lib/server/llms';
+
+export const prerender = true;
+
+export function entries() {
+	return sections.map((section) => ({ path: section.slug }));
+}
+
+export function GET({ params }) {
+	const pkg = params.path;
+
+	const section = sections.find((s) => s.slug === pkg);
+
+	if (!section) {
+		error(404, 'Not Found');
+	}
+
+	const prefix = `<SYSTEM>${get_documentation_title(section)}</SYSTEM>`;
+	const content = `${prefix}\n\n${generate_llm_content({ sections: [section] })}`;
+
+	return new Response(content, {
+		status: 200,
+		headers: {
+			'Content-Type': 'text/plain; charset=utf-8',
+			'Cache-Control': 'public, max-age=3600'
+		}
+	});
+}

apps/svelte.dev/src/routes/docs/llms/+page.svelte

@@ -0,0 +1,46 @@
+<script>
+	import { Text } from '@sveltejs/site-kit/components';
+</script>
+
+<div class="page">
+	<Text>
+		<h1>Docs for LLMs</h1>
+
+		<p>
+			We support the <a href="https://llmstxt.org/">llms.txt</a> convention for making documentation
+			available to large language models and the applications that make use of them.
+		</p>
+
+		<p>Currently, we have the following root-level files...</p>
+
+		<ul>
+			<li><a href="/llms.txt">/llms.txt</a> — a listing of the available files</li>
+			<li>
+				<a href="/llms-full.txt">/llms-full.txt</a> — complete documentation for Svelte, SvelteKit and
+				the CLI
+			</li>
+			<li>
+				<a href="/llms-small.txt">/llms-small.txt</a> — compressed documentation for use with smaller
+				context windows
+			</li>
+		</ul>
+
+		<p>...and package-level documentation:</p>
+
+		<ul>
+			<li><a href="/docs/svelte/llms.txt">/docs/svelte/llms.txt</a></li>
+			<li><a href="/docs/kit/llms.txt">/docs/kit/llms.txt</a></li>
+			<li><a href="/docs/cli/llms.txt">/docs/cli/llms.txt</a></li>
+		</ul>
+	</Text>
+</div>
+
+<style>
+	.page {
+		padding: var(--sk-page-padding-top) var(--sk-page-padding-side);
+		max-width: var(--sk-page-content-width);
+		box-sizing: content-box;
+		margin: auto;
+		text-wrap: balance;
+	}
+</style>

apps/svelte.dev/src/routes/llms-full.txt/+server.ts

@@ -0,0 +1,15 @@
+import { generate_llm_content, sections } from '$lib/server/llms';
+
+export const prerender = true;
+
+export function GET() {
+	const content = `<SYSTEM>This is the full developer documentation for Svelte and SvelteKit.</SYSTEM>\n\n${generate_llm_content({ sections })}`;
+
+	return new Response(content, {
+		status: 200,
+		headers: {
+			'Content-Type': 'text/plain; charset=utf-8',
+			'Cache-Control': 'public, max-age=3600'
+		}
+	});
+}

apps/svelte.dev/src/routes/llms-small.txt/+server.ts

@@ -0,0 +1,47 @@
+import { generate_llm_content, sections } from '$lib/server/llms';
+
+export function GET() {
+	const main_content = generate_llm_content({
+		sections,
+		ignore: [
+			// Svelte ignores
+			'docs/svelte/legacy/**/*',
+			'docs/svelte/misc/custom-elements',
+			'docs/svelte/misc/v4-migration-guide',
+			'docs/svelte/misc/v5-migration-guide',
+			'docs/svelte/misc/faq',
+			'docs/svelte/reference/compiler-errors',
+			'docs/svelte/reference/compiler-warnings',
+			'docs/svelte/reference/runtime-errors',
+			'docs/svelte/reference/runtime-warnings',
+			'docs/svelte/reference/svelte-legacy',
+			'**/xx-*',
+
+			// SvelteKit ignores
+			'docs/kit/advanced/packaging',
+			'docs/kit/appendix/**/*',
+			'docs/kit/best-practices/performance',
+			'docs/kit/build-and-deploy/*adapter-*',
+			'docs/kit/build-and-deploy/writing-adapters'
+		],
+		minimize: {
+			remove_legacy: true,
+			remove_note_blocks: true,
+			remove_details_blocks: true,
+			remove_playground_links: true,
+			remove_prettier_ignore: true,
+			normalize_whitespace: true
+		}
+	});
+	const content = `<SYSTEM>This is the abridged developer documentation for Svelte and SvelteKit.</SYSTEM>\n\n${main_content}`;
+
+	return new Response(content, {
+		status: 200,
+		headers: {
+			'Content-Type': 'text/plain; charset=utf-8',
+			'Cache-Control': 'public, max-age=3600'
+		}
+	});
+}
+
+export const prerender = true;

apps/svelte.dev/src/routes/llms.txt/+server.ts

@@ -0,0 +1,22 @@
+import { get_documentation_title, sections } from '$lib/server/llms';
+import template from './template.md?raw';
+
+const DOMAIN = `https://svelte.dev`;
+
+export const prerender = true;
+
+export function GET() {
+	const package_docs = sections.map(
+		(section) =>
+			`- [${section.title} documentation](${DOMAIN}/docs/${section.slug}/llms.txt): ${get_documentation_title(section)}`
+	);
+
+	const content = template.replace('%PACKAGE_DOCS%', package_docs.join('\n'));
+
+	return new Response(content, {
+		headers: {
+			'Content-Type': 'text/plain; charset=utf-8',
+			'Cache-Control': 'public, max-age=3600'
+		}
+	});
+}

apps/svelte.dev/src/routes/llms.txt/template.md

@@ -0,0 +1,19 @@
+# Svelte Documentation for LLMs
+
+> Svelte is a UI framework that uses a compiler to let you write breathtakingly concise components that do minimal work in the browser, using languages you already know — HTML, CSS and JavaScript.
+
+## Documentation Sets
+
+- [Abridged documentation](https://svelte.dev/llms-small.txt): A minimal version of the Svelte and SvelteKit documentation, with examples and non-essential content removed
+- [Complete documentation](https://svelte.dev/llms-full.txt): The complete Svelte and SvelteKit documentation including all examples and additional content
+
+## Individual Package Documentation
+
+%PACKAGE_DOCS%
+
+## Notes
+
+- The abridged documentation excludes legacy compatibility notes, detailed examples, and supplementary information
+- The complete documentation includes all content from the official documentation
+- Package-specific documentation files contain only the content relevant to that package
+- The content is automatically generated from the same source as the official documentation