feat: generate robots txt

README.md

@@ -36,7 +36,9 @@ import { createSitemapGenerator } from 'remix-sitemap';
 
 // Step 1. setup the generator
 const { isSitemapUrl, sitemap } = createSitemapGenerator({
-  siteUrl: 'https://example.com'
+  siteUrl: 'https://example.com',
+  generateRobotsTxt: true
+  // configure other things here
 })
 
 export default async function handleRequest(
@@ -68,6 +70,7 @@ Create a `remix-sitemap.config.js` file at the project root
 /** @type {import('remix-sitemap').Config} */
 module.exports = {
   siteUrl: 'https://example.com',
+  generateRobotsTxt: true
   // configure other things here
 }
 ```
@@ -87,17 +90,19 @@ For example if you are using `npm-run-all`
 ## Config
 This library is a little inspired in [next-sitemap](https://www.npmjs.com/package/next-sitemap) so the config is pretty much the same
 
-| Property                       | Description                                                                           |
-| ------------------------------ | ------------------------------------------------------------------------------------- |
-| siteUrl                        | Base url of your website                                                              |
-| changefreq (optional)          | Change frequency. Default `daily`                                                     |
-| priority (optional)            | Priority. Default `0.7`                                                               |
-| autoLastmod (optional)         | Add `<lastmod/>` property. Default `true`                                             |
-| sitemapBaseFileName (optional) | The name of the generated sitemap file before the file extension. Default `"sitemap"` |
-| optionalSegments (optional)    | possible values of optional segments                                                  |
-| alternateRefs (optional)       | multi language support by unique url. Default `[]`                                    |
-| outDir                         | The directory to create the sitemaps files. Default `"public"`                        |
-
+| Property                                       | Description                                                                           |
+| ---------------------------------------------- | ------------------------------------------------------------------------------------- |
+| siteUrl                                        | Base url of your website                                                              |
+| changefreq (optional)                          | Change frequency. Default `daily`                                                     |
+| priority (optional)                            | Priority. Default `0.7`                                                               |
+| autoLastmod (optional)                         | Add `<lastmod/>` property. Default `true`                                             |
+| sitemapBaseFileName (optional)                 | The name of the generated sitemap file before the file extension. Default `"sitemap"` |
+| optionalSegments (optional)                    | possible values of optional segments                                                  |
+| alternateRefs (optional)                       | multi language support by unique url. Default `[]`                                    |
+| outDir (optional)                              | The directory to create the sitemaps files. Default `"public"`                        |
+| generateRobotsTxt (optional)                   | Generate `robots.txt` file. Default `false`                                           |
+| robotsTxtOptions.policies (optional)           | Policies for generating `robots.txt`                                                  |
+| robotsTxtOptions.additionalSitemaps (optional) | Add additionals sitemaps to `robots.txt`                                              |
 
 
 ---

src/bin/index.ts

@@ -3,9 +3,10 @@ import path from 'path';
 import fs from 'fs';
 import { getSitemap } from '../sitemap';
 import { getRoutesAndModules } from './routes';
+import { getConfig } from '../lib/config';
+import { getRobots } from '../robots';
 
 import './polyfill';
-import { getConfig } from '../lib/config';
 
 const dir = path.resolve(process.cwd());
 
@@ -44,6 +45,16 @@ async function main() {
     request: {} as any
   });
 
+  if (config.generateRobotsTxt) {
+    const robots = getRobots(config);
+
+    if (robots) {
+      fs.writeFileSync(path.join(dir, config.outDir, 'robots.txt'), robots);
+
+      console.log('✨ Robots.txt generated successfully');
+    }
+  }
+
   fs.writeFileSync(
     path.join(dir, config.outDir, `${config.sitemapBaseFileName}.xml`),
     sitemap

src/index.ts

@@ -1,18 +1,28 @@
 import type { EntryContext } from '@remix-run/server-runtime';
 import type { Config } from './lib/types';
 import { sitemapResponse } from './sitemap';
-import { isSitemapUrl } from './utils/validations';
+import { isSitemapUrl, isRobotsUrl } from './utils/validations';
 import { getConfig } from './lib/config';
+import { robotsResponse } from './robots';
 
 export { SitemapHandle, RemixSitemapConfig, Config } from './lib/types';
 
 export const createSitemapGenerator = (config: Config) => {
   const defaultConfig = getConfig(config);
 
   return {
-    sitemap: (request: Request, context: EntryContext) =>
-      sitemapResponse(defaultConfig, request, context),
+    sitemap: (request: Request, context: EntryContext) => {
+      if (isSitemapUrl(defaultConfig, request)) {
+        return sitemapResponse(defaultConfig, request, context);
+      }
 
-    isSitemapUrl: (request: Request) => isSitemapUrl(defaultConfig, request)
+      if (defaultConfig.generateRobotsTxt && isRobotsUrl(request)) {
+        return robotsResponse(defaultConfig);
+      }
+    },
+
+    isSitemapUrl: (request: Request) =>
+      isSitemapUrl(defaultConfig, request) ||
+      (defaultConfig.generateRobotsTxt && isRobotsUrl(request))
   };
 };

src/lib/config.ts

@@ -7,6 +7,15 @@ export const getConfig = (config: Config) => {
     changefreq: config.changefreq ?? 'daily',
     priority: config.priority ?? 0.7,
     sitemapBaseFileName: config.sitemapBaseFileName ?? 'sitemap',
-    outDir: config.outDir ?? 'public'
+    outDir: config.outDir ?? 'public',
+    robotsTxtOptions: {
+      ...(config.robotsTxtOptions || {}),
+      policies: config.robotsTxtOptions?.policies ?? [
+        {
+          userAgent: '*',
+          allow: '/'
+        }
+      ]
+    }
   };
 };

src/lib/types.ts

@@ -44,6 +44,10 @@ export interface RemixSitemapConfig {
 
   outDir?: string;
 
+  generateRobotsTxt?: boolean;
+
+  robotsTxtOptions?: RobotsTxtOptions;
+
   /**
    * Headers to be added to the sitemap response.
    */
@@ -123,3 +127,14 @@ export interface Handle {
   generateEntries?(request: Request): Promise<SitemapEntry[]>;
   addOptionalSegments?: boolean;
 }
+
+export interface RobotsTxtOptions {
+  policies?: Policy[];
+  additionalSitemaps?: string[];
+}
+
+export type Policy = {
+  allow?: string | string[];
+  disallow?: string | string[];
+  userAgent: string;
+};

src/robots.ts

@@ -0,0 +1,70 @@
+import type { Policy, Config } from './lib/types';
+
+function getPolicy(policy: Policy) {
+  const { allow, disallow, userAgent } = policy;
+
+  let policyStr = `# ${userAgent}\nUser-agent: ${userAgent}`;
+
+  if (allow) {
+    const allowStr = Array.isArray(allow) ? allow.join('\nAllow: ') : allow;
+
+    policyStr += `\nAllow: ${allowStr}`;
+  }
+
+  if (disallow) {
+    const disallowStr = Array.isArray(disallow)
+      ? disallow.join('\nDisallow: ')
+      : disallow;
+
+    policyStr += `\nDisallow: ${disallowStr}`;
+  }
+
+  return policyStr;
+}
+
+function getSitemap(sitemap: string) {
+  return `Sitemap: ${sitemap}`;
+}
+
+export function getRobots(config: Config) {
+  const options = config.robotsTxtOptions;
+
+  let str = '';
+
+  const policies = options?.policies?.map(getPolicy);
+
+  if (policies?.length) {
+    str += policies.join('\n\n');
+
+    str += '\n\n';
+  }
+
+  str += `# Host\nHost: ${config.siteUrl}\n\n`;
+
+  const sitemaps = [`${config.siteUrl}/${config.sitemapBaseFileName}.xml`]
+    .concat(options?.additionalSitemaps || [])
+    .map(getSitemap);
+
+  if (sitemaps?.length) {
+    str += '# Sitemaps\n';
+    str += sitemaps.join('\n');
+  }
+
+  return str;
+}
+
+export function robotsResponse(config: Config) {
+  const robots = getRobots(config);
+
+  if (!robots) return;
+
+  const bytes = new TextEncoder().encode(robots).byteLength;
+
+  return new Response(robots, {
+    headers: {
+      ...(config.headers || {}),
+      'Content-Type': 'text/plain',
+      'Content-Length': bytes.toString()
+    }
+  });
+}

src/utils/validations.ts

@@ -12,6 +12,12 @@ export function isSitemapUrl(config: RemixSitemapConfig, request: Request) {
 export const isDynamicPath = (path?: string) =>
   path?.includes(':') || path?.includes('*');
 
+export const isRobotsUrl = (request: Request) => {
+  const url = new URL(request.url);
+
+  return isEqual(url.pathname, `/robots.txt`);
+};
+
 export function isValidEntry(route: string, context: EntryContext) {
   const { manifest, handle, path, module } = getRouteData(route, context);