mdn: download search-index.json

part of #372
website-local · Jul 25, 2021 · 8d36eeb · 8d36eeb
1 parent be3d886
commit 8d36eeb
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 3 deletions.
diff --git a/src/mdn/process-url/default-initial-url.ts b/src/mdn/process-url/default-initial-url.ts
@@ -21,7 +21,9 @@ export function defaultInitialUrl(locale: string): string[] {
     `https://developer.mozilla.org/${locale}/docs/Games`,
     `https://developer.mozilla.org/${locale}/docs/Glossary`,
     // https://github.com/website-local/mdn-local/issues/214
-    `https://developer.mozilla.org/sitemaps/${locale}/sitemap.xml.gz`
+    `https://developer.mozilla.org/sitemaps/${locale}/sitemap.xml.gz`,
+    // https://github.com/website-local/mdn-local/issues/372
+    `https://developer.mozilla.org/${locale}/search-index.json`,
   ];
   if (locale !== 'en-US') {
     strings.push('https://developer.mozilla.org/sitemaps/en-US/sitemap.xml.gz');

diff --git a/src/mdn/process-url/drop-resource.ts b/src/mdn/process-url/drop-resource.ts
@@ -1,5 +1,5 @@
-import {downloadableHosts, localeArr} from './consts';
-import type {Resource} from 'website-scrap-engine/lib/resource';
+import {downloadableHosts, localeArr, mdnHosts} from './consts';
+import {Resource} from 'website-scrap-engine/lib/resource';
 import type {StaticDownloadOptions} from 'website-scrap-engine/lib/options';
 import URI from 'urijs';
 import type {Cheerio} from 'website-scrap-engine/lib/types';
@@ -30,6 +30,10 @@ export function dropResource(
   if (host === 'mdn.mozillademos.org' && path.startsWith('/files')) {
     return res;
   }
+  // https://github.com/website-local/mdn-local/issues/372
+  if (mdnHosts[host] && path === `/${locale}/search-index.json`) {
+    return res;
+  }
   if (!downloadableHosts[host] ||
     testLocaleRegExp(locale).test(path) ||
     path.startsWith('/search') ||