Generate non-chapter pages in Sitemap based on JSON config (#1027)

* Generate non-chapter pages based on JSON config * Regenerate to include Contributors
HTTPArchive · Jul 14, 2020 · d8aa64a · d8aa64a
1 parent 167b13a
commit d8aa64a
Show file tree

Hide file tree

Showing 3 changed files with 54 additions and 10 deletions.
diff --git a/src/templates/sitemap.xml b/src/templates/sitemap.xml
@@ -121,6 +121,16 @@
         <lastmod>2020-03-02</lastmod>
     </url>
 
+    <url>
+        <loc>https://almanac.httparchive.org/en/2020/</loc>
+        <lastmod>2020-07-11</lastmod>
+    </url>
+
+    <url>
+        <loc>https://almanac.httparchive.org/en/2020/contributors</loc>
+        <lastmod>2020-07-14</lastmod>
+    </url>
+
     <url>
         <loc>https://almanac.httparchive.org/en/accessibility-statement</loc>
         <lastmod>2020-06-21</lastmod>
@@ -166,6 +176,16 @@
         <lastmod>2020-06-21</lastmod>
     </url>
 
+    <url>
+        <loc>https://almanac.httparchive.org/es/2020/</loc>
+        <lastmod>2020-07-11</lastmod>
+    </url>
+
+    <url>
+        <loc>https://almanac.httparchive.org/es/2020/contributors</loc>
+        <lastmod>2020-07-14</lastmod>
+    </url>
+
     <url>
         <loc>https://almanac.httparchive.org/fr/2019/</loc>
         <lastmod>2020-06-21</lastmod>
@@ -231,6 +251,16 @@
         <lastmod>2020-05-05</lastmod>
     </url>
 
+    <url>
+        <loc>https://almanac.httparchive.org/fr/2020/</loc>
+        <lastmod>2020-07-11</lastmod>
+    </url>
+
+    <url>
+        <loc>https://almanac.httparchive.org/fr/2020/contributors</loc>
+        <lastmod>2020-07-14</lastmod>
+    </url>
+
     <url>
         <loc>https://almanac.httparchive.org/ja/2019/</loc>
         <lastmod>2020-06-21</lastmod>
@@ -351,6 +381,16 @@
         <lastmod>2020-05-14</lastmod>
     </url>
 
+    <url>
+        <loc>https://almanac.httparchive.org/ja/2020/</loc>
+        <lastmod>2020-07-11</lastmod>
+    </url>
+
+    <url>
+        <loc>https://almanac.httparchive.org/ja/2020/contributors</loc>
+        <lastmod>2020-07-14</lastmod>
+    </url>
+
     <url>
         <loc>https://almanac.httparchive.org/ja/accessibility-statement</loc>
         <lastmod>2020-06-21</lastmod>

diff --git a/src/tools/generate/generate_chapters.js b/src/tools/generate/generate_chapters.js
@@ -64,7 +64,7 @@ const generate_chapters = async () => {
   await generate_ebooks(ebook_chapters,configs);
   await generate_js();
 
-  const sitemap_path = await generate_sitemap(sitemap);
+  const sitemap_path = await generate_sitemap(sitemap,sitemap_languages);
   await size_of(sitemap_path);
 };
 

diff --git a/src/tools/generate/generate_sitemap.js b/src/tools/generate/generate_sitemap.js
@@ -13,9 +13,9 @@ const static_pages = [
 ];
 const ebook_path = "static/pdfs/web_almanac_";
 
-const generate_sitemap = async (sitemap_chapters) => {
+const generate_sitemap = async (sitemap_chapters,sitemap_languages) => {
 
-  const urls = await get_static_pages(sitemap_chapters);
+  const urls = await get_static_pages(sitemap_languages);
 
   for (let sitemap_chapter of sitemap_chapters) {
     let { language, year, chapter, metadata } = sitemap_chapter;
@@ -39,10 +39,15 @@ const generate_sitemap = async (sitemap_chapters) => {
   return sitemap_path;
 };
 
-const get_static_pages = async (sitemap_chapters) => {
+const get_static_pages = async (sitemap_languages) => {
 
-  // Get distinct languages and years
-  const languages_and_years = [...new Set(sitemap_chapters.map((x) => `${x.language}/${x.year}`))];
+  var languages_and_years = [];
+
+  for (const year in sitemap_languages) {
+    for (const languages in sitemap_languages[year]) {
+      languages_and_years.push(`${sitemap_languages[year][languages]}/${year}`);
+    }
+  }
 
   // Get all of the static pages for each combination
   const files = languages_and_years
@@ -64,10 +69,9 @@ const get_static_pages = async (sitemap_chapters) => {
   }
 
   // For ebooks find out if the PDF exists, get lastmod from template
-  const years = [...new Set(sitemap_chapters.map((x) => `${x.year}`))];
-  const languages = [...new Set(sitemap_chapters.map((x) => `${x.language}`))];
-  for (const year of years) {
-    for (const language of languages) {
+  for (const year in sitemap_languages) {
+    for (const languages in sitemap_languages[year]) {
+      const language = sitemap_languages[year][languages];
       const ebook_pdf = ebook_path + year + '_' + language + '.pdf';
       const ebook_html = 'templates/' + language + '/' + year + '/ebook.html';
       if (fs.existsSync(ebook_pdf)) {