Simplify rewrite

.eslintrc.json

@@ -10,13 +10,7 @@
     "no-underscore-dangle": [2, {
       "allow": ["__content"]
     }],
-    "comma-dangle": ["error", {
-      "arrays": "always-multiline",
-      "objects": "always-multiline",
-      "imports": "always",
-      "exports": "always",
-      "functions": "never"
-    }]
+    "linebreak-style": 0
   },
   "overrides": [{
     "files": ["test/**/*.js"],

.travis.yml

@@ -1,15 +1,21 @@
 language: node_js
+node_js:
+  - "10"
 
 branches:
   only:
     - master
+    - refactor/smaller-broccoli-plugins
+
+jobs:
+  fail_fast: true
 
-matrix:
   include:
-    - node_js: 8
-      os: linux
-    - node_js: 10
-      os: linux
+    - stage: "Tests"
+      name: "Tests"
+      script:
+        - npm run lint:js
+        - npm test
       before_script:
         - curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./cc-test-reporter
         - chmod +x ./cc-test-reporter
@@ -18,23 +24,24 @@ matrix:
       after_script:
         - npm run coverage
         - ./cc-test-reporter after-build --exit-code $TRAVIS_TEST_RESULT
+    - stage: "All OS and Node versions"
+      node_js: 8
+      os: linux
     - node_js: 8
       os: windows
-      script: npm test
+    - node_js: 10
+      os: linux
     - node_js: 10
       os: windows
-      script: npm test
-
+    - node_js: 12
+      os: linux
+    - node_js: 12
+      os: windows
 
 env:
   global:
     - CC_TEST_REPORTER_ID=e86c6de11dd97ad1777b3212cddc0367b150816025083150839f687b7fbce7f9
 
-before_install:
-  - npm config --global set spin false
-  # if npm version is less than 5, upgrade to 6
-  - if [[ $(npm -v | cut -d '.' -f 1) -lt 5 ]]; then npm i -g npm@6; fi
-
 script:
   - npm run lint:js
   - npm test

index.js

@@ -2,17 +2,26 @@ const BroccoliMergeTrees = require('broccoli-merge-trees');
 const BroccoliFunnel = require('broccoli-funnel');
 const { mv } = require('broccoli-stew');
 const TableOfContents = require('./lib/table-of-contents');
-const MarkdownToJsonApi = require('./lib/markdown-to-json');
-const CollateJsonApiBlobs = require('./lib/serialize-json-blobs');
+const CollateJsonApiBlobs = require('./lib/collate-and-paginate');
+const MarkdownToJsonApi = require('./lib/markdown-to-jsonapi');
 
 
 module.exports = function StaticSiteJson(folder, options = {}) {
   const cleanMarkdownFunnel = new BroccoliFunnel(folder, {
     include: ['**/*.md', '**/*.markdown'],
   });
-  const pagesTree = new TableOfContents([folder], options);
-  const jsonTree = new MarkdownToJsonApi(cleanMarkdownFunnel, options);
-  const jsonApiTree = new CollateJsonApiBlobs(jsonTree, options);
-  const compiledTrees = new BroccoliMergeTrees([jsonApiTree, pagesTree]);
+  const tocFunnel = new BroccoliFunnel(folder, {
+    include: ['**/pages.yml', '**/pages.json'],
+  });
+  const pagesTree = new TableOfContents(tocFunnel, options);
+  const jsonApiTree = new MarkdownToJsonApi(cleanMarkdownFunnel, options);
+
+  // the default content folder is "content" and this tree needs to know
+  // about contentFolder for pagination links
+  const collationTree = new CollateJsonApiBlobs(jsonApiTree, {
+    contentFolder: 'content',
+    ...options,
+  });
+  const compiledTrees = new BroccoliMergeTrees([jsonApiTree, pagesTree, collationTree]);
   return mv(compiledTrees, (options.contentFolder || 'content'));
 };

lib/serialize-json-blobs.js → lib/collate-and-paginate.js

@@ -2,12 +2,10 @@ const Plugin = require('broccoli-plugin');
 const {
   readFileSync,
   writeFileSync,
-  mkdirSync,
 } = require('fs');
 const walkSync = require('walk-sync');
 const { join } = require('path');
 const _ = require('lodash');
-const constructSerializer = require('./construct-serializer');
 
 function preparePages(blobs, pageSize, pageSortFunction) {
   return _.chain(blobs)
@@ -25,42 +23,45 @@ class SerializeJsonBlobs extends Plugin {
   constructor(inputNode, options = {}) {
     super([inputNode], options);
     this.options = _.assign({}, {
-      contentFolder: 'content',
-      contentTypes: ['html', 'content'],
       collationFileName: 'all.json',
       pageSize: 10,
     }, options);
-    this.contentSerializer = constructSerializer(options);
   }
 
   build() {
     let blobs = [];
     this.inputPaths.forEach((inputPath) => {
       const paths = walkSync(inputPath);
       const folderConents = [];
+
       paths.forEach((path) => {
         if (path.endsWith('/')) {
-          mkdirSync(join(this.outputPath, path));
           return;
         }
         const fileContent = readFileSync(join(inputPath, path)).toString();
         const deserializedFile = JSON.parse(fileContent);
         folderConents.push(deserializedFile);
-        const jsonApiBlob = this.contentSerializer.serialize(deserializedFile);
-        writeFileSync(join(this.outputPath, path), JSON.stringify(jsonApiBlob));
       });
+
       blobs = [...blobs, ...folderConents];
     });
 
+    function groupData(pageData) {
+      return pageData.reduce((prev, current) => {
+        prev.data.push(current.data);
+        return prev;
+      }, { data: [] });
+    }
+
     if (this.options.collate) {
       if (this.options.paginate) {
         const contentPages = preparePages(
           blobs,
           this.options.pageSize,
-          this.options.paginateSortFunction
+          this.options.paginateSortFunction,
         );
         contentPages.forEach((pageData, index) => {
-          const serializedPageData = this.contentSerializer.serialize(pageData);
+          const serializedPageData = groupData(pageData);
           let fileName;
 
           const fileNameMatch = this.options.collationFileName.match(/(.*)\.json$/);
@@ -86,19 +87,23 @@ class SerializeJsonBlobs extends Plugin {
           }
           writeFileSync(
             join(this.outputPath, fileName),
-            JSON.stringify(serializedPageData)
+            JSON.stringify(serializedPageData),
           );
 
           // also write the default collection name for the first page
           if (index === 0) {
             writeFileSync(
               join(this.outputPath, this.options.collationFileName),
-              JSON.stringify(serializedPageData)
+              JSON.stringify(serializedPageData),
             );
           }
         });
       } else {
-        const collection = this.contentSerializer.serialize(blobs);
+        const collection = blobs.reduce((prev, blob) => {
+          prev.data.push(blob.data);
+          return prev;
+        }, { data: [] });
+
         const outputFile = join(this.outputPath, this.options.collationFileName);
         writeFileSync(outputFile, JSON.stringify(collection));
       }

lib/markdown-to-jsonapi.js

@@ -0,0 +1,82 @@
+const PersistentFilter = require('broccoli-persistent-filter');
+const yamlFront = require('yaml-front-matter');
+const showdown = require('showdown');
+const _ = require('lodash');
+const h2p = require('html2plaintext');
+const { Serializer } = require('jsonapi-serializer');
+
+const supportedContentTypes = ['content', 'html', 'description'];
+
+class MarkDownToJsonApi extends PersistentFilter {
+  constructor(folder, options) {
+    super(folder, options);
+    this.extensions = ['md', 'markdown'];
+    this.targetExtension = 'json';
+    this.options = {
+      contentTypes: ['html', 'content'],
+      type: 'content',
+      attributes: [],
+      references: [],
+      ...options,
+    };
+
+    const unsupportedContentTypes = _.difference(this.options.contentTypes, supportedContentTypes);
+
+    if (unsupportedContentTypes.length) {
+      throw new Error(`Unknown content type: ${unsupportedContentTypes[0]}`);
+    }
+
+    this.converter = new showdown.Converter();
+
+    // build serialiser for jsonapi
+    const serializerOptions = {
+      attributes: _.union(
+        this.options.contentTypes,
+        this.options.attributes,
+        this.options.references,
+      ),
+      keyForAttribute: 'camelCase',
+    };
+
+    this.options.references.forEach((reference) => {
+      serializerOptions[reference] = { ref: true };
+    });
+
+    this.serializer = new Serializer(this.options.type, serializerOptions);
+  }
+
+  processString(content, relativePath) {
+    const front = yamlFront.loadFront(content);
+    const markdown = front.__content.trim();
+
+    const baseProperties = {
+      path: relativePath,
+      id: relativePath.replace(/\.(md|markdown)$/, ''),
+      content: markdown,
+      html: this.converter.makeHtml(markdown),
+    };
+
+    const resultHash = { ...baseProperties, ...front };
+
+    if (!resultHash.description && _.includes(this.options.contentTypes, 'description')) {
+      const description = _.truncate(h2p(resultHash.html), {
+        length: 260,
+        separator: /,?\.* +/,
+      });
+
+      resultHash.description = description;
+    }
+
+    return JSON.stringify(this.serializer.serialize(resultHash));
+  }
+
+  // eslint-disable-next-line class-methods-use-this
+  getDestFilePath(relativePath) {
+    if (relativePath.endsWith('.md') || relativePath.endsWith('.markdown')) {
+      return `${relativePath.replace(/.(md|markdown)$/, '')}.json`;
+    }
+    return null;
+  }
+}
+
+module.exports = MarkDownToJsonApi;