feat: add mdx support

doctoc.js

@@ -16,21 +16,21 @@ function cleanPath(path) {
   return homeExpanded.replace(/\s/g, '\\ ');
 }
 
-function transformAndSave(files, mode, maxHeaderLevel, title, notitle, entryPrefix, processAll, stdOut, updateOnly) {
+function transformAndSave(files, mode, maxHeaderLevel, title, notitle, entryPrefix, processAll, stdOut, updateOnly, syntax) {
   if (processAll) {
     console.log('--all flag is enabled. Including headers before the TOC location.')
   }
 
   if (updateOnly) {
     console.log('--update-only flag is enabled. Only updating files that already have a TOC.')
   }
-  
+
   console.log('\n==================\n');
 
   var transformed = files
     .map(function (x) {
       var content = fs.readFileSync(x.path, 'utf8')
-        , result = transform(content, mode, maxHeaderLevel, title, notitle, entryPrefix, processAll, updateOnly);
+        , result = transform(content, mode, maxHeaderLevel, title, notitle, entryPrefix, processAll, updateOnly, syntax);
       result.path = x.path;
       return result;
     });
@@ -48,7 +48,7 @@ function transformAndSave(files, mode, maxHeaderLevel, title, notitle, entryPref
     console.log('"%s" is up to date', x.path);
   });
 
-  changed.forEach(function (x) { 
+  changed.forEach(function (x) {
     if (stdOut) {
       console.log('==================\n\n"%s" should be updated', x.path)
     } else {
@@ -62,7 +62,7 @@ function printUsageAndExit(isErr) {
 
   var outputFunc = isErr ? console.error : console.info;
 
-  outputFunc('Usage: doctoc [mode] [--entryprefix prefix] [--notitle | --title title] [--maxlevel level] [--all] [--update-only] <path> (where path is some path to a directory (e.g., .) or a file (e.g., README.md))');
+  outputFunc('Usage: doctoc [mode] [--entryprefix prefix] [--notitle | --title title] [--maxlevel level] [--all] [--update-only] [--syntax (' + supportedSyntaxes.join("|") + ')] <path> (where path is some path to a directory (e.g., .) or a file (e.g., README.md))');
   outputFunc('\nAvailable modes are:');
   for (var key in modes) {
     outputFunc('  --%s\t%s', key, modes[key]);
@@ -72,6 +72,7 @@ function printUsageAndExit(isErr) {
   process.exit(isErr ? 2 : 0);
 }
 
+var supportedSyntaxes = ['md', 'mdx']
 var modes = {
     bitbucket : 'bitbucket.org'
   , nodejs    : 'nodejs.org'
@@ -84,14 +85,19 @@ var mode = modes['github'];
 
 var argv = minimist(process.argv.slice(2)
     , { boolean: [ 'h', 'help', 'T', 'notitle', 's', 'stdout', 'all' , 'u', 'update-only'].concat(Object.keys(modes))
-    , string: [ 'title', 't', 'maxlevel', 'm', 'entryprefix' ]
+    , string: [ 'title', 't', 'maxlevel', 'm', 'entryprefix', 'syntax' ]
     , unknown: function(a) { return (a[0] == '-' ? (console.error('Unknown option(s): ' + a), printUsageAndExit(true)) : true); }
     });
 
 if (argv.h || argv.help) {
   printUsageAndExit();
 }
 
+if (argv['syntax'] !== undefined && !supportedSyntaxes.includes(argv['syntax'])) {
+  console.error('Unknown syntax:', argv['syntax'])
+  console.error('Supported options:', supportedSyntaxes.join(", "))
+  process.exit(2)
+}
 for (var key in modes) {
   if (argv[key]) {
     mode = modes[key];
@@ -104,7 +110,7 @@ var entryPrefix = argv.entryprefix || '-';
 var processAll = argv.all;
 var stdOut = argv.s || argv.stdout
 var updateOnly = argv.u || argv['update-only']
-
+var syntax = argv['syntax'] || 'md'
 var maxHeaderLevel = argv.m || argv.maxlevel;
 if (maxHeaderLevel && isNaN(maxHeaderLevel) || maxHeaderLevel < 0) { console.error('Max. heading level specified is not a positive number: ' + maxHeaderLevel), printUsageAndExit(true); }
 
@@ -114,13 +120,13 @@ for (var i = 0; i < argv._.length; i++) {
 
   if (stat.isDirectory()) {
     console.log ('\nDocToccing "%s" and its sub directories for %s.', target, mode);
-    files = file.findMarkdownFiles(target);
+    files = file.findMarkdownFiles(target, syntax);
   } else {
     console.log ('\nDocToccing single file "%s" for %s.', target, mode);
     files = [{ path: target }];
   }
 
-  transformAndSave(files, mode, maxHeaderLevel, title, notitle, entryPrefix, processAll, stdOut, updateOnly);
+  transformAndSave(files, mode, maxHeaderLevel, title, notitle, entryPrefix, processAll, stdOut, updateOnly, syntax);
 
   console.log('\nEverything is OK.');
 }

lib/file.js

@@ -2,58 +2,60 @@ var path  =  require('path')
  ,  fs  =  require('fs')
  ,  _   =  require('underscore');
 
-var markdownExts = ['.md', '.markdown'];
 var ignoredDirs  = ['.', '..', '.git', 'node_modules'];
-
-function separateFilesAndDirs(fileInfos) {
+var extensions = {
+  mdx: ['.mdx'],
+  md: ['.md', '.markdown'],
+}
+function separateFilesAndDirs(fileInfos, syntax) {
   return {
     directories :  _(fileInfos).filter(function (x) {
       return x.isDirectory() && !_(ignoredDirs).include(x.name);
     }),
-    markdownFiles :  _(fileInfos).filter(function (x) { 
-      return x.isFile() && _(markdownExts).include(path.extname(x.name)); 
+    markdownFiles :  _(fileInfos).filter(function (x) {
+      return x.isFile() && _(extensions[syntax]).include(path.extname(x.name));
     })
   };
 }
 
-function findRec(currentPath) {
+function findRec(currentPath, syntax) {
   function getStat (entry) {
     var target = path.join(currentPath, entry),
       stat = fs.statSync(target);
 
-    return  _(stat).extend({ 
+    return  _(stat).extend({
       name: entry,
       path: target
     });
   }
-  
+
   function process (fileInfos) {
-    var res = separateFilesAndDirs(fileInfos);
+    var res = separateFilesAndDirs(fileInfos, syntax);
     var tgts = _(res.directories).pluck('path');
 
-    if (res.markdownFiles.length > 0) 
+    if (res.markdownFiles.length > 0)
       console.log('\nFound %s in "%s"', _(res.markdownFiles).pluck('name').join(', '), currentPath);
-    else 
+    else
       console.log('\nFound nothing in "%s"', currentPath);
 
-    return { 
+    return {
       markdownFiles :  res.markdownFiles,
       subdirs     :  tgts
     };
   }
 
   var stats                  =  _(fs.readdirSync(currentPath)).map(getStat)
     , res                    =  process(stats)
-    , markdownsInSubdirs     =  _(res.subdirs).map(findRec)
+    , markdownsInSubdirs     =  _(res.subdirs).map((subdir)=> findRec(subdir, syntax))
     , allMarkdownsHereAndSub =  res.markdownFiles.concat(markdownsInSubdirs);
 
   return _(allMarkdownsHereAndSub).flatten();
 }
 
 // Finds all markdown files in given directory and its sub-directories
-// @param {String  } dir - the absolute directory to search in 
-exports.findMarkdownFiles = function(dir) {
-  return findRec(dir);
+// @param {String  } dir - the absolute directory to search in
+exports.findMarkdownFiles = function(dir, syntax) {
+  return findRec(dir, syntax);
 };
 
 /* Example:

lib/transform.js

@@ -6,18 +6,33 @@ var _             = require('underscore')
   , getHtmlHeaders = require('./get-html-headers')
   , md            = require('@textlint/markdown-to-ast');
 
-var start = '<!-- START doctoc generated TOC please keep comment here to allow auto update -->\n' +
-            '<!-- DON\'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE -->'
-  , end   = '<!-- END doctoc generated TOC please keep comment here to allow auto update -->'
-  , skipTag = '<!-- DOCTOC SKIP -->';
+var commentTypes = {
+  md: {
+    start: "<!--",
+    end: "-->",
+    escapedStart: "<!--",
+  },
+  mdx: {
+    start: "{/*",
+    end: "*/}",
+    escapedStart: "{\/\\*",
+  }
+}
 
+var { start: startMd , end: endMd } = generateComments('md')
 
-function matchesStart(line) {
-  return (/<!-- START doctoc /).test(line);
+function matchesStart(syntax) {
+  var commentEscapedStart = commentTypes[syntax].escapedStart
+  return function(line){
+    return new RegExp(`${commentEscapedStart} START doctoc `).test(line);
+  }
 }
 
-function matchesEnd(line) {
-  return (/<!-- END doctoc /).test(line);
+function matchesEnd(syntax) {
+  var commentEscapedStart = commentTypes[syntax].escapedStart
+  return function(line){
+    return new RegExp(`${commentEscapedStart} END doctoc `).test(line);
+  }
 }
 
 function notNull(x) { return  x !== null; }
@@ -31,6 +46,19 @@ function isString(y) {
   return typeof y === 'string';
 }
 
+function generateComments(syntax){
+  var commentStart = commentTypes[syntax].start
+  , commentEnd = commentTypes[syntax].end;
+
+  var start = `${commentStart} START doctoc generated TOC please keep comment here to allow auto update ${commentEnd}\n` +
+            `${commentStart} DON\'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE ${commentEnd}`
+  , end   = `${commentStart} END doctoc generated TOC please keep comment here to allow auto update ${commentEnd}`
+  , skipTag = `${commentStart} DOCTOC SKIP ${commentEnd}`;
+
+  exports.start = start;
+  exports.end = end;
+  return { start, end, skipTag}
+}
 
 function getMarkdownHeaders (lines, maxHeaderLevel) {
   function extractText (header) {
@@ -108,7 +136,11 @@ function determineTitle(title, notitle, lines, info) {
   return info.hasStart ? lines[info.startIdx + 2] : defaultTitle;
 }
 
-exports = module.exports = function transform(content, mode, maxHeaderLevel, title, notitle, entryPrefix, processAll, updateOnly) {
+exports = module.exports = function transform(content, mode, maxHeaderLevel, title, notitle, entryPrefix, processAll, updateOnly, syntax) {
+  syntax = syntax || "md"
+  var { skipTag, start, end } = generateComments(syntax)
+  var matchesStartBySyntax = matchesStart(syntax)
+  var matchesEndBySyntax = matchesEnd(syntax)
   if (content.indexOf(skipTag) !== -1) return { transformed: false };
 
   mode = mode || 'github.com';
@@ -118,7 +150,7 @@ exports = module.exports = function transform(content, mode, maxHeaderLevel, tit
   var maxHeaderLevelHtml = maxHeaderLevel || 4;
 
   var lines = content.split('\n')
-    , info = updateSection.parse(lines, matchesStart, matchesEnd)
+    , info = updateSection.parse(lines, matchesStartBySyntax, matchesEndBySyntax)
 
   if (!info.hasStart && updateOnly) {
     return { transformed: false };
@@ -164,9 +196,9 @@ exports = module.exports = function transform(content, mode, maxHeaderLevel, tit
 
   if (currentToc === toc) return { transformed: false };
 
-  var data = updateSection(lines.join('\n'), wrappedToc, matchesStart, matchesEnd, true);
+  var data = updateSection(lines.join('\n'), wrappedToc, matchesStartBySyntax, matchesEndBySyntax, true);
   return { transformed : true, data : data, toc: toc, wrappedToc: wrappedToc };
 };
 
-exports.start = start;
-exports.end = end;
+exports.start = startMd;
+exports.end = endMd;

test/file.js

@@ -0,0 +1,33 @@
+'use strict';
+/*jshint asi: true */
+
+var test = require('tap').test
+  , findMarkdownFiles = require('../lib/file').findMarkdownFiles
+  ,  _   =  require('underscore');
+const fixturesDir = __dirname + "/fixtures"
+test('\nmatch mdx files only with syntax=mdx', function (t) {
+  t.deepEqual(
+    _(findMarkdownFiles(fixturesDir, 'mdx')).every((file) => file.path.endsWith(".mdx"))
+    , true
+    , 'match mdx files only with syntax=mdx'
+  )
+  t.end()
+});
+
+test('\nmatch md files only with syntax=md', function (t) {
+  t.deepEqual(
+    _(findMarkdownFiles(fixturesDir, 'md')).every((file) => file.path.endsWith(".md"))
+    , true
+    , 'match md files only with syntax=md'
+  )
+  t.end()
+});
+
+test('\nmatch md files only with undefined syntax', function (t) {
+  t.deepEqual(
+    _(findMarkdownFiles(fixturesDir, 'md')).every((file) => file.path.endsWith(".md"))
+    , true
+    , 'match md files only with undefined syntax'
+  )
+  t.end()
+});

test/fixtures/readme-syntax.md

@@ -0,0 +1,9 @@
+# Hello, world!
+
+should only be matched if syntax=md
+
+## Installation
+
+## API
+
+## License

test/fixtures/readme-syntax.mdx

@@ -0,0 +1,9 @@
+# Hello, world!
+
+should only be matched if syntax=mdx
+
+## Installation
+
+## API
+
+## License