From e8526029529d1e5147b3ef8b7ff534cddcb856fa Mon Sep 17 00:00:00 2001 From: Islam Naasani Date: Mon, 29 Jan 2024 22:30:56 +0300 Subject: [PATCH 1/9] feat: add mdx support --- doctoc.js | 16 ++++++++-------- lib/file.js | 2 +- lib/transform.js | 46 +++++++++++++++++++++++++++++++++++++--------- 3 files changed, 46 insertions(+), 18 deletions(-) diff --git a/doctoc.js b/doctoc.js index 578dfa9..7cdad22 100755 --- a/doctoc.js +++ b/doctoc.js @@ -16,7 +16,7 @@ function cleanPath(path) { return homeExpanded.replace(/\s/g, '\\ '); } -function transformAndSave(files, mode, maxHeaderLevel, title, notitle, entryPrefix, processAll, stdOut, updateOnly) { +function transformAndSave(files, mode, maxHeaderLevel, title, notitle, entryPrefix, processAll, stdOut, updateOnly, syntax) { if (processAll) { console.log('--all flag is enabled. Including headers before the TOC location.') } @@ -24,13 +24,13 @@ function transformAndSave(files, mode, maxHeaderLevel, title, notitle, entryPref if (updateOnly) { console.log('--update-only flag is enabled. Only updating files that already have a TOC.') } - + console.log('\n==================\n'); var transformed = files .map(function (x) { var content = fs.readFileSync(x.path, 'utf8') - , result = transform(content, mode, maxHeaderLevel, title, notitle, entryPrefix, processAll, updateOnly); + , result = transform(content, mode, maxHeaderLevel, title, notitle, entryPrefix, processAll, updateOnly, syntax); result.path = x.path; return result; }); @@ -48,7 +48,7 @@ function transformAndSave(files, mode, maxHeaderLevel, title, notitle, entryPref console.log('"%s" is up to date', x.path); }); - changed.forEach(function (x) { + changed.forEach(function (x) { if (stdOut) { console.log('==================\n\n"%s" should be updated', x.path) } else { @@ -62,7 +62,7 @@ function printUsageAndExit(isErr) { var outputFunc = isErr ? console.error : console.info; - outputFunc('Usage: doctoc [mode] [--entryprefix prefix] [--notitle | --title title] [--maxlevel level] [--all] [--update-only] (where path is some path to a directory (e.g., .) or a file (e.g., README.md))'); + outputFunc('Usage: doctoc [mode] [--entryprefix prefix] [--notitle | --title title] [--maxlevel level] [--all] [--update-only] [--syntax (md|mdx)] (where path is some path to a directory (e.g., .) or a file (e.g., README.md))'); outputFunc('\nAvailable modes are:'); for (var key in modes) { outputFunc(' --%s\t%s', key, modes[key]); @@ -84,7 +84,7 @@ var mode = modes['github']; var argv = minimist(process.argv.slice(2) , { boolean: [ 'h', 'help', 'T', 'notitle', 's', 'stdout', 'all' , 'u', 'update-only'].concat(Object.keys(modes)) - , string: [ 'title', 't', 'maxlevel', 'm', 'entryprefix' ] + , string: [ 'title', 't', 'maxlevel', 'm', 'entryprefix', 'syntax' ] , unknown: function(a) { return (a[0] == '-' ? (console.error('Unknown option(s): ' + a), printUsageAndExit(true)) : true); } }); @@ -104,7 +104,7 @@ var entryPrefix = argv.entryprefix || '-'; var processAll = argv.all; var stdOut = argv.s || argv.stdout var updateOnly = argv.u || argv['update-only'] - +var syntax = argv['syntax'] || "md" var maxHeaderLevel = argv.m || argv.maxlevel; if (maxHeaderLevel && isNaN(maxHeaderLevel) || maxHeaderLevel < 0) { console.error('Max. heading level specified is not a positive number: ' + maxHeaderLevel), printUsageAndExit(true); } @@ -120,7 +120,7 @@ for (var i = 0; i < argv._.length; i++) { files = [{ path: target }]; } - transformAndSave(files, mode, maxHeaderLevel, title, notitle, entryPrefix, processAll, stdOut, updateOnly); + transformAndSave(files, mode, maxHeaderLevel, title, notitle, entryPrefix, processAll, stdOut, updateOnly, syntax); console.log('\nEverything is OK.'); } diff --git a/lib/file.js b/lib/file.js index 1479be9..47c85a8 100644 --- a/lib/file.js +++ b/lib/file.js @@ -2,7 +2,7 @@ var path = require('path') , fs = require('fs') , _ = require('underscore'); -var markdownExts = ['.md', '.markdown']; +var markdownExts = ['.md', '.markdown', '.mdx']; var ignoredDirs = ['.', '..', '.git', 'node_modules']; function separateFilesAndDirs(fileInfos) { diff --git a/lib/transform.js b/lib/transform.js index afd713c..636a19a 100644 --- a/lib/transform.js +++ b/lib/transform.js @@ -6,18 +6,29 @@ var _ = require('underscore') , getHtmlHeaders = require('./get-html-headers') , md = require('@textlint/markdown-to-ast'); -var start = '\n' + - '' - , end = '' - , skipTag = ''; +var commentTypes = { + md: { + start: "", + escapedStart: " comments if syntax=md', function (t) { + var res = transform( + [ '# My Module' + , 'Some text here' + , '## API' + , '### Method One' + , 'works like this' + , '### Method Two' + , '#### Main Usage' + , 'some main usage here' + ].join('\n') + , undefined + , undefined + , undefined + , undefined + , undefined + , undefined + , undefined + , 'md' + ) + + var lines = res.wrappedToc.split('\n') + var commentLines = lines.slice(0,2).concat(lines[lines.length-1]) + t.deepEqual(commentLines.every((line) => line.startsWith('')), true) + t.end() +}) + +test('should use {/* */} comments if syntax=mdx', function (t) { + var res = transform( + [ '# My Module' + , 'Some text here' + , '## API' + , '### Method One' + , 'works like this' + , '### Method Two' + , '#### Main Usage' + , 'some main usage here' + ].join('\n') + , undefined + , undefined + , undefined + , undefined + , undefined + , undefined + , undefined + , 'mdx' + ) + + var lines = res.wrappedToc.split('\n') + var commentLines = lines.slice(0 ,2).concat(lines[lines.length - 1]) + + t.deepEqual(commentLines.every((line) => line.startsWith('{/*') && line.endsWith('*/}')), true) + t.end() +}) \ No newline at end of file From 24a6c96410acf3bf88ac51d33179ae1dabfc1780 Mon Sep 17 00:00:00 2001 From: Islam Naasani Date: Mon, 3 Jun 2024 20:26:07 +0300 Subject: [PATCH 4/9] fix(transform): remove commentEscapedStart dynamic re-assignment --- lib/transform.js | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/lib/transform.js b/lib/transform.js index 25a0272..1b41091 100644 --- a/lib/transform.js +++ b/lib/transform.js @@ -21,14 +21,18 @@ var commentTypes = { var { start: startMd , end: endMd } = generateComments("md") -var commentEscapedStart = commentTypes.md.escapedStart - -function matchesStart(line) { - return new RegExp(`${commentEscapedStart} START doctoc `).test(line); +function matchesStart(syntax) { + var commentEscapedStart = commentTypes[syntax].escapedStart + return function(line){ + return new RegExp(`${commentEscapedStart} START doctoc `).test(line); + } } -function matchesEnd(line) { - return new RegExp(`${commentEscapedStart} END doctoc `).test(line); +function matchesEnd(syntax) { + var commentEscapedStart = commentTypes[syntax].escapedStart + return function(line){ + return new RegExp(`${commentEscapedStart} END doctoc `).test(line); + } } function notNull(x) { return x !== null; } @@ -134,21 +138,19 @@ function determineTitle(title, notitle, lines, info) { exports = module.exports = function transform(content, mode, maxHeaderLevel, title, notitle, entryPrefix, processAll, updateOnly, syntax) { syntax = syntax || "md" - const { skipTag, start, end } = generateComments(syntax) - + var { skipTag, start, end } = generateComments(syntax) + var matchesStartBySyntax = matchesStart(syntax) + var matchesEndBySyntax = matchesEnd(syntax) if (content.indexOf(skipTag) !== -1) return { transformed: false }; mode = mode || 'github.com'; entryPrefix = entryPrefix || '-'; - // update it for matchesStart and matchesEnd - commentEscapedStart = commentTypes[syntax]?.escapedStart || commentTypes.md.escapedStart - // only limit *HTML* headings by default var maxHeaderLevelHtml = maxHeaderLevel || 4; var lines = content.split('\n') - , info = updateSection.parse(lines, matchesStart, matchesEnd) + , info = updateSection.parse(lines, matchesStartBySyntax, matchesEndBySyntax) if (!info.hasStart && updateOnly) { return { transformed: false }; @@ -194,7 +196,7 @@ exports = module.exports = function transform(content, mode, maxHeaderLevel, tit if (currentToc === toc) return { transformed: false }; - var data = updateSection(lines.join('\n'), wrappedToc, matchesStart, matchesEnd, true); + var data = updateSection(lines.join('\n'), wrappedToc, matchesStartBySyntax, matchesEndBySyntax, true); return { transformed : true, data : data, toc: toc, wrappedToc: wrappedToc }; }; From a86cec9b15fe237bc8a5445ff60b6a739bd41749 Mon Sep 17 00:00:00 2001 From: Islam Naasani Date: Mon, 3 Jun 2024 20:38:54 +0300 Subject: [PATCH 5/9] fix: remove unnecessary 'syntax = md' --- doctoc.js | 2 +- lib/file.js | 2 +- lib/transform.js | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/doctoc.js b/doctoc.js index 202a723..8c0c461 100755 --- a/doctoc.js +++ b/doctoc.js @@ -104,7 +104,7 @@ var entryPrefix = argv.entryprefix || '-'; var processAll = argv.all; var stdOut = argv.s || argv.stdout var updateOnly = argv.u || argv['update-only'] -var syntax = argv['syntax'] || "md" +var syntax = argv['syntax'] || 'md' var maxHeaderLevel = argv.m || argv.maxlevel; if (maxHeaderLevel && isNaN(maxHeaderLevel) || maxHeaderLevel < 0) { console.error('Max. heading level specified is not a positive number: ' + maxHeaderLevel), printUsageAndExit(true); } diff --git a/lib/file.js b/lib/file.js index eaef576..ab284d1 100644 --- a/lib/file.js +++ b/lib/file.js @@ -6,7 +6,7 @@ var markdownExts = ['.md', '.markdown']; var mdxExts = ['.mdx']; var ignoredDirs = ['.', '..', '.git', 'node_modules']; -function separateFilesAndDirs(fileInfos, syntax = "md") {; +function separateFilesAndDirs(fileInfos, syntax) { return { directories : _(fileInfos).filter(function (x) { return x.isDirectory() && !_(ignoredDirs).include(x.name); diff --git a/lib/transform.js b/lib/transform.js index 1b41091..9dabeb5 100644 --- a/lib/transform.js +++ b/lib/transform.js @@ -19,7 +19,7 @@ var commentTypes = { } } -var { start: startMd , end: endMd } = generateComments("md") +var { start: startMd , end: endMd } = generateComments('md') function matchesStart(syntax) { var commentEscapedStart = commentTypes[syntax].escapedStart @@ -47,8 +47,8 @@ function isString(y) { } function generateComments(syntax){ - var commentStart = commentTypes[syntax]?.start || commentTypes.md.start - , commentEnd = commentTypes[syntax]?.end || commentTypes.md.end + var commentStart = commentTypes[syntax].start + , commentEnd = commentTypes[syntax].end; var start = `${commentStart} START doctoc generated TOC please keep comment here to allow auto update ${commentEnd}\n` + `${commentStart} DON\'T EDIT THIS SECTION, INSTEAD RE-RUN doctoc TO UPDATE ${commentEnd}` From 6f4cd585d370b9c7982a52f7ca6c2b7deb8617ab Mon Sep 17 00:00:00 2001 From: Islam Naasani Date: Mon, 3 Jun 2024 20:41:30 +0300 Subject: [PATCH 6/9] fix(file): use look-up object instead of ternary --- lib/file.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/file.js b/lib/file.js index ab284d1..0457cda 100644 --- a/lib/file.js +++ b/lib/file.js @@ -2,17 +2,18 @@ var path = require('path') , fs = require('fs') , _ = require('underscore'); -var markdownExts = ['.md', '.markdown']; -var mdxExts = ['.mdx']; var ignoredDirs = ['.', '..', '.git', 'node_modules']; - +var extensions = { + mdx: ['.mdx'], + md: ['.md', '.markdown'], +} function separateFilesAndDirs(fileInfos, syntax) { return { directories : _(fileInfos).filter(function (x) { return x.isDirectory() && !_(ignoredDirs).include(x.name); }), markdownFiles : _(fileInfos).filter(function (x) { - return x.isFile() && _(syntax === 'mdx'? mdxExts : markdownExts).include(path.extname(x.name)); + return x.isFile() && _(extensions[syntax]).include(path.extname(x.name)); }) }; } From 60fdc56220c0f48dab678754e9acf80cb36cc70e Mon Sep 17 00:00:00 2001 From: Islam Naasani Date: Mon, 3 Jun 2024 21:05:33 +0300 Subject: [PATCH 7/9] fix(doctoc): validate --syntax user input --- doctoc.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/doctoc.js b/doctoc.js index 8c0c461..383d182 100755 --- a/doctoc.js +++ b/doctoc.js @@ -62,7 +62,7 @@ function printUsageAndExit(isErr) { var outputFunc = isErr ? console.error : console.info; - outputFunc('Usage: doctoc [mode] [--entryprefix prefix] [--notitle | --title title] [--maxlevel level] [--all] [--update-only] [--syntax (md|mdx)] (where path is some path to a directory (e.g., .) or a file (e.g., README.md))'); + outputFunc('Usage: doctoc [mode] [--entryprefix prefix] [--notitle | --title title] [--maxlevel level] [--all] [--update-only] [--syntax (' + supportedSyntaxes.join("|") + ')] (where path is some path to a directory (e.g., .) or a file (e.g., README.md))'); outputFunc('\nAvailable modes are:'); for (var key in modes) { outputFunc(' --%s\t%s', key, modes[key]); @@ -72,6 +72,7 @@ function printUsageAndExit(isErr) { process.exit(isErr ? 2 : 0); } +var supportedSyntaxes = ['md', 'mdx'] var modes = { bitbucket : 'bitbucket.org' , nodejs : 'nodejs.org' @@ -92,6 +93,11 @@ if (argv.h || argv.help) { printUsageAndExit(); } +if(argv['syntax']!==undefined && !supportedSyntaxes.includes(argv['syntax'])){ + console.error('Unknown syntax:',argv['syntax']) + console.error('\nSupported Syntaxes are:', supportedSyntaxes.join(", ")) + process.exit(2) +} for (var key in modes) { if (argv[key]) { mode = modes[key]; From f502d3cdcf40633b200ab236b6d335a0ee13e310 Mon Sep 17 00:00:00 2001 From: Islam Naasani Date: Mon, 3 Jun 2024 21:11:44 +0300 Subject: [PATCH 8/9] feat(tests/transform): pull reusable getCommentLines function --- test/transform.js | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/test/transform.js b/test/transform.js index 1950853..30e2ccb 100644 --- a/test/transform.js +++ b/test/transform.js @@ -32,6 +32,11 @@ function check(md, anchors, mode, maxHeaderLevel, title, notitle, entryPrefix, p t.end() }) } + +function getCommentLines(transformRes){ + var lines = transformRes.wrappedToc.split('\n') + return lines.slice(0,2).concat(lines[lines.length - 1]) +} //function check() {} check( @@ -221,7 +226,7 @@ check( test('transforming when old toc exists', function (t) { - var md = [ + var md = [ '# Header above' , '' , 'The above header should be ignored since it is above the existing toc' @@ -232,7 +237,7 @@ test('transforming when old toc exists', function (t) { , '' , '- [OldHeader](#oldheader)' , '' - , '' + , '' , '## Header' , 'some content' , '' @@ -240,7 +245,7 @@ test('transforming when old toc exists', function (t) { var res = transform(md) - t.ok(res.transformed, 'transforms it') + t.ok(res.transformed, 'transforms it') t.deepEqual( res.toc.split('\n') @@ -248,9 +253,9 @@ test('transforming when old toc exists', function (t) { '', '- [Header](#header)', '' ] - , 'replaces old toc' + , 'replaces old toc' ) - + t.deepEqual( res.wrappedToc.split('\n') , [ '', @@ -259,7 +264,7 @@ test('transforming when old toc exists', function (t) { '', '- [Header](#header)', '', - '' + '' ] , 'wraps old toc' ) @@ -281,12 +286,12 @@ test('transforming when old toc exists', function (t) { 'some content', '' ] , 'updates the content with the new toc and ignores header before existing toc' - ) + ) t.end() }) test('transforming when old toc exists and --all flag is set', function (t) { - var md = [ + var md = [ '# Header above' , '' , 'The above header should be ignored since it is above the existing toc' @@ -297,7 +302,7 @@ test('transforming when old toc exists and --all flag is set', function (t) { , '' , '- [OldHeader](#oldheader)' , '' - , '' + , '' , '## Header' , 'some content' , '' @@ -305,7 +310,7 @@ test('transforming when old toc exists and --all flag is set', function (t) { var res = transform(md, undefined, undefined, undefined, undefined, undefined, true) - t.ok(res.transformed, 'transforms it') + t.ok(res.transformed, 'transforms it') t.deepEqual( res.toc.split('\n') @@ -314,9 +319,9 @@ test('transforming when old toc exists and --all flag is set', function (t) { '- [Header above](#header-above)', ' - [Header](#header)', '' ] - , 'replaces old toc' + , 'replaces old toc' ) - + t.deepEqual( res.wrappedToc.split('\n') , [ '', @@ -326,7 +331,7 @@ test('transforming when old toc exists and --all flag is set', function (t) { '- [Header above](#header-above)', ' - [Header](#header)', '', - '' + '' ] , 'wraps old toc' ) @@ -349,7 +354,7 @@ test('transforming when old toc exists and --all flag is set', function (t) { 'some content', '' ] , 'updates the content with the new toc and ignores header before existing toc' - ) + ) t.end() }) @@ -547,8 +552,7 @@ test('should use comments if syntax=md', function (t) { , 'md' ) - var lines = res.wrappedToc.split('\n') - var commentLines = lines.slice(0,2).concat(lines[lines.length-1]) + var commentLines = getCommentLines(res); t.deepEqual(commentLines.every((line) => line.startsWith('')), true) t.end() }) @@ -574,9 +578,7 @@ test('should use {/* */} comments if syntax=mdx', function (t) { , 'mdx' ) - var lines = res.wrappedToc.split('\n') - var commentLines = lines.slice(0 ,2).concat(lines[lines.length - 1]) - + var commentLines = getCommentLines(res); t.deepEqual(commentLines.every((line) => line.startsWith('{/*') && line.endsWith('*/}')), true) t.end() }) \ No newline at end of file From 33bfcf82ba39aa6b1603db91cd12a0adaec44938 Mon Sep 17 00:00:00 2001 From: Islam Nassani <73395945+I-3B@users.noreply.github.com> Date: Wed, 5 Jun 2024 19:25:03 +0300 Subject: [PATCH 9/9] Update doctoc.js Co-authored-by: Andrew Smith --- doctoc.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doctoc.js b/doctoc.js index 383d182..cb22904 100755 --- a/doctoc.js +++ b/doctoc.js @@ -93,9 +93,9 @@ if (argv.h || argv.help) { printUsageAndExit(); } -if(argv['syntax']!==undefined && !supportedSyntaxes.includes(argv['syntax'])){ - console.error('Unknown syntax:',argv['syntax']) - console.error('\nSupported Syntaxes are:', supportedSyntaxes.join(", ")) +if (argv['syntax'] !== undefined && !supportedSyntaxes.includes(argv['syntax'])) { + console.error('Unknown syntax:', argv['syntax']) + console.error('Supported options:', supportedSyntaxes.join(", ")) process.exit(2) } for (var key in modes) {