diff --git a/package-lock.json b/package-lock.json index be95ff4..c37c6d2 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "ecmarkdown", - "version": "8.0.0", + "version": "8.1.0", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "ecmarkdown", - "version": "8.0.0", + "version": "8.1.0", "license": "WTFPL", "dependencies": { "escape-html": "^1.0.1" diff --git a/package.json b/package.json index 011728d..f4588cd 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "ecmarkdown", - "version": "8.0.0", + "version": "8.1.0", "description": "A compiler for \"Ecmarkdown\" algorithm shorthand into HTML.", "main": "dist/ecmarkdown.js", "scripts": { diff --git a/src/tokenizer.ts b/src/tokenizer.ts index 2522f64..24e0aad 100644 --- a/src/tokenizer.ts +++ b/src/tokenizer.ts @@ -3,7 +3,7 @@ import type { Unlocated, Token, AttrToken, Position } from './node-types'; const fieldOrSlotRegexp = /^\[\[[a-zA-Z0-9_]+\]\]/; const tagRegexp = /^<[/!]?(\w[\w-]*)(\s+\w[\w-]*(\s*=\s*("[^"]*"|'[^']*'|[^><"'=`]+))?)*\s*>/; const commentRegexp = /^/; -const attrRegexp = /^\[ *[\w-]+ *= *"(?:[^"\\\x00-\x1F]|\\["\\/bfnrt]|\\u[a-fA-F]{4})*" *(?:, *[\w-]+ *= *"(?:[^"\\\x00-\x1F]|\\["\\/bfnrt]|\\u[a-fA-F]{4})*" *)*] /; +const attrRegexp = /^\[ *[\w-]+ *(?:= *"(?:[^"\\\x00-\x1F]|\\["\\/bfnrt]|\\u[a-fA-F]{4})*")? *(?:, *[\w-]+ *(?:= *"(?:[^"\\\x00-\x1F]|\\["\\/bfnrt]|\\u[a-fA-F]{4})*")? *)*] /; const digitRegexp = /\d/; const opaqueTags = new Set(['emu-grammar', 'emu-production', 'pre', 'code', 'script', 'style']); @@ -158,13 +158,17 @@ export class Tokenizer { // attribute tokens are only valid immediately after list tokens, so we let this be called by the parser. tryScanListItemAttributes() { - const match = this.str.slice(this.pos).match(attrRegexp); + const rest = this.str.slice(this.pos); + const match = rest.match(attrRegexp); if (!match) { + if (rest.startsWith('[')) { + this.raise('could not parse attributes for step', this.getLocation()); + } return []; } const parts = match[0].matchAll( - /([\w-]+) *= *("(?:[^"\\\x00-\x1F]|\\["\\/bfnrt]|\\u[a-fA-F]{4})*")/g + /([\w-]+) *(?:= *("(?:[^"\\\x00-\x1F]|\\["\\/bfnrt]|\\u[a-fA-F]{4})*"))?/g ); const tokens = []; let offset = 0; @@ -178,7 +182,7 @@ export class Tokenizer { const tok: Unlocated = { name: 'attr', key, - value: JSON.parse(value), + value: value == null ? '' : JSON.parse(value), // the parse is guaranteed to succeed because the regex is quite restricted }; this.pos += part.length; this.locate(tok, tokStart); diff --git a/test/cases/list-attrs.ecmarkdown b/test/cases/list-attrs.ecmarkdown index 1dcea7d..855d559 100644 --- a/test/cases/list-attrs.ecmarkdown +++ b/test/cases/list-attrs.ecmarkdown @@ -2,6 +2,7 @@ 1. Item 1. Item 1. [attr="bar"] Item + 1. [no-value, with-empty="", with-value="val"] Item 1. Item 1. Item * Unordered item diff --git a/test/cases/list-attrs.html b/test/cases/list-attrs.html index 4fdcc37..c148b68 100644 --- a/test/cases/list-attrs.html +++ b/test/cases/list-attrs.html @@ -3,6 +3,7 @@
  • Item
  • Item
    1. Item
    2. +
    3. Item
    4. Item
  • diff --git a/test/errors.js b/test/errors.js index dd5e58d..d4a990a 100644 --- a/test/errors.js +++ b/test/errors.js @@ -68,4 +68,13 @@ ${M} * b 'Unexpected token parabreak; expected EOF' ); }); + + it('bad step attributes', function () { + assertAlgError( + positioned` + 1. ${M}[x y] + `, + 'could not parse attributes for step' + ); + }); }); diff --git a/test/parser.js b/test/parser.js index 0d8d5f3..3a8007e 100644 --- a/test/parser.js +++ b/test/parser.js @@ -4,14 +4,16 @@ const { parseAlgorithm, parseFragment } = require('../'); describe('Parser', function () { it('tracks positions in algorithms', function () { - const baseSource = ' 1. [id="thing"] a\n 2. b c\n
    text
    '; + const baseSource = ' 1. [id="thing", other] a\n 2. b c\n
    text
    '; const assertNodeLocation = makeAssertLocation(baseSource); const algorithm = parseAlgorithm(baseSource); assertNodeLocation(algorithm, baseSource); const list = algorithm.contents; - assertNodeLocation(list, ' 1. [id="thing"] a\n 2. b c\n
    text
    '); + assertNodeLocation(list, ' 1. [id="thing", other] a\n 2. b c\n
    text
    '); const item0 = list.contents[0]; - assertNodeLocation(item0, ' 1. [id="thing"] a\n'); + assertNodeLocation(item0, ' 1. [id="thing", other] a\n'); + assertNodeLocation(item0.attrs[0], 'id="thing"'); + assertNodeLocation(item0.attrs[1], 'other'); assertNodeLocation(item0.contents[0], 'a'); const item1 = list.contents[1]; assertNodeLocation(item1, ' 2. b c\n
    text
    ');