From c045374f7090cc476aeb257652c2ed6467224c52 Mon Sep 17 00:00:00 2001 From: Peter Plantinga Date: Mon, 27 Apr 2020 13:18:33 -0400 Subject: [PATCH 1/7] YAML remove inline sequence and mapping chars from strings --- src/languages/yaml.js | 2 +- test/markup/yaml/inline.expect.txt | 2 ++ test/markup/yaml/inline.txt | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) create mode 100644 test/markup/yaml/inline.expect.txt create mode 100644 test/markup/yaml/inline.txt diff --git a/src/languages/yaml.js b/src/languages/yaml.js index 4fc4ba0a28..d523bce067 100644 --- a/src/languages/yaml.js +++ b/src/languages/yaml.js @@ -39,7 +39,7 @@ export default function(hljs) { variants: [ {begin: /'/, end: /'/}, {begin: /"/, end: /"/}, - {begin: /\S+/} + {begin: /[^\s,{}[\]]+/} ], contains: [ hljs.BACKSLASH_ESCAPE, diff --git a/test/markup/yaml/inline.expect.txt b/test/markup/yaml/inline.expect.txt new file mode 100644 index 0000000000..c2f1b443de --- /dev/null +++ b/test/markup/yaml/inline.expect.txt @@ -0,0 +1,2 @@ +foo: [bar, bar2, 1, 2] +foo: {bar: 1, bar: 2} diff --git a/test/markup/yaml/inline.txt b/test/markup/yaml/inline.txt new file mode 100644 index 0000000000..2eedc3d3d0 --- /dev/null +++ b/test/markup/yaml/inline.txt @@ -0,0 +1,2 @@ +foo: [bar, bar2, 1, 2] +foo: {bar: 1, bar: 2} From 348f83bad8c98ca48877cc20a532fb48afa51fa4 Mon Sep 17 00:00:00 2001 From: Peter Plantinga Date: Tue, 28 Apr 2020 18:29:44 -0400 Subject: [PATCH 2/7] Use containers to match inline sequences and mappings --- src/languages/yaml.js | 152 +++++++++++++++++++++++------------------- 1 file changed, 84 insertions(+), 68 deletions(-) diff --git a/src/languages/yaml.js b/src/languages/yaml.js index d523bce067..01d27b3328 100644 --- a/src/languages/yaml.js +++ b/src/languages/yaml.js @@ -39,7 +39,7 @@ export default function(hljs) { variants: [ {begin: /'/, end: /'/}, {begin: /"/, end: /"/}, - {begin: /[^\s,{}[\]]+/} + {begin: /\S+/} ], contains: [ hljs.BACKSLASH_ESCAPE, @@ -56,76 +56,92 @@ export default function(hljs) { begin: '\\b' + DATE_RE + TIME_RE + FRACTION_RE + ZONE_RE + '\\b', } + var TYPES = [ + KEY, + { + className: 'meta', + begin: '^---\s*$', + relevance: 10 + }, + { // multi line string + // Blocks start with a | or > followed by a newline + // + // Indentation of subsequent lines must be the same to + // be considered part of the block + className: 'string', + begin: '[\\|>]([0-9]?[+-])?[ ]*\\n( *)[\\S ]+\\n(\\2[\\S ]+\\n?)*', + }, + { // Ruby/Rails erb + begin: '<%[%=-]?', end: '[%-]?%>', + subLanguage: 'ruby', + excludeBegin: true, + excludeEnd: true, + relevance: 0 + }, + { // named tags + className: 'type', + begin: '!\\w+!' + URI_CHARACTERS, + }, + // https://yaml.org/spec/1.2/spec.html#id2784064 + { // verbatim tags + className: 'type', + begin: '!<' + URI_CHARACTERS + ">", + }, + { // primary tags + className: 'type', + begin: '!' + URI_CHARACTERS, + }, + { // secondary tags + className: 'type', + begin: '!!' + URI_CHARACTERS, + }, + { // fragment id &ref + className: 'meta', + begin: '&' + hljs.UNDERSCORE_IDENT_RE + '$', + }, + { // fragment reference *ref + className: 'meta', + begin: '\\*' + hljs.UNDERSCORE_IDENT_RE + '$' + }, + { // array listing + className: 'bullet', + // TODO: remove |$ hack when we have proper look-ahead support + begin: '\\-(?=[ ]|$)', + relevance: 0 + }, + hljs.HASH_COMMENT_MODE, + { + beginKeywords: LITERALS, + keywords: {literal: LITERALS} + }, + TIMESTAMP, + // numbers are any valid C-style number that + // sit isolated from other words + { + className: 'number', + begin: hljs.C_NUMBER_RE + '\\b' + }, + ]; + var VALUE_CONTAINER = { + end: ',', endsWithParent: true, excludeEnd: true, + contains: TYPES, + keywords: LITERALS + }; + var OBJECT = { + begin: '{', end: '}', + contains: [hljs.inherit(VALUE_CONTAINER)], + illegal: '\\n' + }; + var ARRAY = { + begin: '\\[', end: '\\]', + contains: [hljs.inherit(VALUE_CONTAINER)], + illegal: '\\n' + }; + TYPES.push(OBJECT, ARRAY, STRING); return { name: 'YAML', case_insensitive: true, aliases: ['yml', 'YAML'], - contains: [ - KEY, - { - className: 'meta', - begin: '^---\s*$', - relevance: 10 - }, - { // multi line string - // Blocks start with a | or > followed by a newline - // - // Indentation of subsequent lines must be the same to - // be considered part of the block - className: 'string', - begin: '[\\|>]([0-9]?[+-])?[ ]*\\n( *)[\\S ]+\\n(\\2[\\S ]+\\n?)*', - }, - { // Ruby/Rails erb - begin: '<%[%=-]?', end: '[%-]?%>', - subLanguage: 'ruby', - excludeBegin: true, - excludeEnd: true, - relevance: 0 - }, - { // named tags - className: 'type', - begin: '!\\w+!' + URI_CHARACTERS, - }, - // https://yaml.org/spec/1.2/spec.html#id2784064 - { // verbatim tags - className: 'type', - begin: '!<' + URI_CHARACTERS + ">", - }, - { // primary tags - className: 'type', - begin: '!' + URI_CHARACTERS, - }, - { // secondary tags - className: 'type', - begin: '!!' + URI_CHARACTERS, - }, - { // fragment id &ref - className: 'meta', - begin: '&' + hljs.UNDERSCORE_IDENT_RE + '$', - }, - { // fragment reference *ref - className: 'meta', - begin: '\\*' + hljs.UNDERSCORE_IDENT_RE + '$' - }, - { // array listing - className: 'bullet', - // TODO: remove |$ hack when we have proper look-ahead support - begin: '\\-(?=[ ]|$)', - relevance: 0 - }, - hljs.HASH_COMMENT_MODE, - { - beginKeywords: LITERALS, - keywords: {literal: LITERALS} - }, - TIMESTAMP, - // numbers are any valid C-style number that - // sit isolated from other words - { - className: 'number', - begin: hljs.C_NUMBER_RE + '\\b' - }, - STRING - ] + contains: TYPES, }; } From 795f41caa181a4c9153e12d2ef6d0ce4f13a2fcb Mon Sep 17 00:00:00 2001 From: Peter Plantinga Date: Tue, 28 Apr 2020 19:49:59 -0400 Subject: [PATCH 3/7] Add string type for inside inline elements --- src/languages/yaml.js | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/src/languages/yaml.js b/src/languages/yaml.js index 01d27b3328..c8d495aaec 100644 --- a/src/languages/yaml.js +++ b/src/languages/yaml.js @@ -46,6 +46,19 @@ export default function(hljs) { TEMPLATE_VARIABLES ] }; + var INSIDE_OBJECT_STRING = { + className: 'string', + relevance: 0, + variants: [ + {begin: /'/, end: /'/}, + {begin: /"/, end: /"/}, + {begin: /\S*[^\s,\]}]/} + ], + contains: [ + hljs.BACKSLASH_ESCAPE, + TEMPLATE_VARIABLES + ] + }; var DATE_RE = '[0-9]{4}(-[0-9][0-9]){0,2}'; var TIME_RE = '([Tt \\t][0-9][0-9]?(:[0-9][0-9]){2})?'; @@ -124,24 +137,27 @@ export default function(hljs) { ]; var VALUE_CONTAINER = { end: ',', endsWithParent: true, excludeEnd: true, - contains: TYPES, - keywords: LITERALS + contains: TYPES.concat([INSIDE_OBJECT_STRING]), + keywords: LITERALS, + relevance: 0 }; var OBJECT = { begin: '{', end: '}', contains: [hljs.inherit(VALUE_CONTAINER)], - illegal: '\\n' + illegal: '\\n', + relevance: 0 }; var ARRAY = { begin: '\\[', end: '\\]', contains: [hljs.inherit(VALUE_CONTAINER)], - illegal: '\\n' + illegal: '\\n', + relevance: 0 }; - TYPES.push(OBJECT, ARRAY, STRING); + TYPES.push(OBJECT, ARRAY); return { name: 'YAML', case_insensitive: true, aliases: ['yml', 'YAML'], - contains: TYPES, + contains: TYPES.concat([STRING]), }; } From 91c2c19206b534773dabb5faa97a852b291b256a Mon Sep 17 00:00:00 2001 From: Peter Plantinga Date: Wed, 29 Apr 2020 09:17:05 -0400 Subject: [PATCH 4/7] Handle nested inline sequences and mappings --- src/languages/yaml.js | 11 +++++++---- test/markup/yaml/inline.expect.txt | 5 +++-- test/markup/yaml/inline.txt | 5 +++-- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/src/languages/yaml.js b/src/languages/yaml.js index c8d495aaec..5911352228 100644 --- a/src/languages/yaml.js +++ b/src/languages/yaml.js @@ -46,13 +46,16 @@ export default function(hljs) { TEMPLATE_VARIABLES ] }; + + // Strings inside objects can't start with { or } or [ or ] or , + // and can't end with } or ] or , var INSIDE_OBJECT_STRING = { className: 'string', relevance: 0, variants: [ {begin: /'/, end: /'/}, {begin: /"/, end: /"/}, - {begin: /\S*[^\s,\]}]/} + {begin: /[^\s,{}[\]](\S*[^\s,\]}])?/} ], contains: [ hljs.BACKSLASH_ESCAPE, @@ -136,20 +139,20 @@ export default function(hljs) { }, ]; var VALUE_CONTAINER = { - end: ',', endsWithParent: true, excludeEnd: true, + end: ',', excludeEnd: true, contains: TYPES.concat([INSIDE_OBJECT_STRING]), keywords: LITERALS, relevance: 0 }; var OBJECT = { begin: '{', end: '}', - contains: [hljs.inherit(VALUE_CONTAINER)], + contains: [VALUE_CONTAINER], illegal: '\\n', relevance: 0 }; var ARRAY = { begin: '\\[', end: '\\]', - contains: [hljs.inherit(VALUE_CONTAINER)], + contains: [VALUE_CONTAINER], illegal: '\\n', relevance: 0 }; diff --git a/test/markup/yaml/inline.expect.txt b/test/markup/yaml/inline.expect.txt index c2f1b443de..d12626f0ae 100644 --- a/test/markup/yaml/inline.expect.txt +++ b/test/markup/yaml/inline.expect.txt @@ -1,2 +1,3 @@ -foo: [bar, bar2, 1, 2] -foo: {bar: 1, bar: 2} +foo: [bar, bar2, [1, 2], 3] +foo: {bar: [1, 2], baz: {inside: 3}} +foo: ba{}r,ba[]z diff --git a/test/markup/yaml/inline.txt b/test/markup/yaml/inline.txt index 2eedc3d3d0..8dfa15b2ea 100644 --- a/test/markup/yaml/inline.txt +++ b/test/markup/yaml/inline.txt @@ -1,2 +1,3 @@ -foo: [bar, bar2, 1, 2] -foo: {bar: 1, bar: 2} +foo: [bar, bar2, [1, 2], 3] +foo: {bar: [1, 2], baz: {inside: 3}} +foo: ba{}r,ba[]z From bd5142a272267d7beea56a9033f10c326898364b Mon Sep 17 00:00:00 2001 From: Peter Plantinga Date: Wed, 29 Apr 2020 15:53:37 -0400 Subject: [PATCH 5/7] Disallow all braces brackets and commas from strings inside inline mappings or sequences --- src/languages/yaml.js | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/languages/yaml.js b/src/languages/yaml.js index 5911352228..35bc2b7fff 100644 --- a/src/languages/yaml.js +++ b/src/languages/yaml.js @@ -47,15 +47,14 @@ export default function(hljs) { ] }; - // Strings inside objects can't start with { or } or [ or ] or , - // and can't end with } or ] or , + // Strings inside objects can't contain braces, brackets, or commas var INSIDE_OBJECT_STRING = { className: 'string', relevance: 0, variants: [ {begin: /'/, end: /'/}, {begin: /"/, end: /"/}, - {begin: /[^\s,{}[\]](\S*[^\s,\]}])?/} + {begin: /[^\s,{}[\]]+/} ], contains: [ hljs.BACKSLASH_ESCAPE, @@ -72,7 +71,7 @@ export default function(hljs) { begin: '\\b' + DATE_RE + TIME_RE + FRACTION_RE + ZONE_RE + '\\b', } - var TYPES = [ + var INSIDE_OBJECT_TYPES = [ KEY, { className: 'meta', @@ -139,8 +138,8 @@ export default function(hljs) { }, ]; var VALUE_CONTAINER = { - end: ',', excludeEnd: true, - contains: TYPES.concat([INSIDE_OBJECT_STRING]), + end: ',', endsWithParent: true, excludeEnd: true, + contains: INSIDE_OBJECT_TYPES, keywords: LITERALS, relevance: 0 }; @@ -156,11 +155,15 @@ export default function(hljs) { illegal: '\\n', relevance: 0 }; - TYPES.push(OBJECT, ARRAY); + INSIDE_OBJECT_TYPES.push(OBJECT, ARRAY, INSIDE_OBJECT_STRING); + + // Exclude the INSIDE_OBJECT_STRING from TYPES + TYPES = INSIDE_OBJECT_TYPES.slice(0, -1) + TYPES.push(STRING); return { name: 'YAML', case_insensitive: true, aliases: ['yml', 'YAML'], - contains: TYPES.concat([STRING]), + contains: TYPES, }; } From dbf6d86887069cf9e46a4afe8e51ee2b69e88a3d Mon Sep 17 00:00:00 2001 From: Josh Goebel Date: Thu, 30 Apr 2020 10:54:14 -0400 Subject: [PATCH 6/7] clean up implementation --- src/languages/yaml.js | 74 ++++++++++++++++++++++--------------------- 1 file changed, 38 insertions(+), 36 deletions(-) diff --git a/src/languages/yaml.js b/src/languages/yaml.js index 35bc2b7fff..be513da783 100644 --- a/src/languages/yaml.js +++ b/src/languages/yaml.js @@ -47,20 +47,15 @@ export default function(hljs) { ] }; - // Strings inside objects can't contain braces, brackets, or commas - var INSIDE_OBJECT_STRING = { - className: 'string', - relevance: 0, + // Strings inside of value containers (objects) can't contain braces, + // brackets, or commas + var CONTAINER_STRING = hljs.inherit(STRING, { variants: [ {begin: /'/, end: /'/}, {begin: /"/, end: /"/}, {begin: /[^\s,{}[\]]+/} - ], - contains: [ - hljs.BACKSLASH_ESCAPE, - TEMPLATE_VARIABLES ] - }; + }); var DATE_RE = '[0-9]{4}(-[0-9][0-9]){0,2}'; var TIME_RE = '([Tt \\t][0-9][0-9]?(:[0-9][0-9]){2})?'; @@ -69,9 +64,30 @@ export default function(hljs) { var TIMESTAMP = { className: 'number', begin: '\\b' + DATE_RE + TIME_RE + FRACTION_RE + ZONE_RE + '\\b', - } + }; + + var VALUE_CONTAINER = { + end: ',', + endsWithParent: true, + excludeEnd: true, + contains: [], + keywords: LITERALS, + relevance: 0 + }; + var OBJECT = { + begin: '{', end: '}', + contains: [VALUE_CONTAINER], + illegal: '\\n', + relevance: 0 + }; + var ARRAY = { + begin: '\\[', end: '\\]', + contains: [VALUE_CONTAINER], + illegal: '\\n', + relevance: 0 + }; - var INSIDE_OBJECT_TYPES = [ + var MODES = [ KEY, { className: 'meta', @@ -120,8 +136,8 @@ export default function(hljs) { }, { // array listing className: 'bullet', - // TODO: remove |$ hack when we have proper look-ahead support - begin: '\\-(?=[ ]|$)', + // TODO: remove |$ hack when we have proper look-ahead support + begin: '\\-(?=[ ]|$)', relevance: 0 }, hljs.HASH_COMMENT_MODE, @@ -136,34 +152,20 @@ export default function(hljs) { className: 'number', begin: hljs.C_NUMBER_RE + '\\b' }, + OBJECT, + ARRAY, + STRING ]; - var VALUE_CONTAINER = { - end: ',', endsWithParent: true, excludeEnd: true, - contains: INSIDE_OBJECT_TYPES, - keywords: LITERALS, - relevance: 0 - }; - var OBJECT = { - begin: '{', end: '}', - contains: [VALUE_CONTAINER], - illegal: '\\n', - relevance: 0 - }; - var ARRAY = { - begin: '\\[', end: '\\]', - contains: [VALUE_CONTAINER], - illegal: '\\n', - relevance: 0 - }; - INSIDE_OBJECT_TYPES.push(OBJECT, ARRAY, INSIDE_OBJECT_STRING); - // Exclude the INSIDE_OBJECT_STRING from TYPES - TYPES = INSIDE_OBJECT_TYPES.slice(0, -1) - TYPES.push(STRING); + var VALUE_MODES = [...MODES]; + VALUE_MODES.pop(); + VALUE_MODES.push(CONTAINER_STRING); + VALUE_CONTAINER.contains = VALUE_MODES; + return { name: 'YAML', case_insensitive: true, aliases: ['yml', 'YAML'], - contains: TYPES, + contains: MODES, }; } From ebb47dfa22d5beae76a6fb0b542fe39f44fc02c8 Mon Sep 17 00:00:00 2001 From: Josh Goebel Date: Thu, 30 Apr 2020 10:59:34 -0400 Subject: [PATCH 7/7] feed the linter --- src/languages/yaml.js | 49 +++++++++++++++++++++++-------------------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/src/languages/yaml.js b/src/languages/yaml.js index be513da783..02a1d9cee2 100644 --- a/src/languages/yaml.js +++ b/src/languages/yaml.js @@ -11,7 +11,7 @@ export default function(hljs) { var LITERALS = 'true false yes no null'; // YAML spec allows non-reserved URI characters in tags. - var URI_CHARACTERS = '[\\w#;/?:@&=+$,.~*\\\'()[\\]]+' + var URI_CHARACTERS = '[\\w#;/?:@&=+$,.~*\\\'()[\\]]+'; // Define keys as starting with a word character // ...containing word chars, spaces, colons, forward-slashes, hyphens and periods @@ -21,25 +21,25 @@ export default function(hljs) { className: 'attr', variants: [ { begin: '\\w[\\w :\\/.-]*:(?=[ \t]|$)' }, - { begin: '"\\w[\\w :\\/.-]*":(?=[ \t]|$)' }, //double quoted keys - { begin: '\'\\w[\\w :\\/.-]*\':(?=[ \t]|$)' } //single quoted keys + { begin: '"\\w[\\w :\\/.-]*":(?=[ \t]|$)' }, // double quoted keys + { begin: '\'\\w[\\w :\\/.-]*\':(?=[ \t]|$)' } // single quoted keys ] }; var TEMPLATE_VARIABLES = { className: 'template-variable', variants: [ - { begin: '\{\{', end: '\}\}' }, // jinja templates Ansible - { begin: '%\{', end: '\}' } // Ruby i18n + { begin: '{{', end: '}}' }, // jinja templates Ansible + { begin: '%{', end: '}' } // Ruby i18n ] }; var STRING = { className: 'string', relevance: 0, variants: [ - {begin: /'/, end: /'/}, - {begin: /"/, end: /"/}, - {begin: /\S+/} + { begin: /'/, end: /'/ }, + { begin: /"/, end: /"/ }, + { begin: /\S+/ } ], contains: [ hljs.BACKSLASH_ESCAPE, @@ -51,9 +51,9 @@ export default function(hljs) { // brackets, or commas var CONTAINER_STRING = hljs.inherit(STRING, { variants: [ - {begin: /'/, end: /'/}, - {begin: /"/, end: /"/}, - {begin: /[^\s,{}[\]]+/} + { begin: /'/, end: /'/ }, + { begin: /"/, end: /"/ }, + { begin: /[^\s,{}[\]]+/ } ] }); @@ -63,7 +63,7 @@ export default function(hljs) { var ZONE_RE = '([ \\t])*(Z|[-+][0-9][0-9]?(:[0-9][0-9])?)?'; var TIMESTAMP = { className: 'number', - begin: '\\b' + DATE_RE + TIME_RE + FRACTION_RE + ZONE_RE + '\\b', + begin: '\\b' + DATE_RE + TIME_RE + FRACTION_RE + ZONE_RE + '\\b' }; var VALUE_CONTAINER = { @@ -75,13 +75,15 @@ export default function(hljs) { relevance: 0 }; var OBJECT = { - begin: '{', end: '}', + begin: '{', + end: '}', contains: [VALUE_CONTAINER], illegal: '\\n', relevance: 0 }; var ARRAY = { - begin: '\\[', end: '\\]', + begin: '\\[', + end: '\\]', contains: [VALUE_CONTAINER], illegal: '\\n', relevance: 0 @@ -100,10 +102,11 @@ export default function(hljs) { // Indentation of subsequent lines must be the same to // be considered part of the block className: 'string', - begin: '[\\|>]([0-9]?[+-])?[ ]*\\n( *)[\\S ]+\\n(\\2[\\S ]+\\n?)*', + begin: '[\\|>]([0-9]?[+-])?[ ]*\\n( *)[\\S ]+\\n(\\2[\\S ]+\\n?)*' }, { // Ruby/Rails erb - begin: '<%[%=-]?', end: '[%-]?%>', + begin: '<%[%=-]?', + end: '[%-]?%>', subLanguage: 'ruby', excludeBegin: true, excludeEnd: true, @@ -111,24 +114,24 @@ export default function(hljs) { }, { // named tags className: 'type', - begin: '!\\w+!' + URI_CHARACTERS, + begin: '!\\w+!' + URI_CHARACTERS }, // https://yaml.org/spec/1.2/spec.html#id2784064 { // verbatim tags className: 'type', - begin: '!<' + URI_CHARACTERS + ">", + begin: '!<' + URI_CHARACTERS + ">" }, { // primary tags className: 'type', - begin: '!' + URI_CHARACTERS, + begin: '!' + URI_CHARACTERS }, { // secondary tags className: 'type', - begin: '!!' + URI_CHARACTERS, + begin: '!!' + URI_CHARACTERS }, { // fragment id &ref className: 'meta', - begin: '&' + hljs.UNDERSCORE_IDENT_RE + '$', + begin: '&' + hljs.UNDERSCORE_IDENT_RE + '$' }, { // fragment reference *ref className: 'meta', @@ -143,7 +146,7 @@ export default function(hljs) { hljs.HASH_COMMENT_MODE, { beginKeywords: LITERALS, - keywords: {literal: LITERALS} + keywords: { literal: LITERALS } }, TIMESTAMP, // numbers are any valid C-style number that @@ -166,6 +169,6 @@ export default function(hljs) { name: 'YAML', case_insensitive: true, aliases: ['yml', 'YAML'], - contains: MODES, + contains: MODES }; }