From d0f616802e1636536a79189bbe50f3398cc25bdd Mon Sep 17 00:00:00 2001 From: Jeanine Adkisson Date: Sun, 10 Nov 2024 20:44:19 -0500 Subject: [PATCH] implement a json5 lexer (#1561) * implement a json5 lexer * add a desc * Remove unnecessary rules * Mirror pattern from Pygments JSON5 * Document JSON5 support --------- Co-authored-by: http://jneen.net/ Co-authored-by: Michael Camilleri Co-authored-by: Tan Le --- docs/Languages.md | 1 + lib/rouge/demos/json5 | 12 +++++++ lib/rouge/lexers/json5.rb | 74 +++++++++++++++++++++++++++++++++++++++ spec/lexers/json5_spec.rb | 18 ++++++++++ spec/visual/samples/json5 | 44 +++++++++++++++++++++++ 5 files changed, 149 insertions(+) create mode 100644 lib/rouge/demos/json5 create mode 100644 lib/rouge/lexers/json5.rb create mode 100644 spec/lexers/json5_spec.rb create mode 100644 spec/visual/samples/json5 diff --git a/docs/Languages.md b/docs/Languages.md index 18864ebe0a..4818b54e62 100644 --- a/docs/Languages.md +++ b/docs/Languages.md @@ -103,6 +103,7 @@ - Jinja (`jinja`) - JSL (`jsl`) - JSON (`json`) +- JSON5 (`json5`) - Json-doc (`json-doc`) - Jsonnet (`jsonnet`) - Jsp (`jsp`) diff --git a/lib/rouge/demos/json5 b/lib/rouge/demos/json5 new file mode 100644 index 0000000000..0e72c64622 --- /dev/null +++ b/lib/rouge/demos/json5 @@ -0,0 +1,12 @@ +{ + // comments + unquoted: 'and you can quote me on that', + singleQuotes: 'I can use "double quotes" here', + lineBreaks: "Look, Mom! \ +No \\n's!", + hexadecimal: 0xdecaf, + leadingDecimalPoint: .8675309, andTrailing: 8675309., + positiveSign: +1, + trailingComma: 'in objects', andIn: ['arrays',], + "backwardsCompatible": "with JSON", +} diff --git a/lib/rouge/lexers/json5.rb b/lib/rouge/lexers/json5.rb new file mode 100644 index 0000000000..99185cf70c --- /dev/null +++ b/lib/rouge/lexers/json5.rb @@ -0,0 +1,74 @@ +# -*- coding: utf-8 -*- # +# frozen_string_literal: true + +module Rouge + module Lexers + load_lexer 'javascript.rb' + + class JSON5 < JSON + title 'JSON 5' + tag 'json5' + filenames '*.json5' + mimetypes 'application/json5', 'application/x-json5' + + desc 'JSON 5 extension for JSON (json5.org)' + + append :whitespace do + rule %r://.*$:, Comment + + # comments are non-nesting, so a single regex will do + rule %r:/[*].*?[*]/:m, Comment + end + + prepend :name do + rule Javascript.id_regex, Name::Label + rule %r/".*?"/, Name::Label + end + + # comments can appear between keys and :, so we have to + # manage our states a little more carefully + append :object do + rule %r/:/ do + token Punctuation + goto :object_value + end + end + + state :object_value do + mixin :value + rule %r/,/ do + token Punctuation + goto :object + end + + rule %r/}/, Punctuation, :pop! + end + + append :value do + rule %r/'/, Str::Single, :sstring + end + + state :sstring do + rule %r/[^\\']+/, Str::Single + rule %r/\\./m, Str::Escape + rule %r/'/, Str::Single, :pop! + end + + # can escape newlines + append :string do + rule %r/\\./m, Str::Escape + end + + # override: numbers are very different in json5 + state :constants do + rule %r/\b(?:true|false|null)\b/, Keyword::Constant + rule %r/[+-]?\b(?:Infinity|NaN)\b/, Keyword::Constant + rule %r/[+-]?0x\h+/i, Num::Hex + + rule %r/[+-.]?[0-9]+[.]?[0-9]?([eE][-]?[0-9]+)?/i, Num::Float + rule %r/[+-]?\d+e[+-]?\d+/, Num::Integer + rule %r/[+-]?(?:0|[1-9]\d*)(?:e[+-]?\d+)?/i, Num::Integer + end + end + end +end diff --git a/spec/lexers/json5_spec.rb b/spec/lexers/json5_spec.rb new file mode 100644 index 0000000000..44feb7943e --- /dev/null +++ b/spec/lexers/json5_spec.rb @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- # +# frozen_string_literal: true + +describe Rouge::Lexers::JSON5 do + let(:subject) { Rouge::Lexers::JSON5.new } + + describe 'guessing' do + include Support::Guessing + + it 'guesses by filename' do + assert_guess :filename => 'foo.json5' + end + + it 'guesses by mimetype' do + assert_guess :mimetype => 'application/json5' + end + end +end diff --git a/spec/visual/samples/json5 b/spec/visual/samples/json5 new file mode 100644 index 0000000000..90c725d32e --- /dev/null +++ b/spec/visual/samples/json5 @@ -0,0 +1,44 @@ +{ + // comments + unquoted: 'and you can quote me on that', + singleQuotes: 'I can use "double quotes" here', + lineBreaks: "Look, Mom! \ +No \\n's!", + hexadecimal: 0xdecaf, + leadingDecimalPoint: .8675309, andTrailing: 8675309., + positiveSign: +1, + trailingComma: 'in objects', andIn: ['arrays',], + "backwardsCompatible": "with JSON", + + + a: 1e5, + b: 1.e-5, + c: .1e5, + + /** + * multiline + * comments + */ + + d + : 3, + + "e" + // comment + : 4, + + foo: "bar", + "bar": "foo", + + f //comment + : 5, + + constants: [ + NaN, Infinity, true, false, null, + { NaN: NaN, + Infinity: Infinity, + true: true, + false: false, + null: null }, + ] +}