Skip to content
Permalink

Comparing changes

Choose two branches to see what’s changed or to start a new pull request. If you need to, you can also or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: ocaml-community/yojson
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: 2.2.0
Choose a base ref
...
head repository: ocaml-community/yojson
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: master
Choose a head ref

Commits on May 31, 2024

  1. Copy the full SHA
    c7343a1 View commit details
  2. Copy the full SHA
    dec49cd View commit details
  3. Copy the full SHA
    94b85f4 View commit details
  4. Copy the full SHA
    1f6e58f View commit details
  5. Copy the full SHA
    0bb7265 View commit details
  6. Remove unused cases

    Thanks to @hhugo for pointing them out
    Leonidas-from-XIV committed May 31, 2024
    Copy the full SHA
    73a5166 View commit details
  7. Remove dead code

    This seems to be unused in Yojson and Atd
    Leonidas-from-XIV committed May 31, 2024
    Copy the full SHA
    8bbbc1f View commit details
  8. Copy the full SHA
    c90bd36 View commit details

Commits on Jun 3, 2024

  1. Merge pull request #178 from Leonidas-from-XIV/proper-dependency

    Add proper dependency from `yojson-five` to `yojson`
    Leonidas-from-XIV authored Jun 3, 2024
    Copy the full SHA
    efa855c View commit details
  2. Merge pull request #179 from Leonidas-from-XIV/crlf-test

    Add test for handling CRLF line endings
    Leonidas-from-XIV authored Jun 3, 2024
    Copy the full SHA
    fd75444 View commit details
  3. Dont expose JSON5 internals (#180)

    The rename of the package forgot to rename this file, thus it exposed
    more than it should.
    
    Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
    Leonidas-from-XIV and github-actions[bot] authored Jun 3, 2024
    Copy the full SHA
    e201639 View commit details

Commits on Jun 4, 2024

  1. Copy the full SHA
    1b7083f View commit details
  2. Remove map_string from API

    Its not tested anymore and unused by Yojson
    Leonidas-from-XIV committed Jun 4, 2024
    Copy the full SHA
    d441f4f View commit details
  3. Release 2.2.1

    Leonidas-from-XIV committed Jun 4, 2024
    Copy the full SHA
    a9c234f View commit details

Commits on Jun 7, 2024

  1. Improve the error messages displayed by the JSON5 lexer and parser.

    * Display the line number in the error message
    * Try to harmonize the error messages with the ones displayed by
      the classic yojson parser.
    gcluzel committed Jun 7, 2024
    Copy the full SHA
    b9a1530 View commit details

Commits on Jun 8, 2024

  1. Fix the changelog

    gcluzel committed Jun 8, 2024
    Copy the full SHA
    6401197 View commit details
  2. Copy the full SHA
    d324654 View commit details

Commits on Jun 9, 2024

  1. Copy the full SHA
    4796445 View commit details

Commits on Jun 27, 2024

  1. Copy the full SHA
    08b564b View commit details
  2. Copy the full SHA
    56bc160 View commit details
  3. Copy the full SHA
    7261a28 View commit details
  4. Copy the full SHA
    8599b03 View commit details
  5. Copy the full SHA
    2f2f0a7 View commit details
  6. Copy the full SHA
    1f516e8 View commit details
  7. Merge pull request #182 from gcluzel/better-error-messages

    Improve the error messages displayed by the JSON5 lexer and parser.
    Leonidas-from-XIV authored Jun 27, 2024
    Copy the full SHA
    93f3496 View commit details
  8. Release 2.2.2

    Leonidas-from-XIV committed Jun 27, 2024
    Copy the full SHA
    3f82b79 View commit details

Commits on Jun 28, 2024

  1. Copy the full SHA
    70bad75 View commit details
  2. Copy the full SHA
    ce6e250 View commit details
34 changes: 34 additions & 0 deletions CHANGES.md
Original file line number Diff line number Diff line change
@@ -1,3 +1,37 @@
## unreleased

### Added

### Changed

### Deprecated

### Fixed

### Removed

- Removed support for Tuple and Variant in JSON. It was a non-standard
extension that was rarely used, so this simplifies the Yojson types and the
parser more standard-conforming (#105, #158 @Leonidas-from-XIV)

### Security

## 2.2.2

*2024-06-27*

### Added

- Add locations in the JSON5 parser error messages (@gcluzel, #182)

## 2.2.1

*2024-06-04*

### Fixed

- Don't expose `Yojson_five` internals anymore (@Leonidas_from_XIV, #180)

## 2.2.0

*2024-05-31*
4 changes: 3 additions & 1 deletion dune-project
Original file line number Diff line number Diff line change
@@ -7,7 +7,6 @@

(maintainers "paul-elliot@tarides.com" "nathan@tarides.com" "marek@tarides.com")
(authors "Martin Jambon")
(documentation "https://ocaml-community.github.io/yojson")

(package
(name yojson)
@@ -16,6 +15,7 @@
ydump is a pretty-printing command-line program provided with the
yojson package.")
(documentation "https://ocaml.org/p/yojson/latest")
(depends
(ocaml (>= 4.02.3))
(alcotest (and :with-test (>= 0.8.5)))
@@ -26,9 +26,11 @@ yojson package.")
(synopsis "Yojson-five is a parsing and printing library for the JSON5 format")
(description "Yojson-five is a parsing and printing library for the JSON5 format.
It supports parsing JSON5 to Yojson.Basic.t and Yojson.Safe.t types.")
(documentation "https://ocaml.org/p/yojson-five/latest")
(depends
(ocaml (>= 4.08))
(sedlex (>= 2.5))
(yojson (= :version))
(alcotest (and :with-test (>= 0.8.5)))))

(package
1 change: 0 additions & 1 deletion lib/common.ml
Original file line number Diff line number Diff line change
@@ -6,7 +6,6 @@ let json_error s = raise (Json_error s)

exception End_of_array
exception End_of_object
exception End_of_tuple
exception End_of_input

type lexer_state = {
1 change: 0 additions & 1 deletion lib/common.mli
Original file line number Diff line number Diff line change
@@ -36,7 +36,6 @@ val init_lexer :

exception End_of_array
exception End_of_object
exception End_of_tuple
exception End_of_input

(* end undocumented section *)
4 changes: 4 additions & 0 deletions lib/json5/errors.ml
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
let string_of_position { Lexing.pos_lnum; pos_fname; _ } =
match pos_fname with
| "" -> Printf.sprintf "Line %d" pos_lnum
| fname -> Printf.sprintf "File %s, line %d" fname pos_lnum
4 changes: 4 additions & 0 deletions lib/json5/errors.mli
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
val string_of_position : Lexing.position -> string
(** [string_of_position pos] returns a string that contains the line and, if
supplied, the filename of the position in a way that's appropriate to include
in an error message *)
107 changes: 57 additions & 50 deletions lib/json5/lexer.ml
Original file line number Diff line number Diff line change
@@ -9,7 +9,6 @@ type token =
| CLOSE_BRACKET
| COLON
| COMMA
| COMMENT of string
| TRUE
| FALSE
| NULL
@@ -18,25 +17,38 @@ type token =
| INT of string
| STRING of string
| IDENTIFIER_NAME of string

let pp_token ppf = function
| OPEN_PAREN -> Format.fprintf ppf "'('"
| CLOSE_PAREN -> Format.fprintf ppf "')'"
| OPEN_BRACE -> Format.fprintf ppf "'{'"
| CLOSE_BRACE -> Format.fprintf ppf "'}'"
| OPEN_BRACKET -> Format.fprintf ppf "'['"
| CLOSE_BRACKET -> Format.fprintf ppf "']'"
| COLON -> Format.fprintf ppf "':'"
| COMMA -> Format.fprintf ppf "','"
| COMMENT s -> Format.fprintf ppf "COMMENT '%s'" s
| TRUE -> Format.fprintf ppf "'true'"
| FALSE -> Format.fprintf ppf "'false'"
| NULL -> Format.fprintf ppf "'null'"
| FLOAT s -> Format.fprintf ppf "FLOAT '%s'" s
| INT_OR_FLOAT s -> Format.fprintf ppf "INT_OR_FLOAT '%s'" s
| INT s -> Format.fprintf ppf "INT '%s'" s
| STRING s -> Format.fprintf ppf "STRING '%s'" s
| IDENTIFIER_NAME s -> Format.fprintf ppf "IDENTIFIER_NAME '%s'" s
| EOF

let pp_token ppf =
let ps = Format.pp_print_string ppf in
let pf = Format.fprintf ppf in
function
| OPEN_PAREN -> ps "'('"
| CLOSE_PAREN -> ps "')'"
| OPEN_BRACE -> ps "'{'"
| CLOSE_BRACE -> ps "'}'"
| OPEN_BRACKET -> ps "'['"
| CLOSE_BRACKET -> ps "']'"
| COLON -> ps "':'"
| COMMA -> ps "','"
| TRUE -> ps "'true'"
| FALSE -> ps "'false'"
| NULL -> ps "'null'"
| FLOAT s -> pf "FLOAT %S" s
| INT_OR_FLOAT s -> pf "INT_OR_STRING %S" s
| INT s -> pf "INT %S" s
| STRING s -> pf "%S" s
| IDENTIFIER_NAME s -> pf "IDENTIFIER_NAME %S" s
| EOF -> ps "EOF"

let lexer_error lexbuf =
let pos_start, _pos_end = Sedlexing.lexing_positions lexbuf in
let location = Errors.string_of_position pos_start in
let msg =
Printf.sprintf "%s: Unexpected character '%s'" location
(Sedlexing.Utf8.lexeme lexbuf)
in
Error msg

let source_character = [%sedlex.regexp? any]
let line_terminator = [%sedlex.regexp? 0x000A | 0x000D | 0x2028 | 0x2029]
@@ -182,10 +194,7 @@ let string_lex_single lexbuf strbuf =
| Sub (source_character, ('\'' | line_terminator)) ->
Buffer.add_string strbuf (lexeme lexbuf);
lex lexbuf strbuf
| _ ->
lexeme lexbuf
|> Format.sprintf "Unexpected character: %s"
|> Result.error
| _ -> lexer_error lexbuf
in
lex lexbuf strbuf

@@ -202,50 +211,48 @@ let string_lex_double lexbuf strbuf =
| Sub (source_character, ('"' | line_terminator)) ->
Buffer.add_string strbuf (lexeme lexbuf);
lex lexbuf strbuf
| _ ->
lexeme lexbuf
|> Format.sprintf "Unexpected character: %s"
|> Result.error
| _ -> lexer_error lexbuf
in
lex lexbuf strbuf

let string_lex lexbuf quote =
let strbuf = Buffer.create 200 in
if quote = "'" then string_lex_single lexbuf strbuf
else if quote = {|"|} then string_lex_double lexbuf strbuf
else Error (Format.sprintf "Invalid string quote %S" quote)
match quote with
| "'" -> string_lex_single lexbuf strbuf
| {|"|} -> string_lex_double lexbuf strbuf
| _ -> Error (Printf.sprintf "Invalid string quote %S" quote)

let rec lex tokens buf =
let lexeme = Sedlexing.Utf8.lexeme in
let pos, _ = Sedlexing.lexing_positions buf in
match%sedlex buf with
| '(' -> lex (OPEN_PAREN :: tokens) buf
| ')' -> lex (CLOSE_PAREN :: tokens) buf
| '{' -> lex (OPEN_BRACE :: tokens) buf
| '}' -> lex (CLOSE_BRACE :: tokens) buf
| '[' -> lex (OPEN_BRACKET :: tokens) buf
| ']' -> lex (CLOSE_BRACKET :: tokens) buf
| ':' -> lex (COLON :: tokens) buf
| ',' -> lex (COMMA :: tokens) buf
| '(' -> lex ((OPEN_PAREN, pos) :: tokens) buf
| ')' -> lex ((CLOSE_PAREN, pos) :: tokens) buf
| '{' -> lex ((OPEN_BRACE, pos) :: tokens) buf
| '}' -> lex ((CLOSE_BRACE, pos) :: tokens) buf
| '[' -> lex ((OPEN_BRACKET, pos) :: tokens) buf
| ']' -> lex ((CLOSE_BRACKET, pos) :: tokens) buf
| ':' -> lex ((COLON, pos) :: tokens) buf
| ',' -> lex ((COMMA, pos) :: tokens) buf
| Chars {|"'|} ->
let* s = string_lex buf (lexeme buf) in
lex (STRING s :: tokens) buf
lex ((STRING s, pos) :: tokens) buf
| multi_line_comment | single_line_comment | white_space | line_terminator ->
lex tokens buf
| "true" -> lex (TRUE :: tokens) buf
| "false" -> lex (FALSE :: tokens) buf
| "null" -> lex (NULL :: tokens) buf
| "true" -> lex ((TRUE, pos) :: tokens) buf
| "false" -> lex ((FALSE, pos) :: tokens) buf
| "null" -> lex ((NULL, pos) :: tokens) buf
| json5_float ->
let s = lexeme buf in
lex (FLOAT s :: tokens) buf
lex ((FLOAT s, pos) :: tokens) buf
| json5_int ->
let s = lexeme buf in
lex (INT s :: tokens) buf
lex ((INT s, pos) :: tokens) buf
| json5_int_or_float ->
let s = lexeme buf in
lex (INT_OR_FLOAT s :: tokens) buf
lex ((INT_OR_FLOAT s, pos) :: tokens) buf
| identifier_name ->
let s = lexeme buf in
lex (IDENTIFIER_NAME s :: tokens) buf
| eof -> Ok (List.rev tokens)
| _ ->
lexeme buf |> Format.asprintf "Unexpected character: '%s'" |> Result.error
lex ((IDENTIFIER_NAME s, pos) :: tokens) buf
| eof -> Ok (List.rev ((EOF, pos) :: tokens))
| _ -> lexer_error buf
77 changes: 49 additions & 28 deletions lib/json5/parser.ml
Original file line number Diff line number Diff line change
@@ -1,46 +1,69 @@
open Let_syntax.Result

let parser_error pos error =
let location = Errors.string_of_position pos in
let msg = Printf.sprintf "%s: %s" location error in
Error msg

let rec parse_list acc = function
| [] -> Error "List never ends"
| Lexer.CLOSE_BRACKET :: xs -> Ok (acc, xs)
| [] -> Error "Unexpected end of input"
| [ (Lexer.EOF, pos) ] -> parser_error pos "Unexpected end of input"
| (Lexer.CLOSE_BRACKET, _) :: xs -> Ok (acc, xs)
| xs -> (
let* v, xs = parse xs in
match xs with
| [] -> Error "List was not closed"
| Lexer.CLOSE_BRACKET :: xs | COMMA :: CLOSE_BRACKET :: xs ->
| [] -> Error "Unexpected end of input"
| [ (Lexer.EOF, pos) ] -> parser_error pos "Unexpected end of input"
| (Lexer.CLOSE_BRACKET, _) :: xs | (COMMA, _) :: (CLOSE_BRACKET, _) :: xs
->
Ok (v :: acc, xs)
| COMMA :: xs -> parse_list (v :: acc) xs
| x :: _ ->
| (COMMA, _) :: xs -> parse_list (v :: acc) xs
| (x, pos) :: _ ->
let s =
Format.asprintf "Unexpected list token: %a" Lexer.pp_token x
in
Error s)
parser_error pos s)

and parse_assoc acc = function
| [] -> Error "Assoc never ends"
| Lexer.CLOSE_BRACE :: xs -> Ok (acc, xs)
| STRING k :: COLON :: xs | IDENTIFIER_NAME k :: COLON :: xs -> (
let* v, xs = parse xs in
let item = (k, v) in
| [] -> Error "Unexpected end of input"
| [ (Lexer.EOF, pos) ] -> parser_error pos "Unexpected end of input"
| (CLOSE_BRACE, _) :: xs -> Ok (acc, xs)
| (STRING k, _) :: xs | (IDENTIFIER_NAME k, _) :: xs -> (
match xs with
| [] -> Error "Object was not closed"
| Lexer.CLOSE_BRACE :: xs | COMMA :: CLOSE_BRACE :: xs ->
Ok (item :: acc, xs)
| COMMA :: xs -> parse_assoc (item :: acc) xs
| x :: _ ->
| [] -> Error "Unexpected end of input"
| [ (Lexer.EOF, pos) ] -> parser_error pos "Unexpected end of input"
| (Lexer.COLON, _) :: xs -> (
let* v, xs = parse xs in
let item = (k, v) in
match xs with
| [] -> Error "Unexpected end of input"
| [ (Lexer.EOF, pos) ] -> parser_error pos "Unexpected end of input"
| (CLOSE_BRACE, _) :: xs | (COMMA, _) :: (CLOSE_BRACE, _) :: xs ->
Ok (item :: acc, xs)
| (COMMA, _) :: xs -> parse_assoc (item :: acc) xs
| (x, pos) :: _ ->
let s =
Format.asprintf "Unexpected assoc list token: %a" Lexer.pp_token
x
in
parser_error pos s)
| (x, pos) :: _ ->
let s =
Format.asprintf "Unexpected assoc list token: %a" Lexer.pp_token x
Format.asprintf "Expected %a but found %a" Lexer.pp_token
Lexer.COLON Lexer.pp_token x
in
Error s)
| x :: _ ->
parser_error pos s)
| (x, pos) :: _ ->
let s =
Format.asprintf "Unexpected assoc list token: %a" Lexer.pp_token x
Format.asprintf "Expected string or identifier but found %a"
Lexer.pp_token x
in
Error s
parser_error pos s

and parse = function
| [] -> Error "empty list of tokens"
| token :: xs -> (
| [] -> Error "Unexpected end of input"
| [ (Lexer.EOF, pos) ] -> parser_error pos "Unexpected end of input"
| (token, pos) :: xs -> (
match token with
| TRUE -> Ok (Ast.Bool true, xs)
| FALSE -> Ok (Bool false, xs)
@@ -57,12 +80,10 @@ and parse = function
(Ast.Assoc (List.rev a), xs)
| x ->
let s = Format.asprintf "Unexpected token: %a" Lexer.pp_token x in
Error s)
parser_error pos s)

let parse_from_lexbuf ?fname ?lnum lexbuffer =
let fname = Option.value fname ~default:"" in
let parse_from_lexbuf ?(fname = "") ?(lnum = 1) lexbuffer =
Sedlexing.set_filename lexbuffer fname;
let lnum = Option.value lnum ~default:1 in
let pos =
{ Lexing.pos_fname = fname; pos_lnum = lnum; pos_bol = 0; pos_cnum = 0 }
in
File renamed without changes.
File renamed without changes.
Loading