Fix whitespace conformance to match UAX31 (including permitting LRM/RLM)

spec/message.abnf

@@ -5,41 +5,41 @@ simple-start      = simple-start-char / text-escape / placeholder
 pattern           = *(text-char / text-escape / placeholder)
 placeholder       = expression / markup
 
-complex-message   = *(declaration [s]) complex-body
+complex-message   = *(declaration owsp) complex-body
 declaration       = input-declaration / local-declaration / reserved-statement
 complex-body      = quoted-pattern / matcher
 
-input-declaration = input [s] variable-expression
-local-declaration = local s variable [s] "=" [s] expression
+input-declaration = input owsp variable-expression
+local-declaration = local s variable owsp "=" owsp expression
 
 quoted-pattern    = "{{" pattern "}}"
 
-matcher           = match-statement 1*([s] variant)
-match-statement   = match 1*([s] selector)
+matcher           = match-statement 1*(owsp variant)
+match-statement   = match 1*(owsp selector)
 selector          = expression
-variant           = key *(s key) [s] quoted-pattern
+variant           = key *(wsp key) owsp quoted-pattern
 key               = literal / "*"
 
 ; Expressions
 expression            = literal-expression
                       / variable-expression
                       / annotation-expression
-literal-expression    = "{" [s] literal [s annotation] *(s attribute) [s] "}"
-variable-expression   = "{" [s] variable [s annotation] *(s attribute) [s] "}"
-annotation-expression = "{" [s] annotation *(s attribute) [s] "}"
+literal-expression    = "{" owsp literal [s annotation] *(wsp attribute) owsp "}"
+variable-expression   = "{" owsp variable [s annotation] *(wsp attribute) owsp "}"
+annotation-expression = "{" owsp annotation *(wsp attribute) owsp "}"
 
 annotation            = function
                       / private-use-annotation
                       / reserved-annotation
 
-markup = "{" [s] "#" identifier *(s option) *(s attribute) [s] ["/"] "}"  ; open and standalone
-       / "{" [s] "/" identifier *(s option) *(s attribute) [s] "}"  ; close
+markup = "{" owsp "#" identifier *(wsp option) *(wsp attribute) owsp ["/"] "}"  ; open and standalone
+       / "{" owsp "/" identifier *(wsp option) *(wsp attribute) owsp "}"  ; close
 
 ; Expression and literal parts
-function       = ":" identifier *(s option)
-option         = identifier [s] "=" [s] (literal / variable)
+function       = ":" identifier *(wsp option)
+option         = identifier owsp "=" owsp (literal / variable)
 ; Attributes are reserved for future standardization
-attribute      = "@" identifier [[s] "=" [s] (literal / variable)]
+attribute      = "@" identifier [owsp "=" owsp (literal / variable)]
 
 variable       = "$" name
 literal        = quoted / unquoted
@@ -54,7 +54,7 @@ local = %s".local"
 match = %s".match"
 
 ; Reserve additional .keywords for use by future versions of this specification.
-reserved-statement = reserved-keyword [s reserved-body] 1*([s] expression)
+reserved-statement = reserved-keyword [s reserved-body] 1*(owsp expression)
 ; Note that the following production is a simplification,
 ; as this rule MUST NOT be considered to match existing keywords
 ; (`.input`, `.local`, and `.match`).
@@ -67,7 +67,7 @@ reserved-annotation-start = "!" / "%" / "*" / "+" / "<" / ">" / "?" / "~"
 ; Reserve sigils for private-use by implementations.
 private-use-annotation    = private-start reserved-body
 private-start             = "^" / "&"
-reserved-body             = *([s] 1*(reserved-char / reserved-escape / quoted))
+reserved-body             = *(owsp 1*(reserved-char / reserved-escape / quoted))
 
 ; Names and identifiers
 ; identifier matches https://www.w3.org/TR/REC-xml-names/#NT-QName
@@ -104,5 +104,11 @@ quoted-escape   = backslash ( backslash / "|" )
 reserved-escape = backslash ( backslash / "{" / "|" / "}" )
 backslash       = %x5C ; U+005C REVERSE SOLIDUS "\"
 
-; Whitespace
-s = 1*( SP / HTAB / CR / LF / %x3000 )
+; optional whitespace
+owsp = *( s / %x200E / %x200F )
+
+; required whitespace
+wsp = [ (%x200E / %x200F) ] 1*s [ (%x200E / %x200F) ]
+
+; whitespace characters
+s = ( SP / HTAB / CR / LF / %x3000 )

spec/syntax.md

@@ -905,24 +905,55 @@ backslash       = %x5C ; U+005C REVERSE SOLIDUS "\"
 
 ### Whitespace
 
-**_<dfn>Whitespace</dfn>_** is defined as one or more of
-U+0009 CHARACTER TABULATION (tab), 
-U+000A LINE FEED (new line),
-U+000D CARRIAGE RETURN, 
-U+3000 IDEOGRAPHIC SPACE, 
-or U+0020 SPACE.
+The syntax limits whitespace characters outside of a _pattern_ to the following:
+`U+0009 CHARACTER TABULATION` (tab), 
+`U+000A LINE FEED` (new line),
+`U+000D CARRIAGE RETURN`, 
+`U+3000 IDEOGRAPHIC SPACE`, 
+or `U+0020 SPACE`.
 
 Inside _patterns_ and _quoted literals_,
 whitespace is part of the content and is recorded and stored verbatim.
 Whitespace is not significant outside translatable text, except where required by the syntax.
 
+There are two whitespace productions in the syntax.
+**_<dfn>Optional whitespace</dfn>_** is whitespace that is not required by the syntax, 
+but which users might want to include to increase the readability of a _message_.
+**_<dfn>Required whitespace</dfn>_** is whitespace that is required by the syntax.
+
+Tools SHOULD generate `U+200E LEFT-TO-RIGHT MARK` or `U+200F RIGHT-TO-LEFT MARK` 
+characters where permitted by the syntax before or following _identifiers_,
+_unquoted literals_, or _option_ values that use right-to-left characters 
+so that the _message_ displays intelligibly.
+
+These definitions of _whitespace_ implement
+[UTR#31 Rule 3a-2](https://www.unicode.org/reports/tr31/#R3a-2).
+It is a profile of R3a-1 in that specification because:
+the following pattern whitespace characters are not allowed:
+`U+000B FORM FEED`, 
+`U+000C VERTICAL TABULATION`, 
+`U+0085 NEXT LINE`, 
+`U+2028 LINE SEPARATOR` and 
+`U+2029 PARAGRAPH SEPARATOR`;
+the character `U+3000 IDEOGRAPHIC SPACE`
+_is_ interpreted as whitespace;
+ and the following character is not included in ignorable format controls:
+`U+200F RIGHT-TO-LEFT MARK`. 
+
 > [!NOTE]
 > The character U+3000 IDEOGRAPHIC SPACE is included in whitespace for
 > compatibility with certain East Asian keyboards and input methods,
 > in which users might accidentally create these characters in a _message_.
 
 ```abnf
-s = 1*( SP / HTAB / CR / LF / %x3000 )
+; optional whitespace
+owsp = *( s / %x200E / %x200F )
+
+; required whitespace
+wsp = [ (%x200E / %x200F) ] 1*s [ (%x200E / %x200F) ]
+
+; whitespace characters
+s = ( SP / HTAB / CR / LF / %x3000 )
 ```
 
 ## Complete ABNF