Allow non-alphanum characters in tag content

lex.go

@@ -7,7 +7,6 @@ import (
 	"bytes"
 	"fmt"
 	"strings"
-	"unicode"
 	"unicode/utf8"
 )
 
@@ -30,7 +29,7 @@ type tokenType int
 const (
 	tokenError tokenType = iota // error occurred; value is text of error
 	tokenEOF
-	tokenIdentifier     // alphanumeric identifier
+	tokenIdentifier     // tag identifier: non-whitespace characters NOT containing closing delimiter
 	tokenLeftDelim      // {{ left action delimiter
 	tokenRightDelim     // }} right action delimiter
 	tokenText           // plain text
@@ -279,24 +278,24 @@ func stateTag(l *lexer) stateFn {
 		l.emit(tokenPartial)
 	case r == '{':
 		l.emit(tokenRawStart)
-	case alphanum(r):
+	default:
 		l.backup()
 		return stateIdent
-	default:
-		return l.errorf("unrecognized character in action: %#U", r)
 	}
 	return stateTag
 }
 
-// stateIdent scans an alphanumeric or field.
+// stateIdent scans an partial tag or field.
 func stateIdent(l *lexer) stateFn {
 Loop:
 	for {
-		switch r := l.next(); {
-		case alphanum(r):
-			// absorb.
+		switch r := l.peek(); {
+		case r == eof:
+			return l.errorf("unclosed tag")
+		case !whitespace(r) && !strings.HasPrefix(l.input[l.pos:], l.rightDelim):
+			// absorb
+			l.next()
 		default:
-			l.backup()
 			l.emit(tokenIdentifier)
 			break Loop
 		}
@@ -365,8 +364,3 @@ func whitespace(r rune) bool {
 	}
 	return false
 }
-
-// alphanum reports whether r is an alphabetic, digit, or underscore.
-func alphanum(r rune) bool {
-	return r == '_' || r == '.' || unicode.IsLetter(r) || unicode.IsDigit(r)
-}

lex_test.go

@@ -27,7 +27,7 @@ func TestLexer(t *testing.T) {
 			},
 		},
 		{
-			"\nfoo {{bar}} baz {{=| |=}}\r\n |foo| |={{ }}=| {{bar}}",
+			"\nfoo {{bar}} baz {{=| |=}}\r\n |foo| |={{! !}}=| {{!bar!}}",
 			[]token{
 				{typ: tokenText, val: "\nfoo "},
 				{typ: tokenLeftDelim, val: "{{"},
@@ -42,9 +42,9 @@ func TestLexer(t *testing.T) {
 				{typ: tokenText, val: " "},
 				{typ: tokenSetDelim},
 				{typ: tokenText, val: " "},
-				{typ: tokenLeftDelim, val: "{{"},
+				{typ: tokenLeftDelim, val: "{{!"},
 				{typ: tokenIdentifier, val: "bar"},
-				{typ: tokenRightDelim, val: "}}"},
+				{typ: tokenRightDelim, val: "!}}"},
 				{typ: tokenEOF},
 			},
 		},

parse_test.go

@@ -4,6 +4,7 @@ package mustache
 
 import (
 	"reflect"
+	"strings"
 	"testing"
 )
 
@@ -56,6 +57,36 @@ func TestParser(t *testing.T) {
 				}},
 			},
 		},
+		{
+			"{{#*}}({{.}}){{/*}}",
+			[]node{
+				&sectionNode{"*", false, []node{
+					textNode("("),
+					&varNode{".", true},
+					textNode(")"),
+				}},
+			},
+		},
+		{
+			"{{#list}}({{*}}){{/list}}",
+			[]node{
+				&sectionNode{"list", false, []node{
+					textNode("("),
+					&varNode{"*", true},
+					textNode(")"),
+				}},
+			},
+		},
+		{
+			"{{#list}}({{a}a}}){{/list}}",
+			[]node{
+				&sectionNode{"list", false, []node{
+					textNode("("),
+					&varNode{"a}a", true},
+					textNode(")"),
+				}},
+			},
+		},
 	} {
 		parser := newParser(newLexer(test.template, "{{", "}}"))
 		elems, err := parser.parse()
@@ -69,3 +100,21 @@ func TestParser(t *testing.T) {
 		}
 	}
 }
+
+func TestParserNegative(t *testing.T) {
+	for _, test := range []struct {
+		template string
+		expErr   string
+	}{
+		{
+			"{{foo}",
+			`1:6 syntax error: unreachable code t_error:"unclosed tag"`,
+		},
+	} {
+		parser := newParser(newLexer(test.template, "{{", "}}"))
+		_, err := parser.parse()
+		if err == nil || !strings.Contains(err.Error(), test.expErr) {
+			t.Errorf("expect error: %q, got %q", test.expErr, err)
+		}
+	}
+}