Merge pull request #773 from fangjinuo/master

* judge whitespace after {%, {#, {{
HubSpot · Nov 12, 2021 · b72dd95 · b72dd95
2 parents b0f8c99 + 487ed0c
commit b72dd95
Show file tree

Hide file tree

Showing 4 changed files with 87 additions and 43 deletions.
diff --git a/src/main/java/com/hubspot/jinjava/LegacyOverrides.java b/src/main/java/com/hubspot/jinjava/LegacyOverrides.java
@@ -9,11 +9,13 @@ public class LegacyOverrides {
   private final boolean evaluateMapKeys;
   private final boolean iterateOverMapKeys;
   private final boolean usePyishObjectMapper;
+  private final boolean whitespaceRequiredWithinTokens;
 
   private LegacyOverrides(Builder builder) {
     evaluateMapKeys = builder.evaluateMapKeys;
     iterateOverMapKeys = builder.iterateOverMapKeys;
     usePyishObjectMapper = builder.usePyishObjectMapper;
+    whitespaceRequiredWithinTokens = builder.whitespaceRequiredWithinTokens;
   }
 
   public static Builder newBuilder() {
@@ -32,10 +34,15 @@ public boolean isUsePyishObjectMapper() {
     return usePyishObjectMapper;
   }
 
+  public boolean isWhitespaceRequiredWithinTokens() {
+    return whitespaceRequiredWithinTokens;
+  }
+
   public static class Builder {
     private boolean evaluateMapKeys = false;
     private boolean iterateOverMapKeys = false;
     private boolean usePyishObjectMapper = false;
+    private boolean whitespaceRequiredWithinTokens = false;
 
     private Builder() {}
 
@@ -47,7 +54,8 @@ public static Builder from(LegacyOverrides legacyOverrides) {
       return new Builder()
         .withEvaluateMapKeys(legacyOverrides.evaluateMapKeys)
         .withIterateOverMapKeys(legacyOverrides.iterateOverMapKeys)
-        .withUsePyishObjectMapper(legacyOverrides.usePyishObjectMapper);
+        .withUsePyishObjectMapper(legacyOverrides.usePyishObjectMapper)
+        .withWhitespaceRequiredWithinTokens(legacyOverrides.whitespaceRequiredWithinTokens);
     }
 
     public Builder withEvaluateMapKeys(boolean evaluateMapKeys) {
@@ -64,5 +72,10 @@ public Builder withUsePyishObjectMapper(boolean usePyishObjectMapper) {
       this.usePyishObjectMapper = usePyishObjectMapper;
       return this;
     }
+
+    public Builder withWhitespaceRequiredWithinTokens(boolean useWhitespace) {
+      this.whitespaceRequiredWithinTokens = useWhitespace;
+      return this;
+    }
   }
 }
diff --git a/src/main/java/com/hubspot/jinjava/tree/parse/TokenScanner.java b/src/main/java/com/hubspot/jinjava/tree/parse/TokenScanner.java
@@ -88,48 +88,55 @@ private Token getNextToken() {
       if (c == symbols.getPrefix()) {
         if (currPost < length) {
           c = is[currPost];
-
-          if (c == symbols.getNote()) {
-            if (inComment == 1 || inRaw == 1) {
-              continue;
-            }
-            inComment = 1;
-
-            tokenLength = currPost - tokenStart - 1;
-            if (tokenLength > 0) {
-              // start a new token
-              lastStart = tokenStart;
-              tokenStart = --currPost;
-              tokenKind = c;
-              inComment = 0;
-              return newToken(symbols.getFixed());
-            } else {
-              tokenKind = c;
-            }
-          } else if (c == symbols.getTag() || c == symbols.getExprStart()) {
-            if (inComment > 0) {
-              continue;
-            }
-            if (inRaw > 0 && (c == symbols.getExprStart() || !isEndRaw())) {
-              continue;
-            }
-            // match token two ends
-            if (!matchToken(c) && tokenKind > 0) {
-              continue;
-            }
-            if (inBlock++ > 0) {
-              continue;
-            }
-
-            tokenLength = currPost - tokenStart - 1;
-            if (tokenLength > 0) {
-              // start a new token
-              lastStart = tokenStart;
-              tokenStart = --currPost;
-              tokenKind = c;
-              return newToken(symbols.getFixed());
-            } else {
-              tokenKind = c;
+          boolean startTokenFound = true;
+          if (config.getLegacyOverrides().isWhitespaceRequiredWithinTokens()) {
+            boolean hasNextChar = (currPost + 1) < length;
+            boolean nextCharIsWhitespace = hasNextChar && (' ' == is[currPost + 1]);
+            startTokenFound = nextCharIsWhitespace;
+          }
+          if (startTokenFound) {
+            if (c == symbols.getNote()) {
+              if (inComment == 1 || inRaw == 1) {
+                continue;
+              }
+              inComment = 1;
+
+              tokenLength = currPost - tokenStart - 1;
+              if (tokenLength > 0) {
+                // start a new token
+                lastStart = tokenStart;
+                tokenStart = --currPost;
+                tokenKind = c;
+                inComment = 0;
+                return newToken(symbols.getFixed());
+              } else {
+                tokenKind = c;
+              }
+            } else if (c == symbols.getTag() || c == symbols.getExprStart()) {
+              if (inComment > 0) {
+                continue;
+              }
+              if (inRaw > 0 && (c == symbols.getExprStart() || !isEndRaw())) {
+                continue;
+              }
+              // match token two ends
+              if (!matchToken(c) && tokenKind > 0) {
+                continue;
+              }
+              if (inBlock++ > 0) {
+                continue;
+              }
+
+              tokenLength = currPost - tokenStart - 1;
+              if (tokenLength > 0) {
+                // start a new token
+                lastStart = tokenStart;
+                tokenStart = --currPost;
+                tokenKind = c;
+                return newToken(symbols.getFixed());
+              } else {
+                tokenKind = c;
+              }
             }
           }
         } else { // reach the stream end

diff --git a/src/test/java/com/hubspot/jinjava/tree/parse/TokenScannerTest.java b/src/test/java/com/hubspot/jinjava/tree/parse/TokenScannerTest.java
@@ -7,6 +7,7 @@
 import com.google.common.collect.Lists;
 import com.google.common.io.Resources;
 import com.hubspot.jinjava.JinjavaConfig;
+import com.hubspot.jinjava.LegacyOverrides;
 import java.nio.charset.StandardCharsets;
 import java.util.List;
 import java.util.stream.Collectors;
@@ -259,6 +260,24 @@ public void testEscapedQuoteWithinAttrValue() {
       );
   }
 
+  @Test
+  public void testCommentWithWhitespaceChar() {
+    List<Token> tokens = tokens("comment-without-whitespace");
+    assertThat(tokens.get(0).content.trim()).isEqualTo("$");
+
+    LegacyOverrides legacyOverrides = LegacyOverrides
+      .newBuilder()
+      .withWhitespaceRequiredWithinTokens(true)
+      .build();
+    JinjavaConfig config = JinjavaConfig
+      .newBuilder()
+      .withLegacyOverrides(legacyOverrides)
+      .build();
+    TokenScanner scanner = fixture("comment-without-whitespace", config);
+    tokens = Lists.newArrayList(scanner);
+    assertThat(tokens.get(0).content.trim()).isEqualTo("${#array[@]}");
+  }
+
   @Test
   public void testEscapedBackslashWithinAttrValue() {
     List<Token> tokens = tokens("escape-char-tokens");
@@ -292,6 +311,10 @@ private List<Token> tokens(String fixture) {
   }
 
   private TokenScanner fixture(String fixture) {
+    return fixture(fixture, config);
+  }
+
+  private TokenScanner fixture(String fixture, JinjavaConfig config) {
     try {
       return new TokenScanner(
         Resources.toString(

diff --git a/src/test/resources/parse/tokenizer/comment-without-whitespace.jinja b/src/test/resources/parse/tokenizer/comment-without-whitespace.jinja
@@ -0,0 +1 @@
+${#array[@]}