Implement {% do %} blocks and fix {% set %} block scoping

src/main/java/com/hubspot/jinjava/interpret/JinjavaInterpreter.java

@@ -33,8 +33,8 @@
 import com.hubspot.jinjava.interpret.TemplateError.ErrorReason;
 import com.hubspot.jinjava.interpret.TemplateError.ErrorType;
 import com.hubspot.jinjava.interpret.errorcategory.BasicTemplateErrorCategory;
+import com.hubspot.jinjava.lib.tag.DoTag;
 import com.hubspot.jinjava.lib.tag.ExtendsTag;
-import com.hubspot.jinjava.lib.tag.SetTag;
 import com.hubspot.jinjava.lib.tag.eager.EagerGenericTag;
 import com.hubspot.jinjava.objects.serialization.PyishObjectMapper;
 import com.hubspot.jinjava.objects.serialization.PyishSerializable;
@@ -399,9 +399,9 @@ public String render(Node root, boolean processExtendRoots) {
     resolveBlockStubs(output);
     if (ignoredOutput.length() > 0) {
       return (
-        EagerReconstructionUtils.buildBlockSetTag(
-          SetTag.IGNORED_VARIABLE_NAME,
+        EagerReconstructionUtils.wrapInTag(
           ignoredOutput.toString(),
+          DoTag.TAG_NAME,
           this,
           false
         ) +

src/main/java/com/hubspot/jinjava/lib/expression/EagerExpressionStrategy.java

@@ -6,7 +6,6 @@
 import com.hubspot.jinjava.interpret.JinjavaInterpreter;
 import com.hubspot.jinjava.interpret.TemplateError.ErrorReason;
 import com.hubspot.jinjava.lib.filter.EscapeFilter;
-import com.hubspot.jinjava.lib.tag.RawTag;
 import com.hubspot.jinjava.lib.tag.eager.DeferredToken;
 import com.hubspot.jinjava.lib.tag.eager.EagerExecutionResult;
 import com.hubspot.jinjava.tree.output.RenderedOutputNode;
@@ -67,7 +66,7 @@ private String eagerResolveExpression(
         interpreter
       )
     );
-    String helpers = wrapInExpression(
+    String deferredExpressionImage = wrapInExpression(
       eagerExecutionResult.getResult().toString(),
       interpreter
     );
@@ -76,7 +75,7 @@ private String eagerResolveExpression(
         interpreter,
         new DeferredToken(
           new ExpressionToken(
-            helpers,
+            deferredExpressionImage,
             master.getLineNumber(),
             master.getStartPosition(),
             master.getSymbols()
@@ -95,7 +94,7 @@ private String eagerResolveExpression(
     );
     // There is only a preserving prefix because it couldn't be entirely evaluated.
     return EagerReconstructionUtils.wrapInAutoEscapeIfNeeded(
-      prefixToPreserveState.toString() + helpers,
+      prefixToPreserveState.toString() + deferredExpressionImage,
       interpreter
     );
   }
@@ -125,8 +124,7 @@ public static String postProcessResult(
           }
         }
       } else {
-        // Possible macro/set tag in front of this one. Includes result
-        result = wrapInRawOrExpressionIfNeeded(result, interpreter);
+        result = EagerReconstructionUtils.wrapInRawIfNeeded(result, interpreter);
       }
     }
 
@@ -149,24 +147,6 @@ private static long getParsingErrorsCount(JinjavaInterpreter interpreter) {
       .count();
   }
 
-  private static String wrapInRawOrExpressionIfNeeded(
-    String output,
-    JinjavaInterpreter interpreter
-  ) {
-    JinjavaConfig config = interpreter.getConfig();
-    if (
-      config.getExecutionMode().isPreserveRawTags() &&
-      !interpreter.getContext().isUnwrapRawOverride() &&
-      (
-        output.contains(config.getTokenScannerSymbols().getExpressionStart()) ||
-        output.contains(config.getTokenScannerSymbols().getExpressionStartWithTag())
-      )
-    ) {
-      return EagerReconstructionUtils.wrapInTag(output, RawTag.TAG_NAME, interpreter);
-    }
-    return output;
-  }
-
   private static String wrapInExpression(String output, JinjavaInterpreter interpreter) {
     JinjavaConfig config = interpreter.getConfig();
     return String.format(

src/main/java/com/hubspot/jinjava/lib/tag/DoTag.java

@@ -4,40 +4,44 @@
 import com.hubspot.jinjava.doc.annotations.JinjavaSnippet;
 import com.hubspot.jinjava.doc.annotations.JinjavaTextMateSnippet;
 import com.hubspot.jinjava.interpret.JinjavaInterpreter;
-import com.hubspot.jinjava.interpret.TemplateSyntaxException;
 import com.hubspot.jinjava.tree.TagNode;
+import com.hubspot.jinjava.tree.parse.TagToken;
 import org.apache.commons.lang3.StringUtils;
 
 @JinjavaDoc(
   value = "Evaluates expression without printing out result.",
-  snippets = { @JinjavaSnippet(code = "{% do list.append('value 2') %}") }
+  snippets = {
+    @JinjavaSnippet(code = "{% do list.append('value 2') %}"),
+    @JinjavaSnippet(
+      desc = "Execute a block of code in the same scope while ignoring the output",
+      code = "{% do %}\n" +
+      "{% set foo = [] %}\n" +
+      "{{ foo.append('a') }}\n" +
+      "{% enddo %}"
+    )
+  }
 )
 @JinjavaTextMateSnippet(code = "{% do ${1:expr} %}")
-public class DoTag implements Tag {
+public class DoTag implements Tag, FlexibleTag {
   public static final String TAG_NAME = "do";
 
   @Override
   public String interpret(TagNode tagNode, JinjavaInterpreter interpreter) {
-    if (StringUtils.isBlank(tagNode.getHelpers())) {
-      throw new TemplateSyntaxException(
-        tagNode.getMaster().getImage(),
-        "Tag 'do' expects expression",
-        tagNode.getLineNumber(),
-        tagNode.getStartPosition()
-      );
+    if (hasEndTag((TagToken) tagNode.getMaster())) {
+      tagNode.getChildren().forEach(child -> child.render(interpreter));
+    } else {
+      interpreter.resolveELExpression(tagNode.getHelpers(), tagNode.getLineNumber());
     }
-
-    interpreter.resolveELExpression(tagNode.getHelpers(), tagNode.getLineNumber());
     return "";
   }
 
   @Override
-  public String getEndTagName() {
-    return null;
+  public String getName() {
+    return TAG_NAME;
   }
 
   @Override
-  public String getName() {
-    return TAG_NAME;
+  public boolean hasEndTag(TagToken tagToken) {
+    return StringUtils.isBlank(tagToken.getHelpers());
   }
 }

src/main/java/com/hubspot/jinjava/lib/tag/SetTag.java

@@ -22,6 +22,7 @@
 import com.hubspot.jinjava.interpret.DeferredValue;
 import com.hubspot.jinjava.interpret.DeferredValueException;
 import com.hubspot.jinjava.interpret.JinjavaInterpreter;
+import com.hubspot.jinjava.interpret.JinjavaInterpreter.InterpreterScopeClosable;
 import com.hubspot.jinjava.interpret.TemplateSyntaxException;
 import com.hubspot.jinjava.objects.Namespace;
 import com.hubspot.jinjava.tree.Node;
@@ -70,7 +71,7 @@
       code = "{% set name = 'Jack' %}\n" +
       "{% set message %}\n" +
       "My name is {{ name }}\n" +
-      "{% end_set %}"
+      "{% endset %}"
     )
   }
 )
@@ -136,19 +137,43 @@ private String interpretBlockSet(TagNode tagNode, JinjavaInterpreter interpreter
     if (filterPos >= 0) {
       var = tagNode.getHelpers().substring(0, filterPos).trim();
     }
-    StringBuilder sb = new StringBuilder();
-    for (Node child : tagNode.getChildren()) {
-      sb.append(child.render(interpreter));
-    }
+    String result;
+    result = renderChildren(tagNode, interpreter, var);
     try {
-      executeSetBlock(tagNode, var, sb.toString(), filterPos >= 0, interpreter);
+      executeSetBlock(tagNode, var, result, filterPos >= 0, interpreter);
     } catch (DeferredValueException e) {
       DeferredValueUtils.deferVariables(new String[] { var }, interpreter.getContext());
       throw e;
     }
     return "";
   }
 
+  public static String renderChildren(
+    TagNode tagNode,
+    JinjavaInterpreter interpreter,
+    String var
+  ) {
+    String result;
+    if (IGNORED_VARIABLE_NAME.equals(var)) {
+      result = renderChildren(tagNode, interpreter);
+    } else {
+      try (InterpreterScopeClosable c = interpreter.enterScope()) {
+        result = renderChildren(tagNode, interpreter);
+      }
+    }
+    return result;
+  }
+
+  private static String renderChildren(TagNode tagNode, JinjavaInterpreter interpreter) {
+    String result;
+    StringBuilder sb = new StringBuilder();
+    for (Node child : tagNode.getChildren()) {
+      sb.append(child.render(interpreter));
+    }
+    result = sb.toString();
+    return result;
+  }
+
   private void executeSetBlock(
     TagNode tagNode,
     String var,

src/main/java/com/hubspot/jinjava/lib/tag/eager/EagerBlockSetTagStrategy.java

@@ -5,7 +5,6 @@
 import com.hubspot.jinjava.interpret.DeferredValueException;
 import com.hubspot.jinjava.interpret.JinjavaInterpreter;
 import com.hubspot.jinjava.lib.tag.SetTag;
-import com.hubspot.jinjava.tree.Node;
 import com.hubspot.jinjava.tree.TagNode;
 import com.hubspot.jinjava.tree.parse.TagToken;
 import com.hubspot.jinjava.util.EagerContextWatcher;
@@ -30,19 +29,14 @@ protected EagerBlockSetTagStrategy(SetTag setTag) {
   @Override
   protected EagerExecutionResult getEagerExecutionResult(
     TagNode tagNode,
+    String[] variables,
     String expression,
     JinjavaInterpreter interpreter
   ) {
     EagerExecutionResult result = EagerContextWatcher.executeInChildContext(
       eagerInterpreter ->
         EagerExpressionResult.fromSupplier(
-          () -> {
-            StringBuilder sb = new StringBuilder();
-            for (Node child : tagNode.getChildren()) {
-              sb.append(child.render(eagerInterpreter).getValue());
-            }
-            return sb.toString();
-          },
+          () -> SetTag.renderChildren(tagNode, eagerInterpreter, variables[0]),
           eagerInterpreter
         ),
       interpreter,
@@ -76,6 +70,7 @@ protected Optional<String> resolveSet(
       if (filterPos >= 0) {
         EagerExecutionResult filterResult = EagerInlineSetTagStrategy.INSTANCE.getEagerExecutionResult(
           tagNode,
+          variables,
           tagNode.getHelpers().trim(),
           interpreter
         );
@@ -166,6 +161,7 @@ protected String buildImage(
     if (filterPos >= 0) {
       EagerExecutionResult filterResult = EagerInlineSetTagStrategy.INSTANCE.getEagerExecutionResult(
         tagNode,
+        variables,
         tagNode.getHelpers().trim(),
         interpreter
       );

src/main/java/com/hubspot/jinjava/lib/tag/eager/EagerDoTag.java

@@ -1,14 +1,18 @@
 package com.hubspot.jinjava.lib.tag.eager;
 
 import com.google.common.annotations.Beta;
+import com.hubspot.jinjava.interpret.InterpretException;
 import com.hubspot.jinjava.interpret.JinjavaInterpreter;
-import com.hubspot.jinjava.interpret.TemplateSyntaxException;
 import com.hubspot.jinjava.lib.tag.DoTag;
+import com.hubspot.jinjava.lib.tag.FlexibleTag;
+import com.hubspot.jinjava.tree.TagNode;
 import com.hubspot.jinjava.tree.parse.TagToken;
-import org.apache.commons.lang3.StringUtils;
+import com.hubspot.jinjava.util.EagerContextWatcher;
+import com.hubspot.jinjava.util.EagerExpressionResolver.EagerExpressionResult;
+import com.hubspot.jinjava.util.EagerReconstructionUtils;
 
 @Beta
-public class EagerDoTag extends EagerStateChangingTag<DoTag> {
+public class EagerDoTag extends EagerStateChangingTag<DoTag> implements FlexibleTag {
 
   public EagerDoTag() {
     super(new DoTag());
@@ -19,15 +23,51 @@ public EagerDoTag(DoTag doTag) {
   }
 
   @Override
-  public String getEagerTagImage(TagToken tagToken, JinjavaInterpreter interpreter) {
-    String expr = tagToken.getHelpers();
-    if (StringUtils.isBlank(expr)) {
-      throw new TemplateSyntaxException(
+  public String eagerInterpret(
+    TagNode tagNode,
+    JinjavaInterpreter interpreter,
+    InterpretException e
+  ) {
+    if (hasEndTag((TagToken) tagNode.getMaster())) {
+      EagerExecutionResult eagerExecutionResult = EagerContextWatcher.executeInChildContext(
+        eagerInterpreter ->
+          EagerExpressionResult.fromSupplier(
+            () -> renderChildren(tagNode, interpreter),
+            eagerInterpreter
+          ),
         interpreter,
-        tagToken.getImage(),
-        "Tag 'do' expects expression"
+        EagerContextWatcher
+          .EagerChildContextConfig.newBuilder()
+          .withTakeNewValue(true)
+          .withCheckForContextChanges(!interpreter.getContext().isDeferredExecutionMode())
+          .build()
+      );
+      StringBuilder prefixToPreserveState = new StringBuilder();
+      if (interpreter.getContext().isDeferredExecutionMode()) {
+        prefixToPreserveState.append(eagerExecutionResult.getPrefixToPreserveState());
+      } else {
+        interpreter.getContext().putAll(eagerExecutionResult.getSpeculativeBindings());
+      }
+      if (eagerExecutionResult.getResult().isFullyResolved()) {
+        return (prefixToPreserveState.toString());
+      }
+      return EagerReconstructionUtils.wrapInTag(
+        eagerExecutionResult.asTemplateString(),
+        getName(),
+        interpreter,
+        true
       );
     }
-    return EagerPrintTag.interpretExpression(expr, tagToken, interpreter, false);
+    return EagerPrintTag.interpretExpression(
+      tagNode.getHelpers(),
+      (TagToken) tagNode.getMaster(),
+      interpreter,
+      false
+    );
+  }
+
+  @Override
+  public boolean hasEndTag(TagToken tagToken) {
+    return getTag().hasEndTag(tagToken);
   }
 }

src/main/java/com/hubspot/jinjava/lib/tag/eager/EagerFromTag.java

@@ -1,14 +1,13 @@
 package com.hubspot.jinjava.lib.tag.eager;
 
-import static com.hubspot.jinjava.lib.tag.SetTag.IGNORED_VARIABLE_NAME;
-
 import com.google.common.annotations.Beta;
 import com.google.common.collect.ImmutableMap;
 import com.hubspot.jinjava.interpret.Context;
 import com.hubspot.jinjava.interpret.DeferredValueException;
 import com.hubspot.jinjava.interpret.InterpretException;
 import com.hubspot.jinjava.interpret.JinjavaInterpreter;
 import com.hubspot.jinjava.lib.fn.MacroFunction;
+import com.hubspot.jinjava.lib.tag.DoTag;
 import com.hubspot.jinjava.lib.tag.FromTag;
 import com.hubspot.jinjava.loader.RelativePathResolver;
 import com.hubspot.jinjava.objects.serialization.PyishObjectMapper;
@@ -121,9 +120,9 @@ public String getEagerTagImage(TagToken tagToken, JinjavaInterpreter interpreter
             output +
             EagerReconstructionUtils.buildSetTag(newToOldImportNames, interpreter, true);
         }
-        return EagerReconstructionUtils.buildBlockSetTag(
-          IGNORED_VARIABLE_NAME,
+        return EagerReconstructionUtils.wrapInTag(
           output,
+          DoTag.TAG_NAME,
           interpreter,
           true
         );