Reform and improve string literals (#793)

- Combine and simplify string and backtick lexing.
- Allow newlines in strings and backticks.
- Add triple-delimited indented strings and backticks. Common indented literal non-blank line leading whitespace is stripped.
- If a literal newline is escaped, it will be suppressed.
- Backticks starting with `#!` are reserved for a future upgrade.

GRAMMAR.md

@@ -9,17 +9,20 @@ tokens
 ------
 
 ```
-BACKTICK   = `[^`]*`
-COMMENT    = #([^!].*)?$
-DEDENT     = emitted when indentation decreases
-EOF        = emitted at the end of the file
-INDENT     = emitted when indentation increases
-LINE       = emitted before a recipe line
-NAME       = [a-zA-Z_][a-zA-Z0-9_-]*
-NEWLINE    = \n|\r\n
-RAW_STRING = '[^']*'
-STRING     = "[^"]*" # also processes \n \r \t \" \\ escapes
-TEXT       = recipe text, only matches in a recipe body
+BACKTICK            = `[^`]*`
+INDENTED_BACKTICK   = ```[^(```)]*```
+COMMENT             = #([^!].*)?$
+DEDENT              = emitted when indentation decreases
+EOF                 = emitted at the end of the file
+INDENT              = emitted when indentation increases
+LINE                = emitted before a recipe line
+NAME                = [a-zA-Z_][a-zA-Z0-9_-]*
+NEWLINE             = \n|\r\n
+RAW_STRING          = '[^']*'
+INDENTED_RAW_STRING = '''[^(''')]*'''
+STRING              = "[^"]*" # also processes \n \r \t \" \\ escapes
+INDENTED_STRING     = """[^("""]*""" # also processes \n \r \t \" \\ escapes
+TEXT                = recipe text, only matches in a recipe body
 ```
 
 grammar syntax
@@ -69,14 +72,16 @@ condition     : expression '==' expression
               | expression '!=' expression
 
 value         : NAME '(' sequence? ')'
-              | STRING
-              | RAW_STRING
               | BACKTICK
+              | INDENTED_BACKTICK
               | NAME
+              | string
               | '(' expression ')'
 
 string        : STRING
+              | INDENTED_STRING
               | RAW_STRING
+              | INDENTED_RAW_STRING
 
 sequence      : expression ',' sequence
               | expression ','?

README.adoc

@@ -545,6 +545,8 @@ string-with-newline         := "\n"
 string-with-carriage-return := "\r"
 string-with-double-quote    := "\""
 string-with-slash           := "\\"
+string-with-no-newline      := "\
+"
 ```
 
 ```sh
@@ -553,6 +555,7 @@ $ just --evaluate
 string-with-double-quote    := """
 string-with-newline         := "
 "
+string-with-no-newline      := ""
 string-with-slash           := "\"
 string-with-tab             := "     "
 ```
@@ -580,6 +583,25 @@ $ just --evaluate
 escapes := "\t\n\r\"\\"
 ```
 
+Indented versions of both single- and double-quoted strings, delimited by triple single- or triple double-quotes, are supported. Indented string lines are stripped of leading whitespace common to all non-blank lines:
+
+```make
+# this string will evaluate to `foo\nbar\n`
+x := '''
+  foo
+  bar
+'''
+
+# this string will evaluate to `abc\n  wuv\nbar\n`
+y := """
+  abc
+    wuv
+  xyz
+"""
+```
+
+Similar to unindented strings, indented double-quoted strings process escape sequences, and indented single-quoted strings ignore escape sequences. Escape sequence processing takes place after unindentation. The unindention algorithm does not take escape-sequence produced whitespace or newlines into account.
+
 === Ignoring Errors
 
 Normally, if a command returns a nonzero exit status, execution will stop. To
@@ -716,6 +738,20 @@ serve:
     ./serve {{localhost}} 8080
 ```
 
+Indented backticks, delimited by three backticks, are de-indented in the same manner as indented strings:
+
+```make
+# This backtick evaluates the command `echo foo\necho bar\n`, which produces the value `foo\nbar\n`.
+stuff := ```
+    echo foo
+    echo bar
+  ```
+```
+
+See the <<Strings>> section for details on unindenting.
+
+Backticks may not start with `#!`. This syntax is reserved for a future upgrade.
+
 === Conditional Expressions
 
 `if`/`else` expressions evaluate different branches depending on if two expressions evaluate to the same value:

src/common.rs

@@ -1,6 +1,5 @@
 // stdlib
 pub(crate) use std::{
-  borrow::Cow,
   cmp,
   collections::{BTreeMap, BTreeSet},
   env,
@@ -31,7 +30,9 @@ pub(crate) use unicode_width::{UnicodeWidthChar, UnicodeWidthStr};
 pub(crate) use crate::{config_error, setting};
 
 // functions
-pub(crate) use crate::{default::default, empty::empty, load_dotenv::load_dotenv, output::output};
+pub(crate) use crate::{
+  default::default, empty::empty, load_dotenv::load_dotenv, output::output, unindent::unindent,
+};
 
 // traits
 pub(crate) use crate::{

src/compilation_error.rs

@@ -26,6 +26,9 @@ impl Display for CompilationError<'_> {
           recipe_line.ordinal(),
         )?;
       },
+      BacktickShebang => {
+        writeln!(f, "Backticks may not start with `#!`")?;
+      },
       CircularRecipeDependency { recipe, ref circle } =>
         if circle.len() == 2 {
           writeln!(f, "Recipe `{}` depends on itself", recipe)?;
@@ -242,10 +245,10 @@ impl Display for CompilationError<'_> {
       UnterminatedInterpolation => {
         writeln!(f, "Unterminated interpolation")?;
       },
-      UnterminatedString(StringKind::Cooked) | UnterminatedString(StringKind::Raw) => {
+      UnterminatedString => {
         writeln!(f, "Unterminated string")?;
       },
-      UnterminatedString(StringKind::Backtick) => {
+      UnterminatedBacktick => {
         writeln!(f, "Unterminated backtick")?;
       },
       Internal { ref message } => {

src/compilation_error_kind.rs

@@ -6,6 +6,7 @@ pub(crate) enum CompilationErrorKind<'src> {
     alias:       &'src str,
     recipe_line: usize,
   },
+  BacktickShebang,
   CircularRecipeDependency {
     recipe: &'src str,
     circle: Vec<&'src str>,
@@ -107,5 +108,6 @@ pub(crate) enum CompilationErrorKind<'src> {
     open_line: usize,
   },
   UnterminatedInterpolation,
-  UnterminatedString(StringKind),
+  UnterminatedString,
+  UnterminatedBacktick,
 }

src/evaluator.rs

@@ -107,7 +107,7 @@ impl<'src, 'run> Evaluator<'src, 'run> {
           }),
         }
       },
-      Expression::StringLiteral { string_literal } => Ok(string_literal.cooked.to_string()),
+      Expression::StringLiteral { string_literal } => Ok(string_literal.cooked.clone()),
       Expression::Backtick { contents, token } =>
         if self.config.dry_run {
           Ok(format!("`{}`", contents))

src/expression.rs

@@ -10,7 +10,7 @@ use crate::common::*;
 pub(crate) enum Expression<'src> {
   /// `contents`
   Backtick {
-    contents: &'src str,
+    contents: String,
     token:    Token<'src>,
   },
   /// `name(arguments)`