[MOREL-202] Allow unbounded variables (from and join without in…

…), and remove `suchthat` keyword

We remove `suchthat` because it is superfluous. Previously,
`from e suchthat condition1 where condition2` would allow you
to define a variable `v` and infer its bounded extent based
on conditions, but after this change you can accomplish the
same with `from e where condition1 andalso condition2`, and
there may be even be intervening clauses.

Following #208 `FromBuilder` should flatten nested `from`
with tuple.

Change type of `TuplePat.type` to `RecordLikeType`, because
it is always `PrimitiveType.UNIT` or a `TupleType`.

In `SuchThatShuttle`, infer extent of `x` from `where x =
y.field`, and move `from y` forward if necessary.

In `from x where condition`, `x` must be an identifier, or a
list of identifiers such as `from x, y where condition`.
With `suchthat` we allowed patterns such as `(x, y)`, `{x, y}`,
but this makes no sense because we are doing no assignment,
and the variables have no intrinsic order (except when
composed into a record in the final `yield`).

`FromBuilder` now checks that the type of the scan variable
and extent are consistent. For example, in `from e in emps`,
`emps` must be have a list type, and `e` must have the same
type as an element of the list.

Remove `suchthat` from parser and documentation, remove
Op.SUCH_THAT, and remove dead code.

Fixes #202

docs/reference.md

@@ -37,7 +37,7 @@ just because they take effort to build.
 Contributions are welcome!
 
 In Morel but not Standard ML:
-* `from` expression with `in`, `suchthat`, `join`, `where`, `group`,
+* `from` expression with `in`, `join`, `where`, `group`,
   `compute`, `order`, `skip`, `take`, `yield` clauses
 * `union`, `except`, `intersect`, `elem`, `notelem` operators
 * "*lab* `=`" is optional in `exprow`
@@ -148,8 +148,7 @@ In Standard ML but not in Morel:
                                 match
 <i>matchItem</i> &rarr; <i>pat</i> <b>=&gt;</b> <i>exp</i>
 <i>scan</i> &rarr; <i>pat</i> [ <b>in</b> | <b>=</b> ] <i>exp</i>
-    | <i>var<sub>1</sub></i> <b>,</b> ... <b>,</b> <i>var<sub>v</sub></i> <b>suchThat</b> <i>exp</i>
-                                constrained iteration (<i>v</i> &ge; 1)
+    | <i>var</i>                       unbounded variable
 <i>step</i> &rarr; <b>where</b> <i>exp</i>                filter clause
     | <b>join</b> <i>scan</i> [ <b>on</b> <i>exp</i> ]      join clause
     | <b>group</b> <i>groupKey<sub>1</sub></i> <b>,</b> ... <b>,</b> <i>groupKey<sub>g</sub></i>

etc/morel.lang

@@ -34,7 +34,7 @@ Keywords={
         "Word", "Char", "StringCvt", "Word8", "List", "ListPair", "Vector", "TextIO",
         "BinIO", "div", "mod",
         "compute", "desc", "elem", "except", "exists", "from", "group", "intersect",
-        "notelem", "notExists", "order", "suchthat", "union", "yield"},
+        "notelem", "notExists", "order", "union", "yield"},
   },
   { Id=2,
     List={"bool", "string", "int", "real", "word"},

src/main/java/net/hydromatic/morel/ast/Ast.java

@@ -1612,15 +1612,15 @@ public abstract static class FromStep extends AstNode {
     }
   }
 
-  /** A scan (e.g. "e in emps")
+  /** A scan (e.g. "e in emps", "e")
    * or scan-and-join (e.g. "left join d in depts on e.deptno = d.deptno")
    * in a {@code from} expression. */
   public static class Scan extends FromStep {
     public final Pat pat;
-    public final Exp exp;
+    public final @Nullable Exp exp;
     public final @Nullable Exp condition;
 
-    Scan(Pos pos, Op op, Pat pat, Exp exp, @Nullable Exp condition) {
+    Scan(Pos pos, Op op, Pat pat, @Nullable Exp exp, @Nullable Exp condition) {
       super(pos, op);
       switch (op) {
       case INNER_JOIN:
@@ -1637,26 +1637,17 @@ public static class Scan extends FromStep {
     }
 
     @Override AstWriter unparse(AstWriter w, int left, int right) {
-      final String op;
-      final Exp exp;
-      switch (this.exp.op) {
-      case FROM_EQ:
-        op = " = ";
-        exp = ((PrefixCall) this.exp).a;
-        break;
-      case SUCH_THAT:
-        op = " suchthat ";
-        exp = ((PrefixCall) this.exp).a;
-        break;
-      default:
-        op = " in ";
-        exp = this.exp;
-        break;
+      w.append(op.padded)
+          .append(pat, 0, 0);
+      if (exp != null) {
+        if (exp.op == Op.FROM_EQ) {
+          w.append(" = ")
+              .append(((PrefixCall) this.exp).a, Op.EQ.right, 0);
+        } else {
+          w.append(" in ")
+              .append(this.exp, Op.EQ.right, 0);
+        }
       }
-      w.append(this.op.padded)
-          .append(pat, 0, 0)
-          .append(op)
-          .append(exp, Op.EQ.right, 0);
       if (condition != null) {
         w.append(" on ")
             .append(condition, 0, 0);
@@ -1672,9 +1663,9 @@ public static class Scan extends FromStep {
       visitor.visit(this);
     }
 
-    public Scan copy(Pat pat, Exp exp, Exp condition) {
+    public Scan copy(Pat pat, @Nullable Exp exp, @Nullable Exp condition) {
       return this.pat.equals(pat)
-          && this.exp.equals(exp)
+          && Objects.equals(this.exp, exp)
           && Objects.equals(this.condition, condition)
           ? this
           : new Scan(pos, op, pat, exp, condition);

src/main/java/net/hydromatic/morel/ast/AstBuilder.java

@@ -349,12 +349,6 @@ public Ast.Exp fromEq(Ast.Exp exp) {
     return new Ast.PrefixCall(exp.pos, Op.FROM_EQ, exp);
   }
 
-  /** Wraps an expression to distinguish "from x suchthat e" from
-   * "from x in e". */
-  public Ast.Exp fromSuchThat(Ast.Exp exp) {
-    return new Ast.PrefixCall(exp.pos, Op.SUCH_THAT, exp);
-  }
-
   public Ast.Fn fn(Pos pos, Ast.Match... matchList) {
     return new Ast.Fn(pos, ImmutableList.copyOf(matchList));
   }

src/main/java/net/hydromatic/morel/ast/Core.java

@@ -20,15 +20,18 @@
 
 import net.hydromatic.morel.compile.BuiltIn;
 import net.hydromatic.morel.compile.Environment;
+import net.hydromatic.morel.compile.Extents;
 import net.hydromatic.morel.compile.Resolver;
 import net.hydromatic.morel.eval.Closure;
 import net.hydromatic.morel.eval.Code;
 import net.hydromatic.morel.type.Binding;
 import net.hydromatic.morel.type.DataType;
 import net.hydromatic.morel.type.FnType;
 import net.hydromatic.morel.type.ListType;
+import net.hydromatic.morel.type.PrimitiveType;
 import net.hydromatic.morel.type.RecordLikeType;
 import net.hydromatic.morel.type.RecordType;
+import net.hydromatic.morel.type.TupleType;
 import net.hydromatic.morel.type.Type;
 import net.hydromatic.morel.type.TypeSystem;
 import net.hydromatic.morel.type.TypedValue;
@@ -103,11 +106,6 @@ public Type type() {
       return type;
     }
 
-    /** Returns the type's key. */
-    public Type.Key typeKey() {
-      return type().key();
-    }
-
     @Override public abstract Pat accept(Shuttle shuttle);
   }
 
@@ -383,9 +381,21 @@ public static class Con0Pat extends Pat {
   public static class TuplePat extends Pat {
     public final List<Pat> args;
 
-    TuplePat(Type type, ImmutableList<Pat> args) {
+    /** Creates a TuplePat.
+     *
+     * <p>Type is {@link PrimitiveType#UNIT} if {@code args} is empty,
+     * otherwise a {@link TupleType}. */
+    TuplePat(RecordLikeType type, ImmutableList<Pat> args) {
       super(Op.TUPLE_PAT, type);
       this.args = requireNonNull(args);
+      checkArgument(args.size() == type.argNameTypes().size());
+      checkArgument(args.isEmpty()
+          ? type == PrimitiveType.UNIT
+          : type instanceof TupleType);
+    }
+
+    @Override public RecordLikeType type() {
+      return (RecordLikeType) type;
     }
 
     @Override AstWriter unparse(AstWriter w, int left, int right) {
@@ -407,6 +417,17 @@ public TuplePat copy(TypeSystem typeSystem, List<Pat> args) {
       return args.equals(this.args) ? this
           : core.tuplePat(typeSystem, args);
     }
+
+    /** Returns the names of all components that are named. */
+    public List<String> fieldNames() {
+      final ImmutableList.Builder<String> names = ImmutableList.builder();
+      for (Pat arg : args) {
+        if (arg instanceof NamedPat) {
+          names.add(((NamedPat) arg).name);
+        }
+      }
+      return names.build();
+    }
   }
 
   /** List pattern.
@@ -1127,7 +1148,11 @@ public static class From extends Exp {
       }
     }
 
-    public Exp copy(TypeSystem typeSystem, Environment env,
+    /** Copies this {@code From} with a new set of steps.
+     *
+     * <p>Returns this {@code From} if the steps are the same.
+     * If {@code env} is not null, performs additional checking. */
+    public Exp copy(TypeSystem typeSystem, @Nullable Environment env,
         List<FromStep> steps) {
       return steps.equals(this.steps)
           ? this
@@ -1165,17 +1190,28 @@ public static class Scan extends FromStep {
     Scan(Op op, ImmutableList<Binding> bindings, Pat pat, Exp exp,
         Exp condition) {
       super(op, bindings);
-      switch (op) {
-      case INNER_JOIN:
-      case SUCH_THAT:
-        break;
-      default:
+      if (op != Op.INNER_JOIN) {
         // SCAN and CROSS_JOIN are valid in ast, not core.
-        throw new AssertionError("not a join type " + op);
+        throw new IllegalArgumentException("not a join type " + op);
       }
       this.pat = requireNonNull(pat, "pat");
       this.exp = requireNonNull(exp, "exp");
       this.condition = requireNonNull(condition, "condition");
+      if (!(exp.type instanceof ListType)) {
+        throw new IllegalArgumentException("scan expression must be list: "
+            + exp.type);
+      }
+      final ListType listType = (ListType) exp.type;
+      if (!canAssign(listType.elementType, pat.type)) {
+        throw new IllegalArgumentException(exp.type + " + " + pat.type);
+      }
+    }
+
+    /** Returns whether you can assign a value of {@code fromType} to a variable
+     * of type {@code toType}. */
+    private static boolean canAssign(Type fromType, Type toType) {
+      return fromType.equals(toType)
+          || toType.isProgressive();
     }
 
     @Override public Scan accept(Shuttle shuttle) {
@@ -1188,16 +1224,17 @@ public static class Scan extends FromStep {
 
     @Override protected AstWriter unparse(AstWriter w, From from, int ordinal,
         int left, int right) {
-      final String prefix = ordinal == 0 ? " "
-          : op == Op.SUCH_THAT ? " join "
-          : op.padded;
-      final String infix = op == Op.SUCH_THAT ? " suchthat "
-          : " in ";
-      w.append(prefix)
-          // for these purposes 'in' and 'suchthat' have same precedence as '='
-          .append(pat, 0, Op.EQ.left)
-          .append(infix)
-          .append(exp, Op.EQ.right, 0);
+      w.append(ordinal == 0 ? " " : op.padded)
+          // for these purposes 'in' has same precedence as '='
+          .append(pat, 0, Op.EQ.left);
+      if (Extents.isInfinite(exp)) {
+        // Print "from x : int" rather "from x in extent 'int'"
+        w.append(" : ")
+            .append(((ListType) exp.type).elementType.moniker());
+      } else {
+        w.append(" in ")
+            .append(exp, Op.EQ.right, 0);
+      }
       if (!isLiteralTrue()) {
         w.append("on").append(condition, 0, 0);
       }