Skip to content

Commit

Permalink
More progress
Browse files Browse the repository at this point in the history
  • Loading branch information
@julianhyde
julianhyde committed Dec 12, 2023
1 parent 6dfba5b commit 728923c
Show file tree
Hide file tree
Showing 7 changed files with 162 additions and 68 deletions.
11 changes: 9 additions & 2 deletions src/main/java/net/hydromatic/morel/compile/Environment.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,10 +19,12 @@
package net.hydromatic.morel.compile;

import net.hydromatic.morel.ast.Core;
import net.hydromatic.morel.eval.Codes;
import net.hydromatic.morel.eval.EvalEnv;
import net.hydromatic.morel.eval.Unit;
import net.hydromatic.morel.type.Binding;
import net.hydromatic.morel.type.Type;
import net.hydromatic.morel.type.TypeSystem;

import org.checkerframework.checker.nullness.qual.Nullable;

Expand Down Expand Up @@ -85,11 +87,16 @@ protected Environment bind(Binding binding) {

/** Calls a consumer for each variable and its type.
* Does not visit obscured bindings. */
public void forEachType(BiConsumer<String, Type> consumer) {
public void forEachType(TypeSystem typeSystem,
BiConsumer<String, Type> consumer) {
final Set<String> names = new HashSet<>();
visit(binding -> {
if (names.add(binding.id.name)) {
consumer.accept(binding.id.name, binding.id.type);
final Type type =
binding.value instanceof Codes.TypedValue
? ((Codes.TypedValue) binding.value).typeKey().toType(typeSystem)
: binding.id.type;
consumer.accept(binding.id.name, type);
}
});
}
Expand Down
36 changes: 30 additions & 6 deletions src/main/java/net/hydromatic/morel/compile/Resolver.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -408,6 +408,12 @@ private Core.Apply toCore(Ast.Apply apply) {
final Ast.RecordSelector recordSelector = (Ast.RecordSelector) apply.fn;
coreFn = core.recordSelector(typeMap.typeSystem,
makeProgressive(coreArg), recordSelector.name);
if (type.op() == Op.TY_VAR
&& coreFn.type.op() == Op.FUNCTION_TYPE) {
// If we are dereferencing a field in a progressive type, the type
// available now may be more precise than the deduced type.
type = ((FnType) coreFn.type).resultType;
}
} else {
coreFn = toCore(apply.fn);
}
Expand All @@ -423,9 +429,10 @@ private RecordLikeType makeProgressive(Core.Exp coreArg) {
Object o = valueOf(coreArg);
if (o instanceof Codes.TypedValue) {
final Codes.TypedValue typedValue = (Codes.TypedValue) o;
final Type type = typedValue.typeKey().toType(typeMap.typeSystem);
if (type instanceof RecordLikeType) {
final RecordLikeType recordLikeType2 = (RecordLikeType) type;
final Type.Key typeKey = typedValue.typeKey();
if (typeKey.op == Op.PROGRESSIVE_RECORD_TYPE) {
final RecordLikeType recordLikeType2 =
(RecordLikeType) typeKey.toType(typeMap.typeSystem);
return new RecordLikeType() {
@Override public boolean isProgressive() {
return true;
Expand Down Expand Up @@ -474,13 +481,30 @@ Object valueOf(Core.Exp exp) {
if (exp instanceof Core.Literal) {
return ((Core.Literal) exp).value;
}
if (exp instanceof Core.Id) {
Binding binding = env.getOpt(((Core.Id) exp).idPat);
if (exp.op == Op.ID) {
final Core.Id id = (Core.Id) exp;
Binding binding = env.getOpt(id.idPat);
if (binding != null) {
return binding.value;
}
}
// TODO: field
if (exp.op == Op.APPLY) {
final Core.Apply apply = (Core.Apply) exp;
if (apply.fn.op == Op.RECORD_SELECTOR) {
final Core.RecordSelector recordSelector =
(Core.RecordSelector) apply.fn;
final Object o = valueOf(apply.arg);
@SuppressWarnings("unchecked") List<Object> list =
o instanceof List
? (List<Object>) o
: o instanceof Codes.TypedValue
? ((Codes.TypedValue) o).valueAs(List.class)
: null;
if (list != null) {
return list.get(recordSelector.slot);
}
}
}
return null; // not constant
}

Expand Down
2 changes: 1 addition & 1 deletion src/main/java/net/hydromatic/morel/compile/TypeResolver.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ private Resolved deduceType_(Environment env, Ast.Decl decl) {
});
BuiltIn.forEachStructure(typeSystem, (structure, type) ->
typeEnvs.accept(structure.name, type));
env.forEachType(typeEnvs);
env.forEachType(typeSystem, typeEnvs);
final TypeEnv typeEnv = typeEnvs.typeEnv;
final Map<Ast.IdPat, Unifier.Term> termMap = new LinkedHashMap<>();
final Ast.Decl node2 = deduceDeclType(typeEnv, decl, termMap);
Expand Down
160 changes: 109 additions & 51 deletions src/main/java/net/hydromatic/morel/eval/Directory.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,21 +20,25 @@

import net.hydromatic.morel.type.Keys;
import net.hydromatic.morel.type.PrimitiveType;
import net.hydromatic.morel.type.ProgressiveRecordType;
import net.hydromatic.morel.type.RecordType;
import net.hydromatic.morel.type.Type;
import net.hydromatic.morel.type.TypeSystem;
import net.hydromatic.morel.util.ImmutablePairList;
import net.hydromatic.morel.util.PairList;

import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSortedMap;
import com.google.common.collect.Maps;
import org.apache.calcite.util.Util;
import org.checkerframework.checker.nullness.qual.Nullable;

import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.SortedMap;
import java.util.TreeMap;

import static java.util.Objects.requireNonNull;

Expand All @@ -45,8 +49,7 @@
*/
public class Directory implements Codes.TypedValue {
private final File file;
private final SortedMap<String, Codes.TypedValue> entries =
new TreeMap<>(RecordType.ORDERING);
private @Nullable SortedMap<String, Codes.TypedValue> entries = null;

/** Creates a Directory in the default directory. */
public Directory() {
Expand All @@ -60,87 +63,142 @@ public Directory(File file) {

@Override public <V> V valueAs(Class<V> clazz) {
if (clazz.isAssignableFrom(ImmutableList.class)) {
return clazz.cast(ImmutableList.copyOf(entries.values()));
return clazz.cast(
entries == null
? ImmutableList.of()
: ImmutableList.copyOf(entries.values()));
}
throw new IllegalArgumentException("not a " + clazz);
}

@Override public Type.Key typeKey() {
if (entries == null) {
return Keys.progressiveRecord(
ProgressiveRecordType.DefaultHandler.INSTANCE,
ImmutableSortedMap.of());
}
return Keys.record(Maps.transformValues(entries, Codes.TypedValue::typeKey));
}

@Override public boolean discoverField(TypeSystem typeSystem, String fieldName) {
if (entries != null) {
return false; // already populated
}
if (!file.isDirectory()) {
return false;
}

entries.clear();
final ImmutableSortedMap.Builder<String, Codes.TypedValue> entries =
ImmutableSortedMap.orderedBy(RecordType.ORDERING);
for (File subFile : Util.first(file.listFiles(), new File[0])) {
if (subFile.isDirectory()) {
entries.put(subFile.getName(), new Directory(subFile));
} else if (subFile.isFile()) {
Type.Key typeKey;
String name;
if (subFile.getName().endsWith(".csv")) {
try (BufferedReader r = Util.reader(subFile)) {
String firstLine = r.readLine();
if (firstLine == null) {
typeKey = PrimitiveType.UNIT.key();
} else {
PairList<String, Type.Key> nameTypes = PairList.of();
for (String field : firstLine.split(",")) {
final String[] split = field.split(":");
final String subFieldName = split[0];
final String subFieldType =
split.length > 1 ? split[1] : "string";
Type.Key subType;
switch (subFieldType) {
case "bool":
subType = PrimitiveType.BOOL.key();
break;
case "decimal":
case "double":
subType = PrimitiveType.REAL.key();
break;
case "int":
subType = PrimitiveType.INT.key();
break;
default:
subType = PrimitiveType.STRING.key();
break;
}
nameTypes.add(subFieldName, subType);
}
typeKey =
Keys.list(
Keys.record(
ImmutableSortedMap
.<String, Type.Key>orderedBy(RecordType.ORDERING)
.putAll(nameTypes)
.build()));
}
entries.put(subFile.getName(), new DataFile(subFile, typeKey));
} catch (IOException e) {
// ignore, and skip file
name = removeSuffix(subFile.getName(), ".csv");
final DataFile dataFile = DataFile.create(subFile);
if (dataFile != null) {
entries.put(name, dataFile);
}
}
}
}
this.entries = entries.build();
return true;
}

private static String removeSuffix(String name, String suffix) {
if (!name.endsWith(suffix)) {
throw new IllegalArgumentException(name + " does not end with " + suffix);
}
return name.substring(0, name.length() - suffix.length());
}

private static class DataFile implements Codes.TypedValue {
private final Type.Key typeKey;
private final File file;
private final ImmutablePairList<String, Type.Key> nameTypes;

DataFile(File file, Type.Key typeKey) {
this.typeKey = typeKey;
DataFile(File file, PairList<String, Type.Key> nameTypes) {
this.file = file;
this.nameTypes = nameTypes.immutable();
}

static DataFile create(File subFile) {
try (BufferedReader r = Util.reader(subFile)) {
String firstLine = r.readLine();
PairList<String, Type.Key> nameTypes = PairList.of();
if (firstLine == null) {
// File is empty. There will be no fields, and row type will be unit.
} else {
for (String field : firstLine.split(",")) {
final String[] split = field.split(":");
final String subFieldName = split[0];
final String subFieldType =
split.length > 1 ? split[1] : "string";
Type.Key subType;
switch (subFieldType) {
case "bool":
subType = PrimitiveType.BOOL.key();
break;
case "decimal":
case "double":
subType = PrimitiveType.REAL.key();
break;
case "int":
subType = PrimitiveType.INT.key();
break;
default:
subType = PrimitiveType.STRING.key();
break;
}
nameTypes.add(subFieldName, subType);
}
}
return new DataFile(subFile, nameTypes);
} catch (IOException e) {
// ignore, and skip file
return null;
}
}

@Override public <V> V valueAs(Class<V> clazz) {
return null; // TODO parse file into records
Object[] values = new Object[nameTypes.size()];
List<String> sortedFieldNames =
RecordType.ORDERING.immutableSortedCopy(nameTypes.leftList());
try (BufferedReader r = Util.reader(file)) {
String firstLine = r.readLine();
if (firstLine == null) {
return null;
}
final List<List<Object>> list = new ArrayList<>();
for (;;) {
String line = r.readLine();
if (line == null) {
return clazz.cast(list);
}
String[] fields = line.split(",");
for (int i = 0; i < values.length; i++) {
final int j = nameTypes.leftList().indexOf(sortedFieldNames.get(i));
String field = fields[j];
Type.Key typeKey = nameTypes.rightList().get(j);
values[i] = Integer.parseInt(field);
}
list.add(ImmutableList.copyOf(values));
}
} catch (IOException e) {
// ignore
}
return null;
}

@Override public Type.Key typeKey() {
return typeKey;
return Keys.list(
Keys.record(
ImmutableSortedMap
.<String, Type.Key>orderedBy(RecordType.ORDERING)
.putAll(nameTypes)
.build()));
}
}
}
Expand Down
11 changes: 6 additions & 5 deletions src/main/java/net/hydromatic/morel/type/Keys.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,12 +139,13 @@ public static List<Type.Key> toKeys(List<? extends Type> types) {
/** Describes a record, progressive record, or tuple type. */
static StringBuilder describeRecordType(StringBuilder buf, int left,
int right, SortedMap<String, Type.Key> argNameTypes, Op op) {
final boolean progressive =
op == Op.PROGRESSIVE_RECORD_TYPE
|| argNameTypes.containsKey(ProgressiveRecordType.DUMMY);
switch (argNameTypes.size()) {
case 0:
if (op != Op.PROGRESSIVE_RECORD_TYPE) {
return buf.append("()");
}
// fall through
return buf.append(progressive ? "{...}" : "()");

default:
if (op == Op.TUPLE_TYPE) {
return TypeSystem.unparseList(buf, Op.TIMES, left, right,
Expand All @@ -167,7 +168,7 @@ static StringBuilder describeRecordType(StringBuilder buf, int left,
.append(':')
.append(typeKey);
}
if (op == Op.PROGRESSIVE_RECORD_TYPE) {
if (progressive) {
if (i > 0) {
buf.append(", ");
}
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/net/hydromatic/morel/type/Type.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ default Type substitute(TypeSystem typeSystem, List<? extends Type> types) {

/** Structural identifier of a type. */
abstract class Key {
final Op op;
public final Op op;

/** Creates a key. */
protected Key(Op op) {
Expand Down
8 changes: 6 additions & 2 deletions src/test/resources/script/file.smli
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,15 @@ file;

(*) The "data" subdirectory can be accessed as a field.
file.data;
> val it = {} : {...}

(*) Now we've done into the directory, its type has widened.
file;
> val it = {data={},script={},use={}} : {data:{...}, script:{...}, use:{...}}

file.data.scott;
file.data.scott.emp;

> val it = {} : {...}
file.data.scott.dept;
> val it = {} : {...}
file.data.scott;
(*) End file.smli

0 comments on commit 728923c

Please sign in to comment.