QST (deephaven#715)

jmao-denver · Nov 23, 2021 · 2cdbc58 · 2cdbc58
1 parent 17b94a0
commit 2cdbc58
Show file tree

Hide file tree

Showing 175 changed files with 8,039 additions and 83 deletions.
diff --git a/DB/DB.gradle b/DB/DB.gradle
@@ -57,6 +57,7 @@ dependencies {
             project(':open-api-lang-api'),
             project(':Container'),
             project(':table-api'),
+            project(':qst'),
             "org.flywaydb:flyway-core:5.2.4",
             'com.github.f4b6a3:uuid-creator:3.6.0'
 

diff --git a/DB/src/main/java/io/deephaven/db/tables/ColumnDefinition.java b/DB/src/main/java/io/deephaven/db/tables/ColumnDefinition.java
@@ -5,20 +5,45 @@
 package io.deephaven.db.tables;
 
 import io.deephaven.base.Copyable;
+import io.deephaven.base.formatters.EnumFormatter;
 import io.deephaven.base.log.LogOutput;
 import io.deephaven.base.log.LogOutputAppendable;
-import io.deephaven.base.formatters.EnumFormatter;
 import io.deephaven.datastructures.util.HashCodeUtil;
-import io.deephaven.db.tables.dbarrays.*;
+import io.deephaven.db.tables.dbarrays.DbArray;
+import io.deephaven.db.tables.dbarrays.DbArrayBase;
+import io.deephaven.db.tables.dbarrays.DbBooleanArray;
+import io.deephaven.db.tables.dbarrays.DbByteArray;
+import io.deephaven.db.tables.dbarrays.DbCharArray;
+import io.deephaven.db.tables.dbarrays.DbDoubleArray;
+import io.deephaven.db.tables.dbarrays.DbFloatArray;
+import io.deephaven.db.tables.dbarrays.DbIntArray;
+import io.deephaven.db.tables.dbarrays.DbLongArray;
+import io.deephaven.db.tables.dbarrays.DbShortArray;
 import io.deephaven.db.tables.utils.DBDateTime;
+import io.deephaven.qst.column.header.ColumnHeader;
+import io.deephaven.qst.type.BooleanType;
+import io.deephaven.qst.type.ByteType;
+import io.deephaven.qst.type.CharType;
+import io.deephaven.qst.type.CustomType;
+import io.deephaven.qst.type.DoubleType;
+import io.deephaven.qst.type.FloatType;
+import io.deephaven.qst.type.GenericType;
+import io.deephaven.qst.type.InstantType;
+import io.deephaven.qst.type.IntType;
+import io.deephaven.qst.type.LongType;
+import io.deephaven.qst.type.PrimitiveType;
+import io.deephaven.qst.type.ShortType;
+import io.deephaven.qst.type.StringType;
+import io.deephaven.qst.type.Type;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
 import java.io.Externalizable;
 import java.io.IOException;
 import java.io.ObjectInput;
 import java.io.ObjectOutput;
-import java.util.*;
+import java.util.List;
+import java.util.Objects;
 
 /**
  * Column definition for all Deephaven columns.
@@ -189,6 +214,89 @@ public static <T> ColumnDefinition<T> fromGenericType(String name, Class<T> data
         return cd;
     }
 
+    public static ColumnDefinition<?> from(ColumnHeader<?> header) {
+        return header.type().walk(new ColumnHeaderTranslation(header)).getOut();
+    }
+
+    private static class ColumnHeaderTranslation implements Type.Visitor, PrimitiveType.Visitor, GenericType.Visitor {
+
+        private final ColumnHeader<?> in;
+        private ColumnDefinition<?> out;
+
+        public ColumnHeaderTranslation(ColumnHeader<?> in) {
+            this.in = Objects.requireNonNull(in);
+        }
+
+        public ColumnDefinition<?> getOut() {
+            return Objects.requireNonNull(out);
+        }
+
+        @Override
+        public void visit(PrimitiveType<?> primitiveType) {
+            primitiveType.walk((PrimitiveType.Visitor) this);
+        }
+
+        @Override
+        public void visit(GenericType<?> genericType) {
+            genericType.walk((GenericType.Visitor) this);
+        }
+
+        @Override
+        public void visit(BooleanType booleanType) {
+            out = ofBoolean(in.name());
+        }
+
+        @Override
+        public void visit(ByteType byteType) {
+            out = ofByte(in.name());
+        }
+
+        @Override
+        public void visit(CharType charType) {
+            out = ofChar(in.name());
+        }
+
+        @Override
+        public void visit(ShortType shortType) {
+            out = ofShort(in.name());
+        }
+
+        @Override
+        public void visit(IntType intType) {
+            out = ofInt(in.name());
+        }
+
+        @Override
+        public void visit(LongType longType) {
+            out = ofLong(in.name());
+        }
+
+        @Override
+        public void visit(FloatType floatType) {
+            out = ofFloat(in.name());
+        }
+
+        @Override
+        public void visit(DoubleType doubleType) {
+            out = ofDouble(in.name());
+        }
+
+        @Override
+        public void visit(StringType stringType) {
+            out = ofString(in.name());
+        }
+
+        @Override
+        public void visit(InstantType instantType) {
+            out = ofTime(in.name());
+        }
+
+        @Override
+        public void visit(CustomType<?> customType) {
+            out = fromGenericType(in.name(), customType.clazz()); // todo, array types
+        }
+    }
+
     // needed for deserialization
     public ColumnDefinition() {
     }

diff --git a/DB/src/main/java/io/deephaven/db/tables/OperationsToTableImpl.java b/DB/src/main/java/io/deephaven/db/tables/OperationsToTableImpl.java
@@ -0,0 +1,12 @@
+package io.deephaven.db.tables;
+
+import io.deephaven.qst.TableCreation.OperationsToTable;
+
+enum OperationsToTableImpl implements OperationsToTable<Table, Table> {
+    INSTANCE;
+
+    @Override
+    public final Table of(Table table) {
+        return table;
+    }
+}
diff --git a/DB/src/main/java/io/deephaven/db/tables/Table.java b/DB/src/main/java/io/deephaven/db/tables/Table.java
@@ -30,6 +30,7 @@
 import io.deephaven.db.v2.select.SelectFilter;
 import io.deephaven.db.v2.sources.ColumnSource;
 import io.deephaven.db.v2.utils.Index;
+import io.deephaven.qst.table.TableSpec;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
@@ -44,6 +45,10 @@ public interface Table extends LongSizedDataStructure, LivenessNode, TableOperat
 
     Table[] ZERO_LENGTH_TABLE_ARRAY = new Table[0];
 
+    static Table of(TableSpec table) {
+        return TableCreationImpl.create(table);
+    }
+
     // -----------------------------------------------------------------------------------------------------------------
     // Metadata
     // -----------------------------------------------------------------------------------------------------------------
@@ -2284,10 +2289,10 @@ default Table sort(Collection<SortColumn> columnsToSortBy) {
     // Snapshot Operations
     // -----------------------------------------------------------------------------------------------------------------
 
-    Table snapshot(Table rightTable, boolean doInitialSnapshot, String... stampColumns);
+    Table snapshot(Table baseTable, boolean doInitialSnapshot, String... stampColumns);
 
-    default Table snapshot(Table rightTable, String... stampColumns) {
-        return snapshot(rightTable, true, stampColumns);
+    default Table snapshot(Table baseTable, String... stampColumns) {
+        return snapshot(baseTable, true, stampColumns);
     }
 
     Table snapshotIncremental(Table rightTable, boolean doInitialSnapshot, String... stampColumns);
@@ -2299,8 +2304,8 @@ default Table snapshotIncremental(Table rightTable, String... stampColumns) {
     Table snapshotHistory(final Table rightTable);
 
     @Override
-    default Table snapshot(Table rightTable, boolean doInitialSnapshot, Collection<ColumnName> stampColumns) {
-        return snapshot(rightTable, doInitialSnapshot, stampColumns.stream().map(ColumnName::name).toArray(String[]::new));
+    default Table snapshot(Table baseTable, boolean doInitialSnapshot, Collection<ColumnName> stampColumns) {
+        return snapshot(baseTable, doInitialSnapshot, stampColumns.stream().map(ColumnName::name).toArray(String[]::new));
     }
 
     // -----------------------------------------------------------------------------------------------------------------
@@ -2440,9 +2445,4 @@ default void releaseCachedResources() {
 
     @Deprecated
     void addColumnGrouping(String columnName);
-
-    @Override
-    default Table toTable() {
-        return this;
-    }
 }
diff --git a/DB/src/main/java/io/deephaven/db/tables/TableCreationImpl.java b/DB/src/main/java/io/deephaven/db/tables/TableCreationImpl.java
@@ -0,0 +1,67 @@
+package io.deephaven.db.tables;
+
+import io.deephaven.db.tables.utils.DBDateTime;
+import io.deephaven.db.tables.utils.DBTimeUtils;
+import io.deephaven.db.tables.utils.TableTools;
+import io.deephaven.db.v2.InMemoryTable;
+import io.deephaven.qst.TableCreation;
+import io.deephaven.qst.table.EmptyTable;
+import io.deephaven.qst.table.NewTable;
+import io.deephaven.qst.table.TableSpec;
+import io.deephaven.qst.table.TimeProvider;
+import io.deephaven.qst.table.TimeProviderSystem;
+import io.deephaven.qst.table.TimeTable;
+
+import java.util.Collection;
+import java.util.Objects;
+
+enum TableCreationImpl implements TableCreation<Table> {
+    INSTANCE;
+
+    public static Table create(TableSpec table) {
+        return TableCreation.create(INSTANCE, TableToOperationsImpl.INSTANCE, OperationsToTableImpl.INSTANCE, table);
+    }
+
+    @Override
+    public final Table of(NewTable newTable) {
+        return InMemoryTable.from(newTable);
+    }
+
+    @Override
+    public final Table of(EmptyTable emptyTable) {
+        return TableTools.emptyTable(emptyTable.size());
+    }
+
+    @Override
+    public final Table of(TimeTable timeTable) {
+        final io.deephaven.db.v2.utils.TimeProvider provider = TimeProviderAdapter
+            .of(timeTable.timeProvider());
+        final DBDateTime firstTime = timeTable.startTime().map(DBDateTime::of).orElse(null);
+        return TableTools.timeTable(provider, firstTime, timeTable.interval().toNanos());
+    }
+
+    @Override
+    public final Table merge(Collection<Table> tables) {
+        return TableTools.merge(tables);
+    }
+
+    static class TimeProviderAdapter implements TimeProvider.Visitor {
+
+        public static io.deephaven.db.v2.utils.TimeProvider of(TimeProvider provider) {
+            return provider.walk(new TimeProviderAdapter()).getOut();
+        }
+
+        private static final io.deephaven.db.v2.utils.TimeProvider SYSTEM_PROVIDER = DBTimeUtils::currentTime;
+
+        private io.deephaven.db.v2.utils.TimeProvider out;
+
+        public io.deephaven.db.v2.utils.TimeProvider getOut() {
+            return Objects.requireNonNull(out);
+        }
+
+        @Override
+        public void visit(TimeProviderSystem system) {
+            out = SYSTEM_PROVIDER;
+        }
+    }
+}
diff --git a/DB/src/main/java/io/deephaven/db/tables/TableDefinition.java b/DB/src/main/java/io/deephaven/db/tables/TableDefinition.java
@@ -12,6 +12,7 @@
 import io.deephaven.datastructures.util.HashCodeUtil;
 import io.deephaven.db.v2.InMemoryTable;
 import io.deephaven.db.v2.sources.ColumnSource;
+import io.deephaven.qst.column.header.ColumnHeader;
 import java.util.Map.Entry;
 import org.jetbrains.annotations.NotNull;
 
@@ -44,6 +45,15 @@ public static TableDefinition inferFrom(Map<String, ? extends ColumnSource> sour
         return new TableDefinition(definitions);
     }
 
+    public static TableDefinition from(Iterable<ColumnHeader<?>> headers) {
+        List<ColumnDefinition> definitions = new ArrayList<>();
+        for (ColumnHeader<?> columnHeader : headers) {
+            final ColumnDefinition<?> columnDefinition = ColumnDefinition.from(columnHeader);
+            definitions.add(columnDefinition);
+        }
+        return new TableDefinition(definitions);
+    }
+
     private transient Map<String, ColumnDefinition> columnNameMap;
 
     public TableDefinition() {

diff --git a/DB/src/main/java/io/deephaven/db/tables/TableToOperationsImpl.java b/DB/src/main/java/io/deephaven/db/tables/TableToOperationsImpl.java
@@ -0,0 +1,12 @@
+package io.deephaven.db.tables;
+
+import io.deephaven.qst.TableCreation.TableToOperations;
+
+enum TableToOperationsImpl implements TableToOperations<Table, Table> {
+    INSTANCE;
+
+    @Override
+    public final Table of(Table table) {
+        return table;
+    }
+}
diff --git a/DB/src/main/java/io/deephaven/db/tables/utils/DBDateTime.java b/DB/src/main/java/io/deephaven/db/tables/utils/DBDateTime.java
@@ -44,6 +44,10 @@ public final class DBDateTime implements Comparable<DBDateTime>, Externalizable
     private static final DateTimeFormatter dateTimeFormat = DateTimeFormat.forPattern("yyyy-MM-dd'T'HH:mm:ss.SSS");
     private static final DateTimeFormatter dateFormat = DateTimeFormat.forPattern("yyyy-MM-dd");
 
+    public static DBDateTime of(Instant instant) {
+        return new DBDateTime(DBTimeUtils.nanos(instant));
+    }
+
     /**
      * Create a new DBDatetime initialized to the epoch.
      */

diff --git a/DB/src/main/java/io/deephaven/db/tables/utils/DBTimeUtils.java b/DB/src/main/java/io/deephaven/db/tables/utils/DBTimeUtils.java
@@ -29,6 +29,7 @@
 import java.util.Date;
 import java.util.HashMap;
 import java.util.TimeZone;
+import java.util.concurrent.TimeUnit;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -190,6 +191,13 @@ public static long nanos(DBDateTime dateTime) {
         return dateTime.getNanos();
     }
 
+    public static long nanos(Instant instant) {
+        if (instant == null) {
+            return io.deephaven.util.QueryConstants.NULL_LONG;
+        }
+        return Math.addExact(TimeUnit.SECONDS.toNanos(instant.getEpochSecond()), instant.getNano());
+    }
+
     /**
      * Evaluates whether one {@link DBDateTime} value is earlier than a second {@link DBDateTime} value.
      * @param d1 The first {@link DBDateTime} value to compare.

diff --git a/DB/src/main/java/io/deephaven/db/tables/utils/TableTools.java b/DB/src/main/java/io/deephaven/db/tables/utils/TableTools.java
@@ -1282,6 +1282,20 @@ public static Table timeTable(String startTime, long periodNanos, ReplayerInterf
         return timeTable(DBTimeUtils.convertDateTime(startTime), periodNanos, replayer);
     }
 
+    /**
+     * Creates a table that adds a new row on a regular interval.
+     *
+     * @param timeProvider    the time provider
+     * @param startTime   start time for adding new rows
+     * @param periodNanos time interval between new row additions in nanoseconds.
+     * @return time table
+     */
+    public static Table timeTable(TimeProvider timeProvider, DBDateTime startTime, long periodNanos) {
+        final TimeTable timeTable = new TimeTable(timeProvider, startTime, periodNanos);
+        LiveTableMonitor.DEFAULT.addTable(timeTable);
+        return timeTable;
+    }
+
     // endregion time tables
 
     ///////////  Utilities For Merging Tables /////////////////

diff --git a/DB/src/main/java/io/deephaven/db/v2/HierarchicalTable.java b/DB/src/main/java/io/deephaven/db/v2/HierarchicalTable.java
@@ -323,7 +323,7 @@ public Table reverse() {
     }
 
     @Override
-    public Table snapshot(Table rightTable, boolean doInitialSnapshot, String... stampColumns) {
+    public Table snapshot(Table baseTable, boolean doInitialSnapshot, String... stampColumns) {
         return throwUnsupported("snapshot()");
     }