Skip to content

Commit

Permalink
Add DoubleSelectiveStreamReader
Browse files Browse the repository at this point in the history 
Benchmark                                       (typeSignature)  (withNulls)  Mode  Cnt  Score    Error  Units
BenchmarkSelectiveStreamReaders.read                     double         true  avgt   20  0.151 ±  0.008   s/op
BenchmarkSelectiveStreamReaders.read                     double        false  avgt   20  0.255 ±  0.004   s/op
BenchmarkSelectiveStreamReaders.readAllNull              double          N/A  avgt   20  0.003 ±  0.001   s/op
BenchmarkSelectiveStreamReaders.readWithFilter           double         true  avgt   20  0.193 ±  0.008   s/op
BenchmarkSelectiveStreamReaders.readWithFilter           double        false  avgt   20  0.336 ±  0.012   s/op
  • Loading branch information
@sayhar
Sahar Massachi authored and sayhar committed Aug 29, 2019
1 parent 0ea8208 commit 95b2b33
Show file tree
Hide file tree
Showing 6 changed files with 573 additions and 12 deletions.
56 changes: 46 additions & 10 deletions presto-hive/src/test/java/com/facebook/presto/hive/TestHivePushdownFilterQueries.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,17 +39,19 @@ public class TestHivePushdownFilterQueries
private static final Pattern ARRAY_SUBSCRIPT_PATTERN = Pattern.compile("([a-z_+]+)((\\[[0-9]+\\])+)");

private static final String WITH_LINEITEM_EX = "WITH lineitem_ex AS (\n" +
"SELECT linenumber, orderkey, partkey, suppkey, \n" +
"SELECT linenumber, orderkey, partkey, suppkey, quantity, extendedprice, tax, \n" +
" CASE WHEN linenumber % 5 = 0 THEN null ELSE shipmode = 'AIR' END AS ship_by_air, \n" +
" CASE WHEN linenumber % 7 = 0 THEN null ELSE returnflag = 'R' END AS is_returned, \n" +
" CASE WHEN linenumber % 4 = 0 THEN null ELSE CAST(day(shipdate) AS TINYINT) END AS ship_day, " +
" CASE WHEN linenumber % 6 = 0 THEN null ELSE CAST(month(shipdate) AS TINYINT) END AS ship_month, " +
" CASE WHEN linenumber % 3 = 0 THEN null ELSE CAST(shipdate AS TIMESTAMP) END AS ship_timestamp, \n" +
" CASE WHEN orderkey % 3 = 0 THEN null ELSE CAST(commitdate AS TIMESTAMP) END AS commit_timestamp, \n" +
" CASE WHEN orderkey % 5 = 0 THEN null ELSE CAST(discount AS REAL) END AS discount_real, \n" +
" CASE WHEN orderkey % 43 = 0 THEN null ELSE discount END as discount, \n" +
" CASE WHEN orderkey % 7 = 0 THEN null ELSE CAST(tax AS REAL) END AS tax_real, \n" +
" CASE WHEN linenumber % 2 = 0 THEN null ELSE (CAST(day(shipdate) AS TINYINT) , CAST(month(shipdate) AS TINYINT)) END AS ship_day_month, " +
" CASE WHEN orderkey % 11 = 0 THEN null ELSE (orderkey, partkey, suppkey) END AS keys, \n" +
" CASE WHEN orderkey % 41 = 0 THEN null ELSE (extendedprice, discount, tax) END AS doubles, \n" +
" CASE WHEN orderkey % 13 = 0 THEN null ELSE ((orderkey, partkey), (suppkey,), CASE WHEN orderkey % 17 = 0 THEN null ELSE (orderkey, partkey) END) END AS nested_keys, \n" +
" CASE WHEN orderkey % 17 = 0 THEN null ELSE (shipmode = 'AIR', returnflag = 'R') END as flags, \n" +
" CASE WHEN orderkey % 19 = 0 THEN null ELSE (CAST(discount AS REAL), CAST(tax AS REAL)) END as reals, \n" +
Expand All @@ -59,6 +61,7 @@ public class TestHivePushdownFilterQueries
" (CAST(day(commitdate) AS TINYINT), CAST(month(commitdate) AS TINYINT), CAST(year(commitdate) AS INTEGER)), " +
" (CAST(day(receiptdate) AS TINYINT), CAST(month(receiptdate) AS TINYINT), CAST(year(receiptdate) AS INTEGER))) END AS dates, \n" +
" CASE WHEN orderkey % 37 = 0 THEN null ELSE (CAST(shipdate AS TIMESTAMP), CAST(commitdate AS TIMESTAMP)) END AS timestamps \n" +

"FROM lineitem)\n";

protected TestHivePushdownFilterQueries()
Expand All @@ -76,30 +79,31 @@ private static QueryRunner createQueryRunner()
Optional.empty());

queryRunner.execute(noPushdownFilter(queryRunner.getDefaultSession()),
"CREATE TABLE lineitem_ex (linenumber, orderkey, partkey, suppkey, ship_by_air, is_returned, ship_day, ship_month, ship_timestamp, commit_timestamp, discount_real, tax_real, ship_day_month, keys, nested_keys, flags, reals, info, dates, timestamps) AS " +
"SELECT linenumber, orderkey, partkey, suppkey, " +
"CREATE TABLE lineitem_ex (linenumber, orderkey, partkey, suppkey, quantity, extendedprice, tax, ship_by_air, is_returned, ship_day, ship_month, ship_timestamp, commit_timestamp, discount_real, discount, tax_real, ship_day_month, keys, doubles, nested_keys, flags, reals, info, dates, timestamps) AS " +
"SELECT linenumber, orderkey, partkey, suppkey, quantity, extendedprice, tax, " +
" IF (linenumber % 5 = 0, null, shipmode = 'AIR') AS ship_by_air, " +
" IF (linenumber % 7 = 0, null, returnflag = 'R') AS is_returned, " +
" IF (linenumber % 4 = 0, null, CAST(day(shipdate) AS TINYINT)) AS ship_day, " +
" IF (linenumber % 6 = 0, null, CAST(month(shipdate) AS TINYINT)) AS ship_month, " +
" IF (linenumber % 3 = 0, null, CAST(shipdate AS TIMESTAMP)) AS ship_timestamp, " +
" IF (orderkey % 3 = 0, null, CAST(commitdate AS TIMESTAMP)) AS commit_timestamp, " +
" IF (orderkey % 5 = 0, null, CAST(discount AS REAL)) AS discount_real, " +
" IF (orderkey % 43 = 0, null, discount) AS discount, " +
" IF (orderkey % 7 = 0, null, CAST(tax AS REAL)) AS tax_real, " +
" IF (linenumber % 2 = 0, null, ARRAY[CAST(day(shipdate) AS TINYINT), CAST(month(shipdate) AS TINYINT)]) AS ship_day_month, " +
" IF (orderkey % 11 = 0, null, ARRAY[orderkey, partkey, suppkey]), " +
" IF (orderkey % 13 = 0, null, ARRAY[ARRAY[orderkey, partkey], ARRAY[suppkey], IF (orderkey % 17 = 0, null, ARRAY[orderkey, partkey])]), " +
" IF (orderkey % 17 = 0, null, ARRAY[shipmode = 'AIR', returnflag = 'R']), " +
" IF (orderkey % 11 = 0, null, ARRAY[orderkey, partkey, suppkey]) AS keys, " +
" IF (orderkey % 41 = 0, null, ARRAY[extendedprice, discount, tax]) AS doubles, " +
" IF (orderkey % 13 = 0, null, ARRAY[ARRAY[orderkey, partkey], ARRAY[suppkey], IF (orderkey % 17 = 0, null, ARRAY[orderkey, partkey])]) AS nested_keys, " +
" IF (orderkey % 17 = 0, null, ARRAY[shipmode = 'AIR', returnflag = 'R']) AS flags, " +
" IF (orderkey % 19 = 0, null, ARRAY[CAST(discount AS REAL), CAST(tax AS REAL)]), " +
" IF (orderkey % 23 = 0, null, CAST(ROW(orderkey, linenumber, ROW(day(shipdate), month(shipdate), year(shipdate))) " +
" AS ROW(orderkey BIGINT, linenumber INTEGER, shipdate ROW(ship_day TINYINT, ship_month TINYINT, ship_year INTEGER)))), " +
" IF (orderkey % 31 = 0, NULL, ARRAY[" +
" CAST(ROW(day(shipdate), month(shipdate), year(shipdate)) AS ROW(day TINYINT, month TINYINT, year INTEGER)), " +
" CAST(ROW(day(commitdate), month(commitdate), year(commitdate)) AS ROW(day TINYINT, month TINYINT, year INTEGER)), " +
" CAST(ROW(day(receiptdate), month(receiptdate), year(receiptdate)) AS ROW(day TINYINT, month TINYINT, year INTEGER))]) ," +
" CAST(ROW(day(receiptdate), month(receiptdate), year(receiptdate)) AS ROW(day TINYINT, month TINYINT, year INTEGER))]), " +
" IF (orderkey % 37 = 0, NULL, ARRAY[CAST(shipdate AS TIMESTAMP), CAST(commitdate AS TIMESTAMP)]) AS timestamps " +
"FROM lineitem");

return queryRunner;
}

Expand Down Expand Up @@ -199,6 +203,36 @@ public void testTimestamps()
assertQueryUsingH2Cte("SELECT count(*) from lineitem_ex where year(timestamps[1]) != year(timestamps[2])");
}

@Test
public void testDouble()
{
assertQuery("SELECT quantity, extendedprice, discount, tax FROM lineitem");

assertQueryUsingH2Cte("SELECT count(discount) FROM lineitem_ex");

assertFilterProject("discount IS NULL", "count(*)");

assertFilterProject("discount IS NOT NULL", "sum(quantity), sum(discount)");

assertFilterProject("is_returned = true", "quantity, extendedprice, discount");

assertFilterProject("quantity = 4", "orderkey, tax");

assertFilterProject("quantity = 4 AND discount = 0", "extendedprice, discount");

assertFilterProject("quantity = 4 AND discount = 0 AND tax = .05", "orderkey");

assertFilterProject("(discount + tax) < (quantity / 10)", "tax");

assertFilterProject("doubles[1] > 0.01", "count(*)");

// Compact values
assertFilterProject("discount + tax > .05 AND discount > .01 AND tax > .01", "tax");

// SucceedingPositionsToFail > 0 in readWithFilter
assertFilterProject("is_returned AND doubles[2] = .01 AND doubles[1] + discount > 0.10", "count(*)");
}

@Test
public void testFloats()
{
Expand Down Expand Up @@ -420,10 +454,12 @@ public void testArraysOfNulls()
" CAST(ARRAY[null, null, null] AS ARRAY(TINYINT)) tinyints, " +
" CAST(ARRAY[null, null, null] AS ARRAY(BOOLEAN)) booleans, " +
" CAST(ARRAY[null, null, null] AS ARRAY(TIMESTAMP)) timestamps, " +
" CAST(ARRAY[null, null, null] AS ARRAY(REAL)) floats " +
" CAST(ARRAY[null, null, null] AS ARRAY(REAL)) floats, " +
" CAST(ARRAY[null, null, null] AS ARRAY(DOUBLE)) doubles " +
"FROM lineitem");

List<String> columnNames = ImmutableList.of("bigints", "integers", "tinyints", "smallints", "booleans", "floats");
List<String> columnNames = ImmutableList.of(
"bigints", "integers", "tinyints", "smallints", "booleans", "floats", "doubles");
try {
for (String columnName : columnNames) {
assertQuerySucceeds(getSession(), format("SELECT count(*) FROM test_arrays_of_nulls WHERE %s[1] IS NOT NULL", columnName));
Expand Down
Loading

0 comments on commit 95b2b33

Please sign in to comment.