Merge pull request #62 from pdet/disable_compressed_materialization

Disable Compressed Materialization
substrait-io · Aug 3, 2023 · 0cd88fa · 0cd88fa
2 parents 9657ccc + 52fd072
commit 0cd88fa
Show file tree

Hide file tree

Showing 8 changed files with 24 additions and 37 deletions.
diff --git a/duckdb b/duckdb
diff --git a/src/substrait_extension.cpp b/src/substrait_extension.cpp
@@ -97,6 +97,9 @@ static void ToSubFunctionInternal(ClientContext &context, ToSubstraitFunctionDat
 	// We might want to disable the optimizer of our new connection
 	new_conn.context->config.enable_optimizer = data.enable_optimizer;
 	new_conn.context->config.use_replacement_scans = false;
+	// We want for sure to disable the internal compression optimizations
+	// These are DuckDB specific, no other system implements these
+	new_conn.Query("SET disabled_optimizers to 'compressed_materialization';");
 	query_plan = new_conn.context->ExtractPlan(data.query);
 	DuckDBToSubstrait transformer_d2s(context, *query_plan);
 	serialized = transformer_d2s.SerializeToString();
@@ -134,6 +137,9 @@ static void ToJsonFunctionInternal(ClientContext &context, ToSubstraitFunctionDa
 	// We might want to disable the optimizer of our new connection
 	new_conn.context->config.enable_optimizer = data.enable_optimizer;
 	new_conn.context->config.use_replacement_scans = false;
+	// We want for sure to disable the internal compression optimizations
+	// These are DuckDB specific, no other system implements these
+	new_conn.Query("SET disabled_optimizers to 'compressed_materialization';");
 	query_plan = new_conn.context->ExtractPlan(data.query);
 	DuckDBToSubstrait transformer_d2s(context, *query_plan);
 	serialized = transformer_d2s.SerializeToJson();

diff --git a/test/python/test_set_operation_relation.py b/test/python/test_set_operation_relation.py
@@ -46,15 +46,14 @@ def test_except(self, connection):
 				(3, 4, 5, 6)
 			) as tbl(B, C, D, A)
 		""")
-		# FIXME: test currently fails
-		# query = """
-		# 	select * from tbl1 EXCEPT (select * from tbl2);
-		# """
-		# expected = connection.sql(query).fetchall()
-		# json = connection.get_substrait_json(query).fetchall()[0][0]
-		# rel = connection.from_substrait_json(json)
-		# actual = rel.fetchall()
-		# assert expected == actual
+		query = """
+			select * from tbl1 EXCEPT (select * from tbl2);
+		"""
+		expected = connection.sql(query).fetchall()
+		json = connection.get_substrait_json(query).fetchall()[0][0]
+		rel = connection.from_substrait_json(json)
+		actual = rel.fetchall()
+		assert expected == actual
 
 	def test_intersect(self, connection):
 		connection.execute("""
@@ -69,12 +68,11 @@ def test_intersect(self, connection):
 				(3, 4, 5, 6)
 			) as tbl(B, C, D, A)
 		""")
-		# FIXME: test currently fails
-		# query = """
-		# 	select * from tbl1 INTERSECT (select * from tbl2);
-		# """
-		# expected = connection.sql(query).fetchall()
-		# json = connection.get_substrait_json(query).fetchall()[0][0]
-		# rel = connection.from_substrait_json(json)
-		# actual = rel.fetchall()
-		# assert expected == actual
+		query = """
+			select * from tbl1 INTERSECT (select * from tbl2);
+		"""
+		expected = connection.sql(query).fetchall()
+		json = connection.get_substrait_json(query).fetchall()[0][0]
+		rel = connection.from_substrait_json(json)
+		actual = rel.fetchall()
+		assert expected == actual
diff --git a/test/sql/test_except.test b/test/sql/test_except.test
@@ -8,10 +8,6 @@ statement ok
 PRAGMA enable_verification
 
 # Create two tables to use in the EXCEPT
-
-# FIXME: currently broken
-mode skip
-
 statement ok
 create table tbl1 as select * from (VALUES
 	(1, 2, 3, 4),

diff --git a/test/sql/test_intersect.test b/test/sql/test_intersect.test
@@ -7,11 +7,7 @@ require substrait
 statement ok
 PRAGMA enable_verification
 
-# Create two tables to use in the INTERSECTx
-
-# FIXME: currently broken with:
-mode skip
-
+# Create two tables to use in the INTERSECT
 statement ok
 create table tbl1 as select * from (VALUES
 	(1, 2, 3, 4),

diff --git a/test/sql/test_substrait_parquet.test b/test/sql/test_substrait_parquet.test
@@ -34,9 +34,6 @@ CALL from_substrait('\x12\x09\x1A\x07\x10\x01\x1A\x03sum\x12\x07\x1A\x05\x10\x02
 ----
 19107076.83379995
 
-# TODO FIXME
-mode skip
-
 # Test Globbing
 statement ok
 CALL get_substrait('select * from parquet_scan(''data/parquet-testing/glob*/t?.parquet'') order by i')

diff --git a/test/sql/test_substrait_subqueries.test b/test/sql/test_substrait_subqueries.test
@@ -7,9 +7,6 @@ require substrait
 statement ok
 PRAGMA enable_verification
 
-# FIXME: currently broken
-mode skip
-
 statement ok
 CREATE TABLE integers (i integer);
 

diff --git a/test/sql/test_substrait_tpch.test b/test/sql/test_substrait_tpch.test
@@ -12,9 +12,6 @@ PRAGMA enable_verification
 statement ok
 CALL dbgen(sf=0.01)
 
-# FIXME: currently broken
-mode skip
-
 #Q 01
 statement ok
 CALL get_substrait('SELECT l_returnflag, l_linestatus, sum(l_quantity) AS sum_qty, sum(l_extendedprice) AS sum_base_price, sum(l_extendedprice * (1 - l_discount)) AS sum_disc_price, sum(l_extendedprice * (1 - l_discount) * (1 + l_tax)) AS sum_charge, avg(l_quantity) AS avg_qty, avg(l_extendedprice) AS avg_price, avg(l_discount) AS avg_disc, count(*) AS count_order FROM lineitem WHERE l_shipdate <= CAST(''1998-09-02'' AS date) GROUP BY l_returnflag, l_linestatus ORDER BY l_returnflag, l_linestatus;')