Support translating Spark SQL to Calcite RelNode

linkedin · Mar 31, 2024 · a77bf29 · a77bf29
1 parent c0c151a
commit a77bf29
Show file tree

Hide file tree

Showing 42 changed files with 1,523 additions and 27 deletions.
diff --git a/coral-dbt/src/main/resources/tests/__pycache__/test_incremental.cpython-312.pyc b/coral-dbt/src/main/resources/tests/__pycache__/test_incremental.cpython-312.pyc
diff --git a/coral-spark/build.gradle b/coral-spark/build.gradle
@@ -3,10 +3,15 @@ apply from: "spark_itest.gradle"
 dependencies {
   compile project(':coral-hive')
   compile project(':coral-schema')
+  compile('org.apache.spark:spark-sql_2.13:3.2.0') {
+    exclude group: 'com.fasterxml.jackson', module: 'jackson-bom'
+    exclude group: 'org.apache.avro', module: 'avro-mapred'
+  }
 
 
   compileOnly deps.'spark'.'sql'
 
+  testCompile project(':coral-trino')
   testCompile(deps.'hive'.'hive-exec-core') {
     exclude group: 'org.apache.avro', module: 'avro-tools'
     // These exclusions are required to prevent duplicate classes since we include

diff --git a/coral-spark/src/main/java/com/linkedin/coral/spark/AddExplicitAlias.java b/coral-spark/src/main/java/com/linkedin/coral/spark/AddExplicitAlias.java
@@ -1,5 +1,5 @@
 /**
- * Copyright 2022 LinkedIn Corporation. All rights reserved.
+ * Copyright 2022-2024 LinkedIn Corporation. All rights reserved.
  * Licensed under the BSD-2 Clause license.
  * See LICENSE in the project root for license information.
  */
@@ -11,7 +11,12 @@
 
 import com.google.common.base.Preconditions;
 
-import org.apache.calcite.sql.*;
+import org.apache.calcite.sql.SqlCall;
+import org.apache.calcite.sql.SqlIdentifier;
+import org.apache.calcite.sql.SqlKind;
+import org.apache.calcite.sql.SqlNode;
+import org.apache.calcite.sql.SqlNodeList;
+import org.apache.calcite.sql.SqlSelect;
 import org.apache.calcite.sql.fun.SqlStdOperatorTable;
 import org.apache.calcite.sql.parser.SqlParserPos;
 import org.apache.calcite.sql.util.SqlShuttle;

diff --git a/coral-spark/src/main/java/com/linkedin/coral/spark/CoralSpark.java b/coral-spark/src/main/java/com/linkedin/coral/spark/CoralSpark.java
@@ -1,5 +1,5 @@
 /**
- * Copyright 2018-2023 LinkedIn Corporation. All rights reserved.
+ * Copyright 2018-2024 LinkedIn Corporation. All rights reserved.
  * Licensed under the BSD-2 Clause license.
  * See LICENSE in the project root for license information.
  */

diff --git a/coral-spark/src/main/java/com/linkedin/coral/spark/CoralSqlNodeToSparkSqlNodeConverter.java b/coral-spark/src/main/java/com/linkedin/coral/spark/CoralSqlNodeToSparkSqlNodeConverter.java
@@ -1,5 +1,5 @@
 /**
- * Copyright 2022-2023 LinkedIn Corporation. All rights reserved.
+ * Copyright 2022-2024 LinkedIn Corporation. All rights reserved.
  * Licensed under the BSD-2 Clause license.
  * See LICENSE in the project root for license information.
  */

diff --git a/coral-spark/src/main/java/com/linkedin/coral/spark/CoralToSparkSqlCallConverter.java b/coral-spark/src/main/java/com/linkedin/coral/spark/CoralToSparkSqlCallConverter.java
@@ -1,5 +1,5 @@
 /**
- * Copyright 2023 LinkedIn Corporation. All rights reserved.
+ * Copyright 2023-2024 LinkedIn Corporation. All rights reserved.
  * Licensed under the BSD-2 Clause license.
  * See LICENSE in the project root for license information.
  */

diff --git a/coral-spark/src/main/java/com/linkedin/coral/spark/IRRelToSparkRelTransformer.java b/coral-spark/src/main/java/com/linkedin/coral/spark/IRRelToSparkRelTransformer.java
@@ -1,5 +1,5 @@
 /**
- * Copyright 2018-2023 LinkedIn Corporation. All rights reserved.
+ * Copyright 2018-2024 LinkedIn Corporation. All rights reserved.
  * Licensed under the BSD-2 Clause license.
  * See LICENSE in the project root for license information.
  */

diff --git a/coral-spark/src/main/java/com/linkedin/coral/spark/SparkSqlRewriter.java b/coral-spark/src/main/java/com/linkedin/coral/spark/SparkSqlRewriter.java
@@ -1,5 +1,5 @@
 /**
- * Copyright 2018-2023 LinkedIn Corporation. All rights reserved.
+ * Copyright 2018-2024 LinkedIn Corporation. All rights reserved.
  * Licensed under the BSD-2 Clause license.
  * See LICENSE in the project root for license information.
  */

diff --git a/coral-spark/src/main/java/com/linkedin/coral/spark/containers/SparkRelInfo.java b/coral-spark/src/main/java/com/linkedin/coral/spark/containers/SparkRelInfo.java
@@ -1,5 +1,5 @@
 /**
- * Copyright 2018-2023 LinkedIn Corporation. All rights reserved.
+ * Copyright 2018-2024 LinkedIn Corporation. All rights reserved.
  * Licensed under the BSD-2 Clause license.
  * See LICENSE in the project root for license information.
  */

diff --git a/coral-spark/src/main/java/com/linkedin/coral/spark/containers/SparkUDFInfo.java b/coral-spark/src/main/java/com/linkedin/coral/spark/containers/SparkUDFInfo.java
@@ -1,5 +1,5 @@
 /**
- * Copyright 2018-2022 LinkedIn Corporation. All rights reserved.
+ * Copyright 2018-2024 LinkedIn Corporation. All rights reserved.
  * Licensed under the BSD-2 Clause license.
  * See LICENSE in the project root for license information.
  */

diff --git a/coral-spark/src/main/java/com/linkedin/coral/spark/dialect/SparkSqlDialect.java b/coral-spark/src/main/java/com/linkedin/coral/spark/dialect/SparkSqlDialect.java
@@ -1,5 +1,5 @@
 /**
- * Copyright 2018-2023 LinkedIn Corporation. All rights reserved.
+ * Copyright 2018-2024 LinkedIn Corporation. All rights reserved.
  * Licensed under the BSD-2 Clause license.
  * See LICENSE in the project root for license information.
  */

diff --git a/coral-spark/src/main/java/com/linkedin/coral/spark/exceptions/UnsupportedUDFException.java b/coral-spark/src/main/java/com/linkedin/coral/spark/exceptions/UnsupportedUDFException.java
@@ -1,5 +1,5 @@
 /**
- * Copyright 2019-2021 LinkedIn Corporation. All rights reserved.
+ * Copyright 2019-2024 LinkedIn Corporation. All rights reserved.
  * Licensed under the BSD-2 Clause license.
  * See LICENSE in the project root for license information.
  */

diff --git a/coral-spark/src/main/java/com/linkedin/coral/spark/functions/SqlLateralJoin.java b/coral-spark/src/main/java/com/linkedin/coral/spark/functions/SqlLateralJoin.java
@@ -1,5 +1,5 @@
 /**
- * Copyright 2018-2022 LinkedIn Corporation. All rights reserved.
+ * Copyright 2018-2024 LinkedIn Corporation. All rights reserved.
  * Licensed under the BSD-2 Clause license.
  * See LICENSE in the project root for license information.
  */

diff --git a/coral-spark/src/main/java/com/linkedin/coral/spark/functions/SqlLateralViewAsOperator.java b/coral-spark/src/main/java/com/linkedin/coral/spark/functions/SqlLateralViewAsOperator.java
@@ -1,5 +1,5 @@
 /**
- * Copyright 2018-2022 LinkedIn Corporation. All rights reserved.
+ * Copyright 2018-2024 LinkedIn Corporation. All rights reserved.
  * Licensed under the BSD-2 Clause license.
  * See LICENSE in the project root for license information.
  */