-
Notifications
You must be signed in to change notification settings - Fork 36
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Support function translation to Databricks SQL in TSql and Snowflake (#…
…414) Some functions must be translated from TSQL or Snowflake versions into the equivalent IR for Databricks SQL. In some cases the function must be translated in one dialect, say TSql but is equivalent in another, say Snowflake. Here we upgrade the FunctionBuilder system to be dialect aware, and provide a ConversionStrategy system that allows for any type of conversion from a simple name translation or more complicated specific IR representations when there is no equivalent. For example the TSQL code ```tsql SELECT ISNULL(x, 0) ``` Should translate to: ```sql SELECT IFNULL(x, 0) ``` In Databricks SQL, but in Snowflake SQL: ```snowflake SELECT ISNULL(col) ``` Is directly equivalent to Databricks SQL and needs no conversion. --------- Co-authored-by: Valentin Kasas <[email protected]>
- Loading branch information
Showing
14 changed files
with
768 additions
and
588 deletions.
There are no files selected for viewing
17 changes: 17 additions & 0 deletions
17
core/src/main/scala/com/databricks/labs/remorph/parsers/ConversionStrategy.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,17 @@ | ||
| package com.databricks.labs.remorph.parsers | ||
| import com.databricks.labs.remorph.parsers.{intermediate => ir} | ||
|
|
||
| import java.util.Locale | ||
|
|
||
| trait ConversionStrategy { | ||
| def convert(irName: String, args: Seq[ir.Expression]): ir.Expression | ||
| } | ||
|
|
||
| trait StringConverter { | ||
| // Preserves case if the original name was all lower case. Otherwise, converts to upper case. | ||
| // All bets are off if the original name was mixed case, but that is rarely seen in SQL and we are | ||
| // just making reasonable efforts here. | ||
| def convertString(irName: String, newName: String): String = { | ||
| if (irName.forall(_.isLower)) newName.toLowerCase(Locale.ROOT) else newName | ||
| } | ||
| } |
543 changes: 275 additions & 268 deletions
543
core/src/main/scala/com/databricks/labs/remorph/parsers/FunctionBuilder.scala
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
36 changes: 36 additions & 0 deletions
36
...c/main/scala/com/databricks/labs/remorph/parsers/snowflake/SnowflakeFunctionBuilder.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| package com.databricks.labs.remorph.parsers.snowflake | ||
|
|
||
| import com.databricks.labs.remorph.parsers.{ConversionStrategy, FunctionBuilder, FunctionDefinition, intermediate => ir} | ||
|
|
||
| class SnowflakeFunctionBuilder extends FunctionBuilder { | ||
|
|
||
| private val SnowflakeFunctionDefinitionPf: PartialFunction[String, FunctionDefinition] = { | ||
| case "IFNULL" => FunctionDefinition.standard(2) | ||
| case "ISNULL" => FunctionDefinition.standard(1) | ||
| } | ||
|
|
||
| override def functionDefinition(name: String): Option[FunctionDefinition] = | ||
| // If not found, check common functions | ||
| SnowflakeFunctionDefinitionPf.lift(name.toUpperCase()).orElse(super.functionDefinition(name)) | ||
|
|
||
| def applyConversionStrategy( | ||
| functionArity: FunctionDefinition, | ||
| args: Seq[ir.Expression], | ||
| irName: String): ir.Expression = { | ||
| functionArity.conversionStrategy match { | ||
| case Some(strategy) => strategy.convert(irName, args) | ||
| case _ => ir.CallFunction(irName, args) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| object SnowflakeFunctionConverters { | ||
|
|
||
| object FunctionRename extends ConversionStrategy { | ||
| override def convert(irName: String, args: Seq[ir.Expression]): ir.Expression = { | ||
| irName.toUpperCase() match { | ||
| case _ => ir.CallFunction(irName, args) | ||
| } | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
46 changes: 46 additions & 0 deletions
46
core/src/main/scala/com/databricks/labs/remorph/parsers/tsql/TSqlFunctionBuilder.scala
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,46 @@ | ||
| package com.databricks.labs.remorph.parsers.tsql | ||
|
|
||
| import com.databricks.labs.remorph.parsers.{ConversionStrategy, FunctionBuilder, FunctionDefinition, StringConverter, intermediate => ir} | ||
|
|
||
| class TSqlFunctionBuilder extends FunctionBuilder { | ||
|
|
||
| private val tSqlFunctionDefinitionPf: PartialFunction[String, FunctionDefinition] = { | ||
| case "@@CURSOR_STATUS" => FunctionDefinition.notConvertible(0) | ||
| case "@@FETCH_STATUS" => FunctionDefinition.notConvertible(0) | ||
| // The ConversionStrategy is used to rename ISNULL to IFNULL | ||
| case "ISNULL" => FunctionDefinition.standard(2).withConversionStrategy(TSqlFunctionConverters.FunctionRename) | ||
| case "MODIFY" => FunctionDefinition.xml(1) | ||
| } | ||
|
|
||
| override def functionDefinition(name: String): Option[FunctionDefinition] = | ||
| // If not found, check common functions | ||
| tSqlFunctionDefinitionPf.lift(name.toUpperCase()).orElse(super.functionDefinition(name)) | ||
|
|
||
| def applyConversionStrategy( | ||
| functionArity: FunctionDefinition, | ||
| args: Seq[ir.Expression], | ||
| irName: String): ir.Expression = { | ||
| functionArity.conversionStrategy match { | ||
| case Some(strategy) => strategy.convert(irName, args) | ||
| case _ => ir.CallFunction(irName, args) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| // TSQL specific function converters | ||
| // | ||
| // Note that these are left as objects, though we will possibly have a class per function in the future | ||
| // Each function can specify its own ConversionStrategy, and some will need to be very specific, | ||
| // hence perhaps moving to a class per function may be a better idea. | ||
| object TSqlFunctionConverters { | ||
|
|
||
| object FunctionRename extends ConversionStrategy with StringConverter { | ||
| override def convert(irName: String, args: Seq[ir.Expression]): ir.Expression = { | ||
| irName.toUpperCase() match { | ||
| case "ISNULL" => ir.CallFunction(convertString(irName, "IFNULL"), args) | ||
| case _ => ir.CallFunction(irName, args) | ||
| } | ||
| } | ||
| } | ||
|
|
||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.