Update for new sqlparser API

datafusion/common/src/utils/mod.rs

@@ -775,10 +775,10 @@ pub fn get_available_parallelism() -> usize {
 
 #[cfg(test)]
 mod tests {
+    use super::*;
     use crate::ScalarValue::Null;
     use arrow::array::Float64Array;
-
-    use super::*;
+    use sqlparser::tokenizer::Span;
 
     #[test]
     fn test_bisect_linear_left_and_right() -> Result<()> {
@@ -1006,6 +1006,7 @@ mod tests {
             let expected_parsed = vec![Ident {
                 value: identifier.to_string(),
                 quote_style,
+                span: Span::empty(),
             }];
 
             assert_eq!(

datafusion/core/tests/user_defined/user_defined_scalar_functions.rs

@@ -27,10 +27,6 @@ use arrow_array::{
     Array, ArrayRef, Float32Array, Float64Array, Int32Array, RecordBatch, StringArray,
 };
 use arrow_schema::{DataType, Field, Schema};
-use parking_lot::Mutex;
-use regex::Regex;
-use sqlparser::ast::Ident;
-
 use datafusion::execution::context::{FunctionFactory, RegisterFunction, SessionState};
 use datafusion::prelude::*;
 use datafusion::{execution::registry::FunctionRegistry, test_util};
@@ -48,6 +44,10 @@ use datafusion_expr::{
     Volatility,
 };
 use datafusion_functions_nested::range::range_udf;
+use parking_lot::Mutex;
+use regex::Regex;
+use sqlparser::ast::Ident;
+use sqlparser::tokenizer::Span;
 
 /// test that casting happens on udfs.
 /// c11 is f32, but `custom_sqrt` requires f64. Casting happens but the logical plan and
@@ -1187,6 +1187,7 @@ async fn create_scalar_function_from_sql_statement_postgres_syntax() -> Result<(
             name: Some(Ident {
                 value: "name".into(),
                 quote_style: None,
+                span: Span::empty(),
             }),
             data_type: DataType::Utf8,
             default_expr: None,
@@ -1196,6 +1197,7 @@ async fn create_scalar_function_from_sql_statement_postgres_syntax() -> Result<(
             language: Some(Ident {
                 value: "plrust".into(),
                 quote_style: None,
+                span: Span::empty(),
             }),
             behavior: None,
             function_body: Some(lit(body)),

datafusion/expr/src/expr.rs

@@ -222,6 +222,8 @@ use sqlparser::ast::{
 /// // to 42 = 5 AND b = 6
 /// assert_eq!(rewritten.data, lit(42).eq(lit(5)).and(col("b").eq(lit(6))));
 #[derive(Clone, PartialEq, Eq, PartialOrd, Hash, Debug)]
+// TODO make the enum smaller with more boxing (looks like Wildcard is now bigger)
+#[allow(clippy::large_enum_variant)]
 pub enum Expr {
     /// An expression with a specific name.
     Alias(Alias),

datafusion/sql/src/expr/function.rs

@@ -169,6 +169,11 @@ impl FunctionArgs {
                         "Calling {name}: SEPARATOR not supported in function arguments: {sep}"
                     )
                 }
+                FunctionArgumentClause::JsonNullClause(jn) => {
+                    return not_impl_err!(
+                        "Calling {name}: JSON NULL clause not supported in function arguments: {jn}"
+                    )
+                }
             }
         }
 

datafusion/sql/src/expr/mod.rs

@@ -565,11 +565,11 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
                 }
                 not_impl_err!("AnyOp not supported by ExprPlanner: {binary_expr:?}")
             }
-            SQLExpr::Wildcard => Ok(Expr::Wildcard {
+            SQLExpr::Wildcard(_token) => Ok(Expr::Wildcard {
                 qualifier: None,
                 options: WildcardOptions::default(),
             }),
-            SQLExpr::QualifiedWildcard(object_name) => Ok(Expr::Wildcard {
+            SQLExpr::QualifiedWildcard(object_name, _token) => Ok(Expr::Wildcard {
                 qualifier: Some(self.object_name_to_table_reference(object_name)?),
                 options: WildcardOptions::default(),
             }),

datafusion/sql/src/parser.rs

@@ -875,6 +875,7 @@ mod tests {
     use super::*;
     use sqlparser::ast::Expr::Identifier;
     use sqlparser::ast::{BinaryOperator, DataType, Expr, Ident};
+    use sqlparser::tokenizer::Span;
 
     fn expect_parse_ok(sql: &str, expected: Statement) -> Result<(), ParserError> {
         let statements = DFParser::parse_sql(sql)?;
@@ -910,6 +911,7 @@ mod tests {
             name: Ident {
                 value: name.into(),
                 quote_style: None,
+                span: Span::empty(),
             },
             data_type,
             collation: None,
@@ -1218,6 +1220,7 @@ mod tests {
                     expr: Identifier(Ident {
                         value: "c1".to_owned(),
                         quote_style: None,
+                        span: Span::empty(),
                     }),
                     asc,
                     nulls_first,
@@ -1249,6 +1252,7 @@ mod tests {
                     expr: Identifier(Ident {
                         value: "c1".to_owned(),
                         quote_style: None,
+                        span: Span::empty(),
                     }),
                     asc: Some(true),
                     nulls_first: None,
@@ -1258,6 +1262,7 @@ mod tests {
                     expr: Identifier(Ident {
                         value: "c2".to_owned(),
                         quote_style: None,
+                        span: Span::empty(),
                     }),
                     asc: Some(false),
                     nulls_first: Some(true),
@@ -1289,11 +1294,13 @@ mod tests {
                     left: Box::new(Identifier(Ident {
                         value: "c1".to_owned(),
                         quote_style: None,
+                        span: Span::empty(),
                     })),
                     op: BinaryOperator::Minus,
                     right: Box::new(Identifier(Ident {
                         value: "c2".to_owned(),
                         quote_style: None,
+                        span: Span::empty(),
                     })),
                 },
                 asc: Some(true),
@@ -1334,11 +1341,13 @@ mod tests {
                     left: Box::new(Identifier(Ident {
                         value: "c1".to_owned(),
                         quote_style: None,
+                        span: Span::empty(),
                     })),
                     op: BinaryOperator::Minus,
                     right: Box::new(Identifier(Ident {
                         value: "c2".to_owned(),
                         quote_style: None,
+                        span: Span::empty(),
                     })),
                 },
                 asc: Some(true),

datafusion/sql/src/planner.rs

@@ -339,7 +339,8 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
         plan: LogicalPlan,
         alias: TableAlias,
     ) -> Result<LogicalPlan> {
-        let plan = self.apply_expr_alias(plan, alias.columns)?;
+        let idents = alias.columns.into_iter().map(|c| c.name).collect();
+        let plan = self.apply_expr_alias(plan, idents)?;
 
         LogicalPlanBuilder::from(plan)
             .alias(TableReference::bare(

datafusion/sql/src/select.rs

@@ -654,6 +654,7 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
             opt_rename,
             opt_replace: _opt_replace,
             opt_ilike: _opt_ilike,
+            wildcard_token: _wildcard_token,
         } = options;
 
         if opt_rename.is_some() {

datafusion/sql/src/statement.rs

@@ -54,13 +54,13 @@ use datafusion_expr::{
     TransactionConclusion, TransactionEnd, TransactionIsolationLevel, TransactionStart,
     Volatility, WriteOp,
 };
-use sqlparser::ast::{self, SqliteOnConflict};
+use sqlparser::ast::{self, ShowStatementIn, ShowStatementOptions, SqliteOnConflict};
 use sqlparser::ast::{
     Assignment, AssignmentTarget, ColumnDef, CreateIndex, CreateTable,
     CreateTableOptions, Delete, DescribeAlias, Expr as SQLExpr, FromTable, Ident, Insert,
     ObjectName, ObjectType, OneOrManyWithParens, Query, SchemaName, SetExpr,
-    ShowCreateObject, ShowStatementFilter, Statement, TableConstraint, TableFactor,
-    TableWithJoins, TransactionMode, UnaryOperator, Value,
+    ShowCreateObject, Statement, TableConstraint, TableFactor, TableWithJoins,
+    TransactionMode, UnaryOperator, Value,
 };
 use sqlparser::parser::ParserError::ParserError;
 
@@ -685,19 +685,99 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
             Statement::ShowTables {
                 extended,
                 full,
-                db_name,
-                filter,
-                // SHOW TABLES IN/FROM are equivalent, this field specifies which the user
-                // specified, but it doesn't affect the plan so ignore the field
-                clause: _,
-            } => self.show_tables_to_plan(extended, full, db_name, filter),
+                terse,
+                history,
+                external,
+                show_options,
+            } => {
+                // We only support the basic "SHOW TABLES"
+                // https://github.com/apache/datafusion/issues/3188
+                if extended {
+                    return not_impl_err!("SHOW TABLES EXTENDED not supported")?;
+                }
+                if full {
+                    return not_impl_err!("SHOW FULL TABLES not supported")?;
+                }
+                if terse {
+                    return not_impl_err!("SHOW TERSE TABLES not supported")?;
+                }
+                if history {
+                    return not_impl_err!("SHOW TABLES HISTORY not supported")?;
+                }
+                if external {
+                    return not_impl_err!("SHOW EXTERNAL TABLES not supported")?;
+                }
+                let ShowStatementOptions {
+                    show_in,
+                    starts_with,
+                    limit,
+                    limit_from,
+                    filter_position,
+                } = show_options;
+                if show_in.is_some() {
+                    return not_impl_err!("SHOW TABLES IN not supported")?;
+                }
+                if starts_with.is_some() {
+                    return not_impl_err!("SHOW TABLES LIKE not supported")?;
+                }
+                if limit.is_some() {
+                    return not_impl_err!("SHOW TABLES LIMIT not supported")?;
+                }
+                if limit_from.is_some() {
+                    return not_impl_err!("SHOW TABLES LIMIT FROM not supported")?;
+                }
+                if filter_position.is_some() {
+                    return not_impl_err!("SHOW TABLES FILTER not supported")?;
+                }
+                self.show_tables_to_plan()
+            }
 
             Statement::ShowColumns {
                 extended,
                 full,
-                table_name,
-                filter,
-            } => self.show_columns_to_plan(extended, full, table_name, filter),
+                show_options,
+            } => {
+                let ShowStatementOptions {
+                    show_in,
+                    starts_with,
+                    limit,
+                    limit_from,
+                    filter_position,
+                } = show_options;
+                if starts_with.is_some() {
+                    return not_impl_err!("SHOW COLUMNS LIKE not supported")?;
+                }
+                if limit.is_some() {
+                    return not_impl_err!("SHOW COLUMNS LIMIT not supported")?;
+                }
+                if limit_from.is_some() {
+                    return not_impl_err!("SHOW COLUMNS LIMIT FROM not supported")?;
+                }
+                if filter_position.is_some() {
+                    return not_impl_err!(
+                        "SHOW COLUMNS with WHERE or LIKE is not supported"
+                    )?;
+                }
+                let Some(ShowStatementIn {
+                    // specifies if the syntax was `SHOW COLUMNS IN` or `SHOW
+                    // COLUMNS FROM` which is not different in DataFusion
+                    clause: _,
+                    parent_type,
+                    parent_name,
+                }) = show_in
+                else {
+                    return plan_err!("SHOW COLUMNS requires a table name");
+                };
+
+                if let Some(parent_type) = parent_type {
+                    return not_impl_err!("SHOW COLUMNS IN {parent_type} not supported");
+                }
+                let Some(table_name) = parent_name else {
+                    return plan_err!("SHOW COLUMNS requires a table name");
+                };
+
+                self.show_columns_to_plan(extended, full, table_name)
+            }
 
             Statement::Insert(Insert {
                 or,
@@ -766,10 +846,14 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
                 from,
                 selection,
                 returning,
+                or,
             } => {
                 if returning.is_some() {
                     plan_err!("Update-returning clause not yet supported")?;
                 }
+                if or.is_some() {
+                    plan_err!("ON conflict not supported")?;
+                }
                 self.update_to_plan(table, assignments, from, selection)
             }
 
@@ -1065,24 +1149,12 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
     }
 
     /// Generate a logical plan from a "SHOW TABLES" query
-    fn show_tables_to_plan(
-        &self,
-        extended: bool,
-        full: bool,
-        db_name: Option<Ident>,
-        filter: Option<ShowStatementFilter>,
-    ) -> Result<LogicalPlan> {
+    fn show_tables_to_plan(&self) -> Result<LogicalPlan> {
         if self.has_table("information_schema", "tables") {
-            // We only support the basic "SHOW TABLES"
-            // https://github.com/apache/datafusion/issues/3188
-            if db_name.is_some() || filter.is_some() || full || extended {
-                plan_err!("Unsupported parameters to SHOW TABLES")
-            } else {
-                let query = "SELECT * FROM information_schema.tables;";
-                let mut rewrite = DFParser::parse_sql(query)?;
-                assert_eq!(rewrite.len(), 1);
-                self.statement_to_plan(rewrite.pop_front().unwrap()) // length of rewrite is 1
-            }
+            let query = "SELECT * FROM information_schema.tables;";
+            let mut rewrite = DFParser::parse_sql(query)?;
+            assert_eq!(rewrite.len(), 1);
+            self.statement_to_plan(rewrite.pop_front().unwrap()) // length of rewrite is 1
         } else {
             plan_err!("SHOW TABLES is not supported unless information_schema is enabled")
         }
@@ -1842,22 +1914,18 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
         extended: bool,
         full: bool,
         sql_table_name: ObjectName,
-        filter: Option<ShowStatementFilter>,
     ) -> Result<LogicalPlan> {
-        if filter.is_some() {
-            return plan_err!("SHOW COLUMNS with WHERE or LIKE is not supported");
-        }
+        // Figure out the where clause
+        let where_clause = object_name_to_qualifier(
+            &sql_table_name,
+            self.options.enable_ident_normalization,
+        );
 
         if !self.has_table("information_schema", "columns") {
             return plan_err!(
                 "SHOW COLUMNS is not supported unless information_schema is enabled"
             );
         }
-        // Figure out the where clause
-        let where_clause = object_name_to_qualifier(
-            &sql_table_name,
-            self.options.enable_ident_normalization,
-        );
 
         // Do a table lookup to verify the table exists
         let table_ref = self.object_name_to_table_reference(sql_table_name)?;