Sync upstream

.github/workflows/rust.yml

@@ -27,7 +27,7 @@ jobs:
       - uses: actions/checkout@v4
       - name: Setup Rust Toolchain
         uses: ./.github/actions/setup-builder
-      - run: cargo fmt -- --check 
+      - run: cargo fmt --all -- --check
 
   lint:
     runs-on: ubuntu-latest

Cargo.toml

@@ -18,7 +18,7 @@
 [package]
 name = "sqlparser"
 description = "Extensible SQL Lexer and Parser with support for ANSI SQL:2011"
-version = "0.52.0"
+version = "0.53.0"
 authors = ["Apache DataFusion <[email protected]>"]
 homepage = "https://github.com/apache/datafusion-sqlparser-rs"
 documentation = "https://docs.rs/sqlparser/"
@@ -37,15 +37,18 @@ name = "sqlparser"
 path = "src/lib.rs"
 
 [features]
-default = ["std"]
+default = ["std", "recursive-protection"]
 std = []
+recursive-protection = ["std", "recursive"]
 # Enable JSON output in the `cli` example:
 json_example = ["serde_json", "serde"]
 visitor = ["sqlparser_derive"]
 
 [dependencies]
 bigdecimal = { version = "0.4.1", features = ["serde"], optional = true }
 log = "0.4"
+recursive = { version = "0.1.1", optional = true}
+
 serde = { version = "1.0", features = ["derive"], optional = true }
 # serde_json is only used in examples/cli, but we have to put it outside
 # of dev-dependencies because of

README.md

@@ -63,7 +63,7 @@ The following optional [crate  features](https://doc.rust-lang.org/cargo/referen
 
 * `serde`: Adds [Serde](https://serde.rs/) support by implementing  `Serialize` and `Deserialize` for all AST nodes.
 * `visitor`: Adds a `Visitor` capable of recursively walking the AST tree.
-
+* `recursive-protection` (enabled by default), uses [recursive](https://docs.rs/recursive/latest/recursive/) for stack overflow protection. 
 
 ## Syntax vs Semantics
 
@@ -240,11 +240,14 @@ You can run them with:
 ```
 git checkout main
 cd sqlparser_bench
-cargo bench
+cargo bench -- --save-baseline main
 git checkout <your branch>
-cargo bench
+cargo bench -- --baseline main
 ```
 
+By adding the `--save-baseline main` and `--baseline main` you can track the
+progress of your improvements as you continue working on the feature branch.
+
 ## Licensing
 
 All code in this repository is licensed under the [Apache Software License 2.0](LICENSE.txt).

changelog/0.53.0.md

@@ -0,0 +1,95 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+
+# sqlparser-rs 0.53.0 Changelog
+
+This release consists of 47 commits from 16 contributors. See credits at the end of this changelog for more information.
+
+**Other:**
+
+- hive: support for special not expression `!a` and raise error for `a!` factorial operator [#1472](https://github.com/apache/datafusion-sqlparser-rs/pull/1472) (wugeer)
+- Add support for MSSQL's `OPENJSON WITH` clause [#1498](https://github.com/apache/datafusion-sqlparser-rs/pull/1498) (gaoqiangz)
+- Parse true and false as identifiers in mssql [#1510](https://github.com/apache/datafusion-sqlparser-rs/pull/1510) (lovasoa)
+- Fix the parsing error in MSSQL for multiple statements that include `DECLARE` statements [#1497](https://github.com/apache/datafusion-sqlparser-rs/pull/1497) (wugeer)
+- Add support for Snowflake SHOW DATABASES/SCHEMAS/TABLES/VIEWS/COLUMNS statements [#1501](https://github.com/apache/datafusion-sqlparser-rs/pull/1501) (yoavcloud)
+- Add support of COMMENT ON syntax for Snowflake [#1516](https://github.com/apache/datafusion-sqlparser-rs/pull/1516) (git-hulk)
+- Add support for MYSQL's `CREATE TABLE SELECT` expr [#1515](https://github.com/apache/datafusion-sqlparser-rs/pull/1515) (wugeer)
+- Add support for MSSQL's `XQuery` methods [#1500](https://github.com/apache/datafusion-sqlparser-rs/pull/1500) (gaoqiangz)
+- Add support for Hive's `LOAD DATA` expr [#1520](https://github.com/apache/datafusion-sqlparser-rs/pull/1520) (wugeer)
+- Fix ClickHouse document link from `Russian` to `English` [#1527](https://github.com/apache/datafusion-sqlparser-rs/pull/1527) (git-hulk)
+- Support ANTI and SEMI joins without LEFT/RIGHT [#1528](https://github.com/apache/datafusion-sqlparser-rs/pull/1528) (delamarch3)
+- support sqlite's OR clauses in update statements [#1530](https://github.com/apache/datafusion-sqlparser-rs/pull/1530) (lovasoa)
+- support column type definitions in table aliases [#1526](https://github.com/apache/datafusion-sqlparser-rs/pull/1526) (lovasoa)
+- Add support for MSSQL's `JSON_ARRAY`/`JSON_OBJECT` expr [#1507](https://github.com/apache/datafusion-sqlparser-rs/pull/1507) (gaoqiangz)
+- Add support for PostgreSQL `UNLISTEN` syntax and Add support for Postgres `LOAD extension` expr [#1531](https://github.com/apache/datafusion-sqlparser-rs/pull/1531) (wugeer)
+- Parse byte/bit string literals in MySQL and Postgres [#1532](https://github.com/apache/datafusion-sqlparser-rs/pull/1532) (mvzink)
+- Allow example CLI to read from stdin [#1536](https://github.com/apache/datafusion-sqlparser-rs/pull/1536) (mvzink)
+- recursive select calls are parsed with bad trailing_commas parameter [#1521](https://github.com/apache/datafusion-sqlparser-rs/pull/1521) (tomershaniii)
+- PartiQL queries in Redshift [#1534](https://github.com/apache/datafusion-sqlparser-rs/pull/1534) (yoavcloud)
+- Include license file in sqlparser_derive crate [#1543](https://github.com/apache/datafusion-sqlparser-rs/pull/1543) (ankane)
+- Fallback to identifier parsing if expression parsing fails [#1513](https://github.com/apache/datafusion-sqlparser-rs/pull/1513) (yoavcloud)
+- support `json_object('k':'v')` in postgres [#1546](https://github.com/apache/datafusion-sqlparser-rs/pull/1546) (lovasoa)
+- Document micro benchmarks [#1555](https://github.com/apache/datafusion-sqlparser-rs/pull/1555) (alamb)
+- Implement `Spanned` to retrieve source locations on AST nodes [#1435](https://github.com/apache/datafusion-sqlparser-rs/pull/1435) (Nyrox)
+- Fix error in benchmark queries [#1560](https://github.com/apache/datafusion-sqlparser-rs/pull/1560) (alamb)
+- Fix clippy warnings on rust 1.83 [#1570](https://github.com/apache/datafusion-sqlparser-rs/pull/1570) (iffyio)
+- Support relation visitor to visit the `Option` field [#1556](https://github.com/apache/datafusion-sqlparser-rs/pull/1556) (goldmedal)
+- Rename `TokenWithLocation` to `TokenWithSpan`, in backwards compatible way [#1562](https://github.com/apache/datafusion-sqlparser-rs/pull/1562) (alamb)
+- Support MySQL size variants for BLOB and TEXT columns [#1564](https://github.com/apache/datafusion-sqlparser-rs/pull/1564) (mvzink)
+- Increase version of sqlparser_derive from 0.2.2 to 0.3.0 [#1571](https://github.com/apache/datafusion-sqlparser-rs/pull/1571) (alamb)
+- `json_object('k' VALUE 'v')` in postgres [#1547](https://github.com/apache/datafusion-sqlparser-rs/pull/1547) (lovasoa)
+- Support snowflake double dot notation for object name [#1540](https://github.com/apache/datafusion-sqlparser-rs/pull/1540) (ayman-sigma)
+- Update comments / docs for `Spanned` [#1549](https://github.com/apache/datafusion-sqlparser-rs/pull/1549) (alamb)
+- Support Databricks struct literal [#1542](https://github.com/apache/datafusion-sqlparser-rs/pull/1542) (ayman-sigma)
+- Encapsulate CreateFunction [#1573](https://github.com/apache/datafusion-sqlparser-rs/pull/1573) (philipcristiano)
+- Support BIT column types [#1577](https://github.com/apache/datafusion-sqlparser-rs/pull/1577) (mvzink)
+- Support parsing optional nulls handling for unique constraint [#1567](https://github.com/apache/datafusion-sqlparser-rs/pull/1567) (mvzink)
+- Fix displaying WORK or TRANSACTION after BEGIN [#1565](https://github.com/apache/datafusion-sqlparser-rs/pull/1565) (mvzink)
+- Add support of the ENUM8|ENUM16 for ClickHouse dialect [#1574](https://github.com/apache/datafusion-sqlparser-rs/pull/1574) (git-hulk)
+- Parse Snowflake USE ROLE and USE SECONDARY ROLES [#1578](https://github.com/apache/datafusion-sqlparser-rs/pull/1578) (yoavcloud)
+- Snowflake ALTER TABLE clustering options [#1579](https://github.com/apache/datafusion-sqlparser-rs/pull/1579) (yoavcloud)
+- Support INSERT OVERWRITE INTO syntax [#1584](https://github.com/apache/datafusion-sqlparser-rs/pull/1584) (yuval-illumex)
+- Parse `INSERT` with subquery when lacking column names [#1586](https://github.com/apache/datafusion-sqlparser-rs/pull/1586) (iffyio)
+- Add support for ODBC functions [#1585](https://github.com/apache/datafusion-sqlparser-rs/pull/1585) (iffyio)
+
+## Credits
+
+Thank you to everyone who contributed to this release. Here is a breakdown of commits (PRs merged) per contributor.
+
+```
+     8	Andrew Lamb
+     6	Michael Victor Zink
+     5	Ophir LOJKINE
+     5	Yoav Cohen
+     5	wugeer
+     3	Ifeanyi Ubah
+     3	gaoqiangz
+     3	hulk
+     2	Ayman Elkfrawy
+     1	Andrew Kane
+     1	Jax Liu
+     1	Mark-Oliver Junge
+     1	Philip Cristiano
+     1	Yuval Shkolar
+     1	delamarch3
+     1	tomershaniii
+```
+
+Thank you also to everyone who contributed in other ways such as filing issues, reviewing PRs, and providing feedback on this release.
+

derive/src/lib.rs

@@ -18,7 +18,11 @@
 use proc_macro2::TokenStream;
 use quote::{format_ident, quote, quote_spanned, ToTokens};
 use syn::spanned::Spanned;
-use syn::{parse::{Parse, ParseStream}, parse_macro_input, parse_quote, Attribute, Data, DeriveInput, Fields, GenericParam, Generics, Ident, Index, LitStr, Meta, Token, Type, TypePath};
+use syn::{
+    parse::{Parse, ParseStream},
+    parse_macro_input, parse_quote, Attribute, Data, DeriveInput, Fields, GenericParam, Generics,
+    Ident, Index, LitStr, Meta, Token, Type, TypePath,
+};
 use syn::{Path, PathArguments};
 
 /// Implementation of `[#derive(Visit)]`
@@ -74,7 +78,10 @@ fn derive_visit(input: proc_macro::TokenStream, visit_type: &VisitType) -> proc_
 
     let expanded = quote! {
         // The generated impl.
+        // Note that it uses [`recursive::recursive`] to protect from stack overflow.
+        // See tests in https://github.com/apache/datafusion-sqlparser-rs/pull/1522/ for more info.
         impl #impl_generics sqlparser::ast::#visit_trait for #name #ty_generics #where_clause {
+             #[cfg_attr(feature = "recursive-protection", recursive::recursive)]
             fn visit<V: sqlparser::ast::#visitor_trait>(
                 &#modifier self,
                 visitor: &mut V
@@ -267,7 +274,11 @@ fn visit_children(
 }
 
 fn is_option(ty: &Type) -> bool {
-    if let Type::Path(TypePath { path: Path { segments, .. }, .. }) = ty {
+    if let Type::Path(TypePath {
+        path: Path { segments, .. },
+        ..
+    }) = ty
+    {
         if let Some(segment) = segments.last() {
             if segment.ident == "Option" {
                 if let PathArguments::AngleBracketed(args) = &segment.arguments {

dev/release/README.md

@@ -146,6 +146,12 @@ Move artifacts to the release location in SVN, using the `release-tarball.sh` sc
 ```shell
 ./dev/release/release-tarball.sh 0.52.0 1
 ```
+
+Promote the rc tag to the release tag
+```shell
+git tag v0.52.0 v0.52.0-rc3
+git push apache v0.52.0
+```
 
 Congratulations! The release is now official!
 

sqlparser_bench/benches/sqlparser_bench.rs

@@ -42,6 +42,46 @@ fn basic_queries(c: &mut Criterion) {
     group.bench_function("sqlparser::with_select", |b| {
         b.iter(|| Parser::parse_sql(&dialect, with_query).unwrap());
     });
+
+    let large_statement = {
+        let expressions = (0..1000)
+            .map(|n| format!("FN_{}(COL_{})", n, n))
+            .collect::<Vec<_>>()
+            .join(", ");
+        let tables = (0..1000)
+            .map(|n| format!("TABLE_{}", n))
+            .collect::<Vec<_>>()
+            .join(" JOIN ");
+        let where_condition = (0..1000)
+            .map(|n| format!("COL_{} = {}", n, n))
+            .collect::<Vec<_>>()
+            .join(" OR ");
+        let order_condition = (0..1000)
+            .map(|n| format!("COL_{} DESC", n))
+            .collect::<Vec<_>>()
+            .join(", ");
+
+        format!(
+            "SELECT {} FROM {} WHERE {} ORDER BY {}",
+            expressions, tables, where_condition, order_condition
+        )
+    };
+
+    group.bench_function("parse_large_statement", |b| {
+        b.iter(|| Parser::parse_sql(&dialect, criterion::black_box(large_statement.as_str())));
+    });
+
+    let large_statement = Parser::parse_sql(&dialect, large_statement.as_str())
+        .unwrap()
+        .pop()
+        .unwrap();
+
+    group.bench_function("format_large_statement", |b| {
+        b.iter(|| {
+            let formatted_query = large_statement.to_string();
+            assert_eq!(formatted_query, large_statement);
+        });
+    });
 }
 
 criterion_group!(benches, basic_queries);

src/ast/data_type.rs

@@ -25,10 +25,21 @@ use serde::{Deserialize, Serialize};
 #[cfg(feature = "visitor")]
 use sqlparser_derive::{Visit, VisitMut};
 
-use crate::ast::{display_comma_separated, ObjectName, StructField, UnionField};
+use crate::ast::{display_comma_separated, Expr, ObjectName, StructField, UnionField};
 
 use super::{value::escape_single_quote_string, ColumnDef};
 
+#[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)]
+#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
+#[cfg_attr(feature = "visitor", derive(Visit, VisitMut))]
+pub enum EnumMember {
+    Name(String),
+    /// ClickHouse allows to specify an integer value for each enum value.
+    ///
+    /// [clickhouse](https://clickhouse.com/docs/en/sql-reference/data-types/enum)
+    NamedValue(String, Expr),
+}
+
 /// SQL data types
 #[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)]
 #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))]
@@ -243,7 +254,7 @@ pub enum DataType {
     /// [postgresql]: https://www.postgresql.org/docs/15/datatype.html
     Float8,
     /// Double
-    Double,
+    Double(ExactNumberInfo),
     /// Double PRECISION e.g. [standard], [postgresql]
     ///
     /// [standard]: https://jakewheat.github.io/sql-overview/sql-2016-foundation-grammar.html#approximate-numeric-type
@@ -307,6 +318,16 @@ pub enum DataType {
     FixedString(u64),
     /// Bytea
     Bytea,
+    /// Bit string, e.g. [Postgres], [MySQL], or [MSSQL]
+    ///
+    /// [Postgres]: https://www.postgresql.org/docs/current/datatype-bit.html
+    /// [MySQL]: https://dev.mysql.com/doc/refman/9.1/en/bit-type.html
+    /// [MSSQL]: https://learn.microsoft.com/en-us/sql/t-sql/data-types/bit-transact-sql?view=sql-server-ver16
+    Bit(Option<u64>),
+    /// Variable-length bit string e.g. [Postgres]
+    ///
+    /// [Postgres]: https://www.postgresql.org/docs/current/datatype-bit.html
+    BitVarying(Option<u64>),
     /// Custom type such as enums
     Custom(ObjectName, Vec<String>),
     /// Arrays
@@ -324,7 +345,7 @@ pub enum DataType {
     /// [clickhouse]: https://clickhouse.com/docs/en/sql-reference/data-types/nested-data-structures/nested
     Nested(Vec<ColumnDef>),
     /// Enums
-    Enum(Vec<String>),
+    Enum(Vec<EnumMember>, Option<u8>),
     /// Set
     Set(Vec<String>),
     /// Struct
@@ -352,6 +373,10 @@ pub enum DataType {
     ///
     /// [postgresql]: https://www.postgresql.org/docs/current/plpgsql-trigger.html
     Trigger,
+    /// Any data type, used in BigQuery UDF definitions for templated parameters
+    ///
+    /// [bigquery]: https://cloud.google.com/bigquery/docs/user-defined-functions#templated-sql-udf-parameters
+    AnyType,
 }
 
 impl fmt::Display for DataType {
@@ -362,7 +387,6 @@ impl fmt::Display for DataType {
             DataType::CharacterVarying(size) => {
                 format_character_string_type(f, "CHARACTER VARYING", size)
             }
-
             DataType::CharVarying(size) => format_character_string_type(f, "CHAR VARYING", size),
             DataType::Varchar(size) => format_character_string_type(f, "VARCHAR", size),
             DataType::Nvarchar(size) => format_character_string_type(f, "NVARCHAR", size),
@@ -484,7 +508,7 @@ impl fmt::Display for DataType {
             DataType::Float4 => write!(f, "FLOAT4"),
             DataType::Float32 => write!(f, "Float32"),
             DataType::Float64 => write!(f, "FLOAT64"),
-            DataType::Double => write!(f, "DOUBLE"),
+            DataType::Double(info) => write!(f, "DOUBLE{info}"),
             DataType::Float8 => write!(f, "FLOAT8"),
             DataType::DoublePrecision => write!(f, "DOUBLE PRECISION"),
             DataType::Bool => write!(f, "BOOL"),
@@ -518,6 +542,10 @@ impl fmt::Display for DataType {
             DataType::LongText => write!(f, "LONGTEXT"),
             DataType::String(size) => format_type_with_optional_length(f, "STRING", size, false),
             DataType::Bytea => write!(f, "BYTEA"),
+            DataType::Bit(size) => format_type_with_optional_length(f, "BIT", size, false),
+            DataType::BitVarying(size) => {
+                format_type_with_optional_length(f, "BIT VARYING", size, false)
+            }
             DataType::Array(ty) => match ty {
                 ArrayElemTypeDef::None => write!(f, "ARRAY"),
                 ArrayElemTypeDef::SquareBracket(t, None) => write!(f, "{t}[]"),
@@ -532,13 +560,24 @@ impl fmt::Display for DataType {
                     write!(f, "{}({})", ty, modifiers.join(", "))
                 }
             }
-            DataType::Enum(vals) => {
-                write!(f, "ENUM(")?;
+            DataType::Enum(vals, bits) => {
+                match bits {
+                    Some(bits) => write!(f, "ENUM{}", bits),
+                    None => write!(f, "ENUM"),
+                }?;
+                write!(f, "(")?;
                 for (i, v) in vals.iter().enumerate() {
                     if i != 0 {
                         write!(f, ", ")?;
                     }
-                    write!(f, "'{}'", escape_single_quote_string(v))?;
+                    match v {
+                        EnumMember::Name(name) => {
+                            write!(f, "'{}'", escape_single_quote_string(name))?
+                        }
+                        EnumMember::NamedValue(name, value) => {
+                            write!(f, "'{}' = {}", escape_single_quote_string(name), value)?
+                        }
+                    }
                 }
                 write!(f, ")")
             }
@@ -590,6 +629,7 @@ impl fmt::Display for DataType {
             }
             DataType::Unspecified => Ok(()),
             DataType::Trigger => write!(f, "TRIGGER"),
+            DataType::AnyType => write!(f, "ANY TYPE"),
         }
     }
 }