Skip to content

Commit

Permalink
output to_timestamp signature as ns
Browse files Browse the repository at this point in the history
  • Loading branch information
comphead committed Oct 20, 2023
1 parent 830a4c8 commit 5fe44db
Show file tree
Hide file tree
Showing 10 changed files with 40 additions and 35 deletions.
4 changes: 2 additions & 2 deletions datafusion-examples/examples/expr_api.rs
Original file line number Diff line number Diff line change
Expand Up @@ -82,10 +82,10 @@ fn simplify_demo() -> Result<()> {
let expr = simplifier.simplify(expr)?;

// DataFusion has simplified the expression to a comparison with a constant
// ts = 1599566400i64; Tada!
// ts = 1599566400000000000; Tada!
assert_eq!(
expr,
col("ts").eq(lit(ScalarValue::TimestampSecond(Some(1599566400i64), None)))
col("ts").eq(lit_timestamp_nano(1599566400000000000i64))
);

// here are some other examples of what DataFusion is capable of
Expand Down
2 changes: 1 addition & 1 deletion datafusion/core/tests/sql/expr.rs
Original file line number Diff line number Diff line change
Expand Up @@ -639,7 +639,7 @@ async fn test_uuid_expression() -> Result<()> {
async fn test_extract_date_part() -> Result<()> {
test_expression!("date_part('YEAR', CAST('2000-01-01' AS DATE))", "2000.0");
test_expression!(
"EXTRACT(year FROM to_timestamp('2020-09-08T12:00:00+00:00'))",
"EXTRACT(year FROM timestamp '2020-09-08T12:00:00+00:00')",
"2020.0"
);
test_expression!("date_part('QUARTER', CAST('2000-01-01' AS DATE))", "1.0");
Expand Down
2 changes: 1 addition & 1 deletion datafusion/expr/src/built_in_function.rs
Original file line number Diff line number Diff line change
Expand Up @@ -748,7 +748,7 @@ impl BuiltinScalarFunction {
return plan_err!("The to_hex function can only accept integers.");
}
}),
BuiltinScalarFunction::ToTimestamp => Ok(Timestamp(Second, None)),
BuiltinScalarFunction::ToTimestamp => Ok(Timestamp(Nanosecond, None)),
BuiltinScalarFunction::ToTimestampMillis => Ok(Timestamp(Millisecond, None)),
BuiltinScalarFunction::ToTimestampMicros => Ok(Timestamp(Microsecond, None)),
BuiltinScalarFunction::ToTimestampNanos => Ok(Timestamp(Nanosecond, None)),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1504,10 +1504,7 @@ mod tests {
// to_timestamp("2020-09-08T12:00:00+00:00") --> timestamp(1599566400i64)
let expr =
call_fn("to_timestamp", vec![lit("2020-09-08T12:00:00+00:00")]).unwrap();
test_evaluate(
expr,
lit(ScalarValue::TimestampSecond(Some(1599566400i64), None)),
);
test_evaluate(expr, lit_timestamp_nano(1599566400000000000i64));

// check that non foldable arguments are folded
// to_timestamp(a) --> to_timestamp(a) [no rewrite possible]
Expand Down Expand Up @@ -1543,10 +1540,10 @@ mod tests {
let expr = cast_to_int64_expr(now_expr()) + lit(100_i64);
test_evaluate_with_start_time(expr, lit(ts_nanos + 100), &time);

// CAST(now() as int64) < cast(to_timestamp(...) as int64) + 50000_i64 ---> false
// CAST(now() as int64) < cast(to_timestamp(...) as int64) + 50000_i64 ---> true
let expr = cast_to_int64_expr(now_expr())
.lt(cast_to_int64_expr(to_timestamp_expr(ts_string)) + lit(50000i64));
test_evaluate_with_start_time(expr, lit(false), &time);
test_evaluate_with_start_time(expr, lit(true), &time);
}

#[test]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -460,7 +460,7 @@ mod tests {
.project(proj)?
.build()?;

let expected = "Projection: TimestampSecond(1599566400, None) AS to_timestamp(Utf8(\"2020-09-08T12:00:00+00:00\"))\
let expected = "Projection: TimestampNanosecond(1599566400000000000, None) AS to_timestamp(Utf8(\"2020-09-08T12:00:00+00:00\"))\
\n TableScan: test"
.to_string();
let actual = get_optimized_plan_formatted(&plan, &Utc::now());
Expand Down Expand Up @@ -571,8 +571,8 @@ mod tests {
.build()?;

// Note that constant folder runs and folds the entire
// expression down to a single constant (false)
let expected = "Filter: Boolean(false)\
// expression down to a single constant (true)
let expected = "Filter: Boolean(true)\
\n TableScan: test";
let actual = get_optimized_plan_formatted(&plan, &time);

Expand Down
10 changes: 5 additions & 5 deletions datafusion/physical-expr/src/datetime_expressions.rs
Original file line number Diff line number Diff line change
Expand Up @@ -129,9 +129,9 @@ fn string_to_timestamp_nanos_shim(s: &str) -> Result<i64> {

/// to_timestamp SQL function
pub fn to_timestamp(args: &[ColumnarValue]) -> Result<ColumnarValue> {
handle::<TimestampSecondType, _, TimestampSecondType>(
handle::<TimestampNanosecondType, _, TimestampNanosecondType>(
args,
|s| string_to_timestamp_nanos_shim(s).map(|n| n / 1_000_000_000),
string_to_timestamp_nanos_shim,
"to_timestamp",
)
}
Expand Down Expand Up @@ -969,10 +969,10 @@ mod tests {
// ensure that arrow array implementation is wired up and handles nulls correctly

let mut string_builder = StringBuilder::with_capacity(2, 1024);
let mut ts_builder = TimestampSecondArray::builder(2);
let mut ts_builder = TimestampNanosecondArray::builder(2);

string_builder.append_value("2020-09-08T13:42:29");
ts_builder.append_value(1599572549);
string_builder.append_value("2020-09-08T13:42:29.190855");
ts_builder.append_value(1599572549190855000);

string_builder.append_null();
ts_builder.append_null();
Expand Down
23 changes: 14 additions & 9 deletions datafusion/physical-expr/src/functions.rs
Original file line number Diff line number Diff line change
Expand Up @@ -74,15 +74,20 @@ pub fn create_physical_expr(
// so we don't have to pay a per-array/batch cost.
BuiltinScalarFunction::ToTimestamp => {
Arc::new(match input_phy_exprs[0].data_type(input_schema) {
Ok(DataType::Int64) | Ok(DataType::Timestamp(_, None)) => {
|col_values: &[ColumnarValue]| {
cast_column(
&col_values[0],
&DataType::Timestamp(TimeUnit::Second, None),
None,
)
}
}
Ok(DataType::Int64) => |col_values: &[ColumnarValue]| {
cast_column(
&col_values[0],
&DataType::Timestamp(TimeUnit::Second, None),
None,
)
},
Ok(DataType::Timestamp(_, None)) => |col_values: &[ColumnarValue]| {
cast_column(
&col_values[0],
&DataType::Timestamp(TimeUnit::Nanosecond, None),
None,
)
},
Ok(DataType::Utf8) => datetime_expressions::to_timestamp,
other => {
return internal_err!(
Expand Down
2 changes: 1 addition & 1 deletion datafusion/sql/tests/sql_integration.rs
Original file line number Diff line number Diff line change
Expand Up @@ -444,7 +444,7 @@ Dml: op=[Insert Into] table=[test_decimal]
)]
#[case::type_mismatch(
"INSERT INTO test_decimal SELECT '2022-01-01', to_timestamp('2022-01-01T12:00:00')",
"Error during planning: Cannot automatically convert Timestamp(Second, None) to Decimal128(10, 2)"
"Error during planning: Cannot automatically convert Timestamp(Nanosecond, None) to Decimal128(10, 2)"
)]
#[case::target_column_count_mismatch(
"INSERT INTO person (id, first_name, last_name) VALUES ($1, $2)",
Expand Down
8 changes: 4 additions & 4 deletions datafusion/sqllogictest/test_files/timestamps.slt
Original file line number Diff line number Diff line change
Expand Up @@ -249,9 +249,9 @@ SELECT to_timestamp_seconds(ts) FROM ts_data_micros LIMIT 3
query P
SELECT to_timestamp(ts) FROM ts_data_micros LIMIT 3
----
2020-09-08T13:42:29
2020-09-08T12:42:29
2020-09-08T11:42:29
2020-09-08T13:42:29.190855
2020-09-08T12:42:29.190855
2020-09-08T11:42:29.190855

# query_cast_timestamp_from_unixtime

Expand All @@ -266,7 +266,7 @@ SELECT from_unixtime(ts / 1000000000) FROM ts_data LIMIT 3;
# to_timestamp

query I
SELECT COUNT(*) FROM ts_data_nanos where ts > to_timestamp('2020-09-08T12:00:00+00:00')
SELECT COUNT(*) FROM ts_data_nanos where ts > timestamp '2020-09-08T12:00:00+00:00'
----
2

Expand Down
9 changes: 6 additions & 3 deletions docs/source/user-guide/sql/scalar_functions.md
Original file line number Diff line number Diff line change
Expand Up @@ -1391,9 +1391,9 @@ extract(field FROM source)

### `to_timestamp`

Converts a value to RFC3339 second timestamp format (`YYYY-MM-DDT00:00:00Z`).
Converts a value to RFC3339 nanosecond timestamp format (`YYYY-MM-DDT00:00:00Z`).
Supports timestamp, integer, and unsigned integer types as input.
Integers and unsigned integers are parsed as Unix nanosecond timestamps and
Integers and unsigned integers are parsed as Unix second timestamps and
return the corresponding RFC3339 timestamp.

```
Expand Down Expand Up @@ -1450,7 +1450,10 @@ to_timestamp_nanos(expression)

### `to_timestamp_seconds`

Alias for `to_timestamp()` function.
Converts a value to RFC3339 second timestamp format (`YYYY-MM-DDT00:00:00Z`).
Supports timestamp, integer, and unsigned integer types as input.
Integers and unsigned integers are parsed as Unix nanosecond timestamps and
return the corresponding RFC3339 timestamp.

```
to_timestamp_seconds(expression)
Expand Down

0 comments on commit 5fe44db

Please sign in to comment.