diff --git a/datafusion/functions/src/datetime/date_part.rs b/datafusion/functions/src/datetime/date_part.rs index 765b215a885e..e9ae3188694b 100644 --- a/datafusion/functions/src/datetime/date_part.rs +++ b/datafusion/functions/src/datetime/date_part.rs @@ -152,6 +152,7 @@ impl ScalarUDFImpl for DatePartFunc { fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { let [field, _] = take_function_args(self.name(), args.scalar_arguments)?; + let nullable = args.arg_fields[1].is_nullable(); field .and_then(|sv| { @@ -160,9 +161,9 @@ impl ScalarUDFImpl for DatePartFunc { .filter(|s| !s.is_empty()) .map(|part| { if is_epoch(part) { - Field::new(self.name(), DataType::Float64, true) + Field::new(self.name(), DataType::Float64, nullable) } else { - Field::new(self.name(), DataType::Int32, true) + Field::new(self.name(), DataType::Int32, nullable) } }) }) diff --git a/datafusion/spark/src/function/datetime/date_part.rs b/datafusion/spark/src/function/datetime/date_part.rs new file mode 100644 index 000000000000..e30a162ef42d --- /dev/null +++ b/datafusion/spark/src/function/datetime/date_part.rs @@ -0,0 +1,139 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use arrow::datatypes::{DataType, Field, FieldRef}; +use datafusion_common::types::logical_date; +use datafusion_common::{ + Result, ScalarValue, internal_err, types::logical_string, utils::take_function_args, +}; +use datafusion_expr::expr::ScalarFunction; +use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext}; +use datafusion_expr::{ + Coercion, ColumnarValue, Expr, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl, + Signature, TypeSignature, TypeSignatureClass, Volatility, +}; +use std::{any::Any, sync::Arc}; + +/// Wrapper around datafusion date_part function to handle +/// Spark behavior returning day of the week 1-indexed instead of 0-indexed and different part aliases. +/// +#[derive(Debug, PartialEq, Eq, Hash)] +pub struct SparkDatePart { + signature: Signature, + aliases: Vec, +} + +impl Default for SparkDatePart { + fn default() -> Self { + Self::new() + } +} + +impl SparkDatePart { + pub fn new() -> Self { + Self { + signature: Signature::one_of( + vec![ + TypeSignature::Coercible(vec![ + Coercion::new_exact(TypeSignatureClass::Native(logical_string())), + Coercion::new_exact(TypeSignatureClass::Timestamp), + ]), + TypeSignature::Coercible(vec![ + Coercion::new_exact(TypeSignatureClass::Native(logical_string())), + Coercion::new_exact(TypeSignatureClass::Native(logical_date())), + ]), + ], + Volatility::Immutable, + ), + aliases: vec![String::from("datepart")], + } + } +} + +impl ScalarUDFImpl for SparkDatePart { + fn as_any(&self) -> &dyn Any { + self + } + + fn name(&self) -> &str { + "date_part" + } + + fn aliases(&self) -> &[String] { + &self.aliases + } + + fn signature(&self) -> &Signature { + &self.signature + } + + fn return_type(&self, _arg_types: &[DataType]) -> Result { + internal_err!("Use return_field_from_args in this case instead.") + } + + fn return_field_from_args(&self, args: ReturnFieldArgs) -> Result { + let nullable = args.arg_fields.iter().any(|f| f.is_nullable()); + + Ok(Arc::new(Field::new(self.name(), DataType::Int32, nullable))) + } + + fn invoke_with_args(&self, _args: ScalarFunctionArgs) -> Result { + internal_err!("spark date_part should have been simplified to standard date_part") + } + + fn simplify( + &self, + args: Vec, + _info: &SimplifyContext, + ) -> Result { + let [part_expr, date_expr] = take_function_args(self.name(), args)?; + + let part = match part_expr.as_literal() { + Some(ScalarValue::Utf8(Some(v))) + | Some(ScalarValue::Utf8View(Some(v))) + | Some(ScalarValue::LargeUtf8(Some(v))) => v.to_lowercase(), + _ => { + return internal_err!( + "First argument of `DATE_PART` must be non-null scalar Utf8" + ); + } + }; + + // Map Spark-specific date part aliases to datafusion ones + let part = match part.as_str() { + "yearofweek" | "year_iso" => "isoyear", + "dayofweek" => "dow", + "dayofweek_iso" | "dow_iso" => "isodow", + other => other, + }; + + let part_expr = Expr::Literal(ScalarValue::new_utf8(part), None); + + let date_part_expr = Expr::ScalarFunction(ScalarFunction::new_udf( + datafusion_functions::datetime::date_part(), + vec![part_expr, date_expr], + )); + + match part { + // Add 1 for day-of-week parts to convert 0-indexed to 1-indexed + "dow" | "isodow" => Ok(ExprSimplifyResult::Simplified( + date_part_expr + Expr::Literal(ScalarValue::Int32(Some(1)), None), + )), + _ => Ok(ExprSimplifyResult::Simplified(date_part_expr)), + } + } +} diff --git a/datafusion/spark/src/function/datetime/mod.rs b/datafusion/spark/src/function/datetime/mod.rs index 849aa2089599..99618320e102 100644 --- a/datafusion/spark/src/function/datetime/mod.rs +++ b/datafusion/spark/src/function/datetime/mod.rs @@ -16,6 +16,7 @@ // under the License. pub mod date_add; +pub mod date_part; pub mod date_sub; pub mod extract; pub mod last_day; @@ -36,6 +37,7 @@ make_udf_function!(last_day::SparkLastDay, last_day); make_udf_function!(make_dt_interval::SparkMakeDtInterval, make_dt_interval); make_udf_function!(make_interval::SparkMakeInterval, make_interval); make_udf_function!(next_day::SparkNextDay, next_day); +make_udf_function!(date_part::SparkDatePart, date_part); pub mod expr_fn { use datafusion_functions::export_functions; @@ -83,6 +85,11 @@ pub mod expr_fn { "Returns the first date which is later than start_date and named as indicated. The function returns NULL if at least one of the input parameters is NULL.", arg1 arg2 )); + export_functions!(( + date_part, + "Extracts a part of the date or time from a date, time, or timestamp expression.", + arg1 arg2 + )); } pub fn functions() -> Vec> { @@ -96,5 +103,6 @@ pub fn functions() -> Vec> { make_dt_interval(), make_interval(), next_day(), + date_part(), ] } diff --git a/datafusion/spark/src/lib.rs b/datafusion/spark/src/lib.rs index f67367734cf9..6218f5bc7b30 100644 --- a/datafusion/spark/src/lib.rs +++ b/datafusion/spark/src/lib.rs @@ -92,6 +92,24 @@ //! let expr = sha2(col("my_data"), lit(256)); //! ``` //! +//! # Example: using the Spark expression planner +//! +//! The [`planner::SparkFunctionPlanner`] provides Spark-compatible expression +//! planning, such as mapping SQL `EXTRACT` expressions to Spark's `date_part` +//! function. To use it, register it with your session context: +//! +//! ```ignore +//! use std::sync::Arc; +//! use datafusion::prelude::SessionContext; +//! use datafusion_spark::planner::SparkFunctionPlanner; +//! +//! let mut ctx = SessionContext::new(); +//! // Register the Spark expression planner +//! ctx.register_expr_planner(Arc::new(SparkFunctionPlanner))?; +//! // Now EXTRACT expressions will use Spark semantics +//! let df = ctx.sql("SELECT EXTRACT(YEAR FROM timestamp_col) FROM my_table").await?; +//! ``` +//! //![`Expr`]: datafusion_expr::Expr pub mod function; diff --git a/datafusion/spark/src/planner.rs b/datafusion/spark/src/planner.rs index 8b6861782817..2dafbb1f9a57 100644 --- a/datafusion/spark/src/planner.rs +++ b/datafusion/spark/src/planner.rs @@ -23,6 +23,15 @@ use datafusion_expr::planner::{ExprPlanner, PlannerResult}; pub struct SparkFunctionPlanner; impl ExprPlanner for SparkFunctionPlanner { + fn plan_extract( + &self, + args: Vec, + ) -> datafusion_common::Result>> { + Ok(PlannerResult::Planned(Expr::ScalarFunction( + ScalarFunction::new_udf(crate::function::datetime::date_part(), args), + ))) + } + fn plan_substring( &self, args: Vec, diff --git a/datafusion/sqllogictest/test_files/spark/datetime/date_part.slt b/datafusion/sqllogictest/test_files/spark/datetime/date_part.slt index cd3271cdc7df..48216bd55169 100644 --- a/datafusion/sqllogictest/test_files/spark/datetime/date_part.slt +++ b/datafusion/sqllogictest/test_files/spark/datetime/date_part.slt @@ -15,48 +15,262 @@ # specific language governing permissions and limitations # under the License. -# This file was originally created by a porting script from: -# https://github.com/lakehq/sail/tree/43b6ed8221de5c4c4adbedbb267ae1351158b43c/crates/sail-spark-connect/tests/gold_data/function -# This file is part of the implementation of the datafusion-spark function library. -# For more information, please see: -# https://github.com/apache/datafusion/issues/15914 - -## Original Query: SELECT date_part('MINUTE', INTERVAL '123 23:55:59.002001' DAY TO SECOND); -## PySpark 3.5.5 Result: {"date_part(MINUTE, INTERVAL '123 23:55:59.002001' DAY TO SECOND)": 55, "typeof(date_part(MINUTE, INTERVAL '123 23:55:59.002001' DAY TO SECOND))": 'tinyint', 'typeof(MINUTE)': 'string', "typeof(INTERVAL '123 23:55:59.002001' DAY TO SECOND)": 'interval day to second'} -#query -#SELECT date_part('MINUTE'::string, INTERVAL '123 23:55:59.002001' DAY TO SECOND::interval day to second); - -## Original Query: SELECT date_part('MONTH', INTERVAL '2021-11' YEAR TO MONTH); -## PySpark 3.5.5 Result: {"date_part(MONTH, INTERVAL '2021-11' YEAR TO MONTH)": 11, "typeof(date_part(MONTH, INTERVAL '2021-11' YEAR TO MONTH))": 'tinyint', 'typeof(MONTH)': 'string', "typeof(INTERVAL '2021-11' YEAR TO MONTH)": 'interval year to month'} -#query -#SELECT date_part('MONTH'::string, INTERVAL '2021-11' YEAR TO MONTH::interval year to month); - -## Original Query: SELECT date_part('SECONDS', timestamp'2019-10-01 00:00:01.000001'); -## PySpark 3.5.5 Result: {"date_part(SECONDS, TIMESTAMP '2019-10-01 00:00:01.000001')": Decimal('1.000001'), "typeof(date_part(SECONDS, TIMESTAMP '2019-10-01 00:00:01.000001'))": 'decimal(8,6)', 'typeof(SECONDS)': 'string', "typeof(TIMESTAMP '2019-10-01 00:00:01.000001')": 'timestamp'} -#query -#SELECT date_part('SECONDS'::string, TIMESTAMP '2019-10-01 00:00:01.000001'::timestamp); - -## Original Query: SELECT date_part('YEAR', TIMESTAMP '2019-08-12 01:00:00.123456'); -## PySpark 3.5.5 Result: {"date_part(YEAR, TIMESTAMP '2019-08-12 01:00:00.123456')": 2019, "typeof(date_part(YEAR, TIMESTAMP '2019-08-12 01:00:00.123456'))": 'int', 'typeof(YEAR)': 'string', "typeof(TIMESTAMP '2019-08-12 01:00:00.123456')": 'timestamp'} -#query -#SELECT date_part('YEAR'::string, TIMESTAMP '2019-08-12 01:00:00.123456'::timestamp); - -## Original Query: SELECT date_part('days', interval 5 days 3 hours 7 minutes); -## PySpark 3.5.5 Result: {"date_part(days, INTERVAL '5 03:07' DAY TO MINUTE)": 5, "typeof(date_part(days, INTERVAL '5 03:07' DAY TO MINUTE))": 'int', 'typeof(days)': 'string', "typeof(INTERVAL '5 03:07' DAY TO MINUTE)": 'interval day to minute'} -#query -#SELECT date_part('days'::string, INTERVAL '5 03:07' DAY TO MINUTE::interval day to minute); - -## Original Query: SELECT date_part('doy', DATE'2019-08-12'); -## PySpark 3.5.5 Result: {"date_part(doy, DATE '2019-08-12')": 224, "typeof(date_part(doy, DATE '2019-08-12'))": 'int', 'typeof(doy)': 'string', "typeof(DATE '2019-08-12')": 'date'} -#query -#SELECT date_part('doy'::string, DATE '2019-08-12'::date); - -## Original Query: SELECT date_part('seconds', interval 5 hours 30 seconds 1 milliseconds 1 microseconds); -## PySpark 3.5.5 Result: {"date_part(seconds, INTERVAL '05:00:30.001001' HOUR TO SECOND)": Decimal('30.001001'), "typeof(date_part(seconds, INTERVAL '05:00:30.001001' HOUR TO SECOND))": 'decimal(8,6)', 'typeof(seconds)': 'string', "typeof(INTERVAL '05:00:30.001001' HOUR TO SECOND)": 'interval hour to second'} -#query -#SELECT date_part('seconds'::string, INTERVAL '05:00:30.001001' HOUR TO SECOND::interval hour to second); - -## Original Query: SELECT date_part('week', timestamp'2019-08-12 01:00:00.123456'); -## PySpark 3.5.5 Result: {"date_part(week, TIMESTAMP '2019-08-12 01:00:00.123456')": 33, "typeof(date_part(week, TIMESTAMP '2019-08-12 01:00:00.123456'))": 'int', 'typeof(week)': 'string', "typeof(TIMESTAMP '2019-08-12 01:00:00.123456')": 'timestamp'} -#query -#SELECT date_part('week'::string, TIMESTAMP '2019-08-12 01:00:00.123456'::timestamp); +# YEAR +query I +SELECT date_part('YEAR'::string, '2000-01-01'::date); +---- +2000 + +query I +SELECT date_part('YEARS'::string, '2000-01-01'::date); +---- +2000 + +query I +SELECT date_part('Y'::string, '2000-01-01'::date); +---- +2000 + +query I +SELECT date_part('YR'::string, '2000-01-01'::date); +---- +2000 + +query I +SELECT date_part('YRS'::string, '2000-01-01'::date); +---- +2000 + +# YEAROFWEEK +query I +SELECT date_part('YEAROFWEEK'::string, '2000-01-01'::date); +---- +1999 + +# QUARTER +query I +SELECT date_part('QUARTER'::string, '2000-01-01'::date); +---- +1 + +query I +SELECT date_part('QTR'::string, '2000-01-01'::date); +---- +1 + +# MONTH +query I +SELECT date_part('MONTH'::string, '2000-01-01'::date); +---- +1 + +query I +SELECT date_part('MON'::string, '2000-01-01'::date); +---- +1 + +query I +SELECT date_part('MONS'::string, '2000-01-01'::date); +---- +1 + +query I +SELECT date_part('MONTHS'::string, '2000-01-01'::date); +---- +1 + +# WEEK +query I +SELECT date_part('WEEK'::string, '2000-01-01'::date); +---- +52 + +query I +SELECT date_part('WEEKS'::string, '2000-01-01'::date); +---- +52 + +query I +SELECT date_part('W'::string, '2000-01-01'::date); +---- +52 + +# DAYS +query I +SELECT date_part('DAY'::string, '2000-01-01'::date); +---- +1 + +query I +SELECT date_part('D'::string, '2000-01-01'::date); +---- +1 + +query I +SELECT date_part('DAYS'::string, '2000-01-01'::date); +---- +1 + +# DAYOFWEEK +query I +SELECT date_part('DAYOFWEEK'::string, '2000-01-01'::date); +---- +7 + +query I +SELECT date_part('DOW'::string, '2000-01-01'::date); +---- +7 + +# DAYOFWEEK_ISO +query I +SELECT date_part('DAYOFWEEK_ISO'::string, '2000-01-01'::date); +---- +6 + +query I +SELECT date_part('DOW_ISO'::string, '2000-01-01'::date); +---- +6 + +# DOY +query I +SELECT date_part('DOY'::string, '2000-01-01'::date); +---- +1 + +# HOUR +query I +SELECT date_part('HOUR'::string, '2000-01-01 12:30:45'::timestamp); +---- +12 + +query I +SELECT date_part('H'::string, '2000-01-01 12:30:45'::timestamp); +---- +12 + +query I +SELECT date_part('HOURS'::string, '2000-01-01 12:30:45'::timestamp); +---- +12 + +query I +SELECT date_part('HR'::string, '2000-01-01 12:30:45'::timestamp); +---- +12 + +query I +SELECT date_part('HRS'::string, '2000-01-01 12:30:45'::timestamp); +---- +12 + +# MINUTE +query I +SELECT date_part('MINUTE'::string, '2000-01-01 12:30:45'::timestamp); +---- +30 + +query I +SELECT date_part('M'::string, '2000-01-01 12:30:45'::timestamp); +---- +30 + +query I +SELECT date_part('MIN'::string, '2000-01-01 12:30:45'::timestamp); +---- +30 + +query I +SELECT date_part('MINS'::string, '2000-01-01 12:30:45'::timestamp); +---- +30 + +query I +SELECT date_part('MINUTES'::string, '2000-01-01 12:30:45'::timestamp); +---- +30 + +# SECOND +query I +SELECT date_part('SECOND'::string, '2000-01-01 12:30:45'::timestamp); +---- +45 + +query I +SELECT date_part('S'::string, '2000-01-01 12:30:45'::timestamp); +---- +45 + +query I +SELECT date_part('SEC'::string, '2000-01-01 12:30:45'::timestamp); +---- +45 + +query I +SELECT date_part('SECONDS'::string, '2000-01-01 12:30:45'::timestamp); +---- +45 + +query I +SELECT date_part('SECS'::string, '2000-01-01 12:30:45'::timestamp); +---- +45 + +# NULL input +query I +SELECT date_part('year'::string, NULL::timestamp); +---- +NULL + +query error Internal error: First argument of `DATE_PART` must be non-null scalar Utf8 +SELECT date_part(NULL::string, '2000-01-01'::date); + +# Invalid part +query error DataFusion error: Execution error: Date part 'test' not supported +SELECT date_part('test'::string, '2000-01-01'::date); + +query I +SELECT date_part('year', column1) +FROM VALUES +('2022-03-15'::date), +('1999-12-31'::date), +('2000-01-01'::date), +(NULL::date); +---- +2022 +1999 +2000 +NULL + +query I +SELECT date_part('minutes', column1) +FROM VALUES +('2022-03-15 12:30:45'::timestamp), +('1999-12-31 12:32:45'::timestamp), +('2000-01-01 12:00:45'::timestamp), +(NULL::timestamp); +---- +30 +32 +0 +NULL + +# alias datepart +query I +SELECT datepart('YEAR'::string, '2000-01-01'::date); +---- +2000 + +query I +SELECT datepart('year', column1) +FROM VALUES +('2022-03-15'::date), +('1999-12-31'::date), +('2000-01-01'::date), +(NULL::date); +---- +2022 +1999 +2000 +NULL diff --git a/datafusion/sqllogictest/test_files/spark/datetime/datepart.slt b/datafusion/sqllogictest/test_files/spark/datetime/datepart.slt deleted file mode 100644 index b2dd0089c282..000000000000 --- a/datafusion/sqllogictest/test_files/spark/datetime/datepart.slt +++ /dev/null @@ -1,62 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at - -# http://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# This file was originally created by a porting script from: -# https://github.com/lakehq/sail/tree/43b6ed8221de5c4c4adbedbb267ae1351158b43c/crates/sail-spark-connect/tests/gold_data/function -# This file is part of the implementation of the datafusion-spark function library. -# For more information, please see: -# https://github.com/apache/datafusion/issues/15914 - -## Original Query: SELECT datepart('MINUTE', INTERVAL '123 23:55:59.002001' DAY TO SECOND); -## PySpark 3.5.5 Result: {"datepart(MINUTE FROM INTERVAL '123 23:55:59.002001' DAY TO SECOND)": 55, "typeof(datepart(MINUTE FROM INTERVAL '123 23:55:59.002001' DAY TO SECOND))": 'tinyint', 'typeof(MINUTE)': 'string', "typeof(INTERVAL '123 23:55:59.002001' DAY TO SECOND)": 'interval day to second'} -#query -#SELECT datepart('MINUTE'::string, INTERVAL '123 23:55:59.002001' DAY TO SECOND::interval day to second); - -## Original Query: SELECT datepart('MONTH', INTERVAL '2021-11' YEAR TO MONTH); -## PySpark 3.5.5 Result: {"datepart(MONTH FROM INTERVAL '2021-11' YEAR TO MONTH)": 11, "typeof(datepart(MONTH FROM INTERVAL '2021-11' YEAR TO MONTH))": 'tinyint', 'typeof(MONTH)': 'string', "typeof(INTERVAL '2021-11' YEAR TO MONTH)": 'interval year to month'} -#query -#SELECT datepart('MONTH'::string, INTERVAL '2021-11' YEAR TO MONTH::interval year to month); - -## Original Query: SELECT datepart('SECONDS', timestamp'2019-10-01 00:00:01.000001'); -## PySpark 3.5.5 Result: {"datepart(SECONDS FROM TIMESTAMP '2019-10-01 00:00:01.000001')": Decimal('1.000001'), "typeof(datepart(SECONDS FROM TIMESTAMP '2019-10-01 00:00:01.000001'))": 'decimal(8,6)', 'typeof(SECONDS)': 'string', "typeof(TIMESTAMP '2019-10-01 00:00:01.000001')": 'timestamp'} -#query -#SELECT datepart('SECONDS'::string, TIMESTAMP '2019-10-01 00:00:01.000001'::timestamp); - -## Original Query: SELECT datepart('YEAR', TIMESTAMP '2019-08-12 01:00:00.123456'); -## PySpark 3.5.5 Result: {"datepart(YEAR FROM TIMESTAMP '2019-08-12 01:00:00.123456')": 2019, "typeof(datepart(YEAR FROM TIMESTAMP '2019-08-12 01:00:00.123456'))": 'int', 'typeof(YEAR)': 'string', "typeof(TIMESTAMP '2019-08-12 01:00:00.123456')": 'timestamp'} -#query -#SELECT datepart('YEAR'::string, TIMESTAMP '2019-08-12 01:00:00.123456'::timestamp); - -## Original Query: SELECT datepart('days', interval 5 days 3 hours 7 minutes); -## PySpark 3.5.5 Result: {"datepart(days FROM INTERVAL '5 03:07' DAY TO MINUTE)": 5, "typeof(datepart(days FROM INTERVAL '5 03:07' DAY TO MINUTE))": 'int', 'typeof(days)': 'string', "typeof(INTERVAL '5 03:07' DAY TO MINUTE)": 'interval day to minute'} -#query -#SELECT datepart('days'::string, INTERVAL '5 03:07' DAY TO MINUTE::interval day to minute); - -## Original Query: SELECT datepart('doy', DATE'2019-08-12'); -## PySpark 3.5.5 Result: {"datepart(doy FROM DATE '2019-08-12')": 224, "typeof(datepart(doy FROM DATE '2019-08-12'))": 'int', 'typeof(doy)': 'string', "typeof(DATE '2019-08-12')": 'date'} -#query -#SELECT datepart('doy'::string, DATE '2019-08-12'::date); - -## Original Query: SELECT datepart('seconds', interval 5 hours 30 seconds 1 milliseconds 1 microseconds); -## PySpark 3.5.5 Result: {"datepart(seconds FROM INTERVAL '05:00:30.001001' HOUR TO SECOND)": Decimal('30.001001'), "typeof(datepart(seconds FROM INTERVAL '05:00:30.001001' HOUR TO SECOND))": 'decimal(8,6)', 'typeof(seconds)': 'string', "typeof(INTERVAL '05:00:30.001001' HOUR TO SECOND)": 'interval hour to second'} -#query -#SELECT datepart('seconds'::string, INTERVAL '05:00:30.001001' HOUR TO SECOND::interval hour to second); - -## Original Query: SELECT datepart('week', timestamp'2019-08-12 01:00:00.123456'); -## PySpark 3.5.5 Result: {"datepart(week FROM TIMESTAMP '2019-08-12 01:00:00.123456')": 33, "typeof(datepart(week FROM TIMESTAMP '2019-08-12 01:00:00.123456'))": 'int', 'typeof(week)': 'string', "typeof(TIMESTAMP '2019-08-12 01:00:00.123456')": 'timestamp'} -#query -#SELECT datepart('week'::string, TIMESTAMP '2019-08-12 01:00:00.123456'::timestamp);