diff --git a/datafusion/common/src/config.rs b/datafusion/common/src/config.rs index d823883e58b5d..bbdbd4f2dcaf6 100644 --- a/datafusion/common/src/config.rs +++ b/datafusion/common/src/config.rs @@ -271,6 +271,10 @@ config_namespace! { /// When set to true, SQL parser will parse float as decimal type pub parse_float_as_decimal: bool, default = false + /// When set to true, insignificant trailing zeros are removed from decimal literals. + /// For example, `10.00` is planned as `DECIMAL(2, 0)` instead of `DECIMAL(4, 2)`. + pub trim_decimal_literal_trailing_zeros: bool, default = false + /// When set to true, SQL parser will normalize ident (convert ident to lowercase when not quoted) pub enable_ident_normalization: bool, default = true diff --git a/datafusion/core/src/execution/session_state.rs b/datafusion/core/src/execution/session_state.rs index 4c4abdebe9211..ee707162908a4 100644 --- a/datafusion/core/src/execution/session_state.rs +++ b/datafusion/core/src/execution/session_state.rs @@ -581,6 +581,8 @@ impl SessionState { ParserOptions { parse_float_as_decimal: sql_parser_options.parse_float_as_decimal, + trim_decimal_literal_trailing_zeros: sql_parser_options + .trim_decimal_literal_trailing_zeros, enable_ident_normalization: sql_parser_options.enable_ident_normalization, enable_options_value_normalization: sql_parser_options .enable_options_value_normalization, diff --git a/datafusion/sql/src/expr/value.rs b/datafusion/sql/src/expr/value.rs index 1307e917e4251..e1a6965fca590 100644 --- a/datafusion/sql/src/expr/value.rs +++ b/datafusion/sql/src/expr/value.rs @@ -23,7 +23,7 @@ use arrow::datatypes::{ DECIMAL128_MAX_PRECISION, DECIMAL256_MAX_PRECISION, FieldRef, i256, }; use bigdecimal::num_bigint::BigInt; -use bigdecimal::{BigDecimal, Signed, ToPrimitive}; +use bigdecimal::{BigDecimal, Signed, ToPrimitive, Zero}; use datafusion_common::{ DFSchema, DataFusionError, Result, ScalarValue, internal_datafusion_err, not_impl_err, plan_err, @@ -108,7 +108,11 @@ impl SqlToRel<'_, S> { } if self.options.parse_float_as_decimal { - parse_decimal(unsigned_number, negative) + parse_decimal( + unsigned_number, + negative, + self.options.trim_decimal_literal_trailing_zeros, + ) } else { signed_number.parse::().map(lit).map_err(|_| { DataFusionError::from(ParserError(format!( @@ -375,7 +379,11 @@ fn bigint_to_i256(v: &BigInt) -> Option { } } -fn parse_decimal(unsigned_number: &str, negative: bool) -> Result { +fn parse_decimal( + unsigned_number: &str, + negative: bool, + trim_trailing_zeros: bool, +) -> Result { let mut dec = BigDecimal::from_str(unsigned_number).map_err(|e| { DataFusionError::from(ParserError(format!( "Cannot parse {unsigned_number} as BigDecimal: {e}" @@ -384,9 +392,14 @@ fn parse_decimal(unsigned_number: &str, negative: bool) -> Result { if negative { dec = dec.neg(); } - - let digits = dec.digits(); - let (int_val, scale) = dec.into_bigint_and_exponent(); + let (mut int_val, mut scale) = dec.into_bigint_and_exponent(); + if trim_trailing_zeros { + while scale > 0 && (&int_val % 10_u8).is_zero() { + int_val /= 10_u8; + scale -= 1; + } + } + let digits = BigDecimal::new(int_val.clone(), scale).digits(); if scale < i8::MIN as i64 { return not_impl_err!( "Decimal scale {} exceeds the minimum supported scale: {}", @@ -394,7 +407,12 @@ fn parse_decimal(unsigned_number: &str, negative: bool) -> Result { i8::MIN ); } - let precision = if scale > 0 { + let precision = if trim_trailing_zeros && scale > 0 { + // Exact numeric types include the zero before the decimal point in their + // precision. This makes `0.00100` normalize to DECIMAL(4, 3), matching + // engines such as Snowflake. + std::cmp::max(digits, scale.unsigned_abs() + 1) + } else if scale > 0 { // arrow-rs requires the precision to include the positive scale. // See std::cmp::max(digits, scale.unsigned_abs()) @@ -509,28 +527,48 @@ mod tests { ), ]; for (input, expect) in cases { - let output = parse_decimal(input, true).unwrap(); + let output = parse_decimal(input, true, false).unwrap(); assert_eq!( output, Expr::Literal(expect.arithmetic_negate().unwrap(), None) ); - let output = parse_decimal(input, false).unwrap(); + let output = parse_decimal(input, false, false).unwrap(); assert_eq!(output, Expr::Literal(expect, None)); } // scale < i8::MIN assert_eq!( - parse_decimal("1e129", false).unwrap_err().strip_backtrace(), + parse_decimal("1e129", false, false) + .unwrap_err() + .strip_backtrace(), "This feature is not implemented: Decimal scale -129 exceeds the minimum supported scale: -128" ); // Unsupported precision assert_eq!( - parse_decimal(&"1".repeat(77), false) + parse_decimal(&"1".repeat(77), false, false) .unwrap_err() .strip_backtrace(), "This feature is not implemented: Decimal precision 77 exceeds the maximum supported precision: 76" ); } + + #[test] + fn test_parse_decimal_trims_insignificant_trailing_zeros() { + let cases = [ + ("10.00", ScalarValue::Decimal128(Some(10), 2, 0)), + ("10.10", ScalarValue::Decimal128(Some(101), 3, 1)), + ("0.00100", ScalarValue::Decimal128(Some(1), 4, 3)), + ("100.0001", ScalarValue::Decimal128(Some(1_000_001), 7, 4)), + ("1.2300e2", ScalarValue::Decimal128(Some(123), 3, 0)), + ]; + + for (input, expected) in cases { + assert_eq!( + parse_decimal(input, false, true).unwrap(), + Expr::Literal(expected, None) + ); + } + } } diff --git a/datafusion/sql/src/planner.rs b/datafusion/sql/src/planner.rs index 89af194e1a4aa..a4f42653dd352 100644 --- a/datafusion/sql/src/planner.rs +++ b/datafusion/sql/src/planner.rs @@ -45,6 +45,8 @@ use sqlparser::ast::{DataType as SQLDataType, Ident, ObjectName, TableAlias}; pub struct ParserOptions { /// Whether to parse float as decimal. pub parse_float_as_decimal: bool, + /// Whether to remove insignificant trailing zeros from decimal literals. + pub trim_decimal_literal_trailing_zeros: bool, /// Whether to normalize identifiers. pub enable_ident_normalization: bool, /// Whether to support varchar with length. @@ -73,6 +75,7 @@ impl ParserOptions { pub fn new() -> Self { Self { parse_float_as_decimal: false, + trim_decimal_literal_trailing_zeros: false, enable_ident_normalization: true, support_varchar_with_length: true, map_string_types_to_utf8view: true, @@ -98,6 +101,12 @@ impl ParserOptions { self } + /// Sets the `trim_decimal_literal_trailing_zeros` option. + pub fn with_trim_decimal_literal_trailing_zeros(mut self, value: bool) -> Self { + self.trim_decimal_literal_trailing_zeros = value; + self + } + /// Sets the `enable_ident_normalization` option. /// /// # Examples @@ -153,6 +162,8 @@ impl From<&SqlParserOptions> for ParserOptions { fn from(options: &SqlParserOptions) -> Self { Self { parse_float_as_decimal: options.parse_float_as_decimal, + trim_decimal_literal_trailing_zeros: options + .trim_decimal_literal_trailing_zeros, enable_ident_normalization: options.enable_ident_normalization, support_varchar_with_length: options.support_varchar_with_length, map_string_types_to_utf8view: options.map_string_types_to_utf8view, diff --git a/datafusion/sql/tests/sql_integration.rs b/datafusion/sql/tests/sql_integration.rs index 7855e755e1a7d..d980a4de7eb21 100644 --- a/datafusion/sql/tests/sql_integration.rs +++ b/datafusion/sql/tests/sql_integration.rs @@ -194,6 +194,21 @@ fn parse_decimals_9() { ); } +#[test] +fn parse_decimals_trim_insignificant_trailing_zeros() { + let sql = "SELECT 10.00, 10.10, 0.00100, 100.0001, 1.2300e2"; + let options = + parse_decimals_parser_options().with_trim_decimal_literal_trailing_zeros(true); + let plan = logical_plan_with_options(sql, options).unwrap(); + assert_snapshot!( + plan, + @r" + Projection: Decimal128(10,2,0), Decimal128(10.1,3,1), Decimal128(0.001,4,3), Decimal128(100.0001,7,4), Decimal128(123,3,0) + EmptyRelation: rows=1 + " + ); +} + #[test] fn parse_ident_normalization_1() { let sql = "SELECT CHARACTER_LENGTH('str')"; @@ -4066,6 +4081,7 @@ impl ScalarUDFImpl for DummyUDF { fn parse_decimals_parser_options() -> ParserOptions { ParserOptions { parse_float_as_decimal: true, + trim_decimal_literal_trailing_zeros: false, enable_ident_normalization: false, support_varchar_with_length: false, map_string_types_to_utf8view: true, @@ -4078,6 +4094,7 @@ fn parse_decimals_parser_options() -> ParserOptions { fn ident_normalization_parser_options_no_ident_normalization() -> ParserOptions { ParserOptions { parse_float_as_decimal: true, + trim_decimal_literal_trailing_zeros: false, enable_ident_normalization: false, support_varchar_with_length: false, map_string_types_to_utf8view: true, @@ -4090,6 +4107,7 @@ fn ident_normalization_parser_options_no_ident_normalization() -> ParserOptions fn ident_normalization_parser_options_ident_normalization() -> ParserOptions { ParserOptions { parse_float_as_decimal: true, + trim_decimal_literal_trailing_zeros: false, enable_ident_normalization: true, support_varchar_with_length: false, map_string_types_to_utf8view: true, diff --git a/docs/source/user-guide/configs.md b/docs/source/user-guide/configs.md index 088c94308699d..aeecf2772b869 100644 --- a/docs/source/user-guide/configs.md +++ b/docs/source/user-guide/configs.md @@ -197,6 +197,7 @@ The following configuration settings are available: | datafusion.explain.analyze_level | dev | Verbosity level for "EXPLAIN ANALYZE". Default is "dev" "summary" shows common metrics for high-level insights. "dev" provides deep operator-level introspection for developers. | | datafusion.explain.analyze_categories | all | Which metric categories to include in "EXPLAIN ANALYZE" output. Comma-separated list of: "rows", "bytes", "timing", "uncategorized". Use "none" to show plan structure only, or "all" (default) to show everything. Metrics without a declared category are treated as "uncategorized". | | datafusion.sql_parser.parse_float_as_decimal | false | When set to true, SQL parser will parse float as decimal type | +| datafusion.sql_parser.trim_decimal_literal_trailing_zeros | false | When set to true, insignificant trailing zeros are removed from decimal literals. For example, `10.00` is planned as `DECIMAL(2, 0)` instead of `DECIMAL(4, 2)`. | | datafusion.sql_parser.enable_ident_normalization | true | When set to true, SQL parser will normalize ident (convert ident to lowercase when not quoted) | | datafusion.sql_parser.enable_options_value_normalization | false | When set to true, SQL parser will normalize options value (convert value to lowercase). Note that this option is ignored and will be removed in the future. All case-insensitive values are normalized automatically. | | datafusion.sql_parser.dialect | generic | Configure the SQL dialect used by DataFusion's parser; supported values include: Generic, MySQL, PostgreSQL, Hive, SQLite, Snowflake, Redshift, MsSQL, ClickHouse, BigQuery, Ansi, DuckDB, Databricks, Spark. |