Skip to content

Add optional decimal literal normalization #50

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
osipovartem merged 2 commits into from
Sep 1, 2026
Merged

Add optional decimal literal normalization #50

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
dd65516
Add optional decimal literal normalization
osipovartem Sep 1, 2026
970c995
Merge remote-tracking branch 'origin/expr-planner-aggregate-schema' i…
osipovartem Sep 1, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions datafusion/common/src/config.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -271,6 +271,10 @@ config_namespace! {
/// When set to true, SQL parser will parse float as decimal type
pub parse_float_as_decimal: bool, default = false

/// When set to true, insignificant trailing zeros are removed from decimal literals.
/// For example, `10.00` is planned as `DECIMAL(2, 0)` instead of `DECIMAL(4, 2)`.
pub trim_decimal_literal_trailing_zeros: bool, default = false

/// When set to true, SQL parser will normalize ident (convert ident to lowercase when not quoted)
pub enable_ident_normalization: bool, default = true

Expand Down
2 changes: 2 additions & 0 deletions datafusion/core/src/execution/session_state.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -581,6 +581,8 @@ impl SessionState {

ParserOptions {
parse_float_as_decimal: sql_parser_options.parse_float_as_decimal,
trim_decimal_literal_trailing_zeros: sql_parser_options
.trim_decimal_literal_trailing_zeros,
enable_ident_normalization: sql_parser_options.enable_ident_normalization,
enable_options_value_normalization: sql_parser_options
.enable_options_value_normalization,
Expand Down
60 changes: 49 additions & 11 deletions datafusion/sql/src/expr/value.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ use arrow::datatypes::{
DECIMAL128_MAX_PRECISION, DECIMAL256_MAX_PRECISION, FieldRef, i256,
};
use bigdecimal::num_bigint::BigInt;
use bigdecimal::{BigDecimal, Signed, ToPrimitive};
use bigdecimal::{BigDecimal, Signed, ToPrimitive, Zero};
use datafusion_common::{
DFSchema, DataFusionError, Result, ScalarValue, internal_datafusion_err,
not_impl_err, plan_err,
Expand Down Expand Up @@ -108,7 +108,11 @@ impl<S: ContextProvider> SqlToRel<'_, S> {
}

if self.options.parse_float_as_decimal {
parse_decimal(unsigned_number, negative)
parse_decimal(
unsigned_number,
negative,
self.options.trim_decimal_literal_trailing_zeros,
)
} else {
signed_number.parse::<f64>().map(lit).map_err(|_| {
DataFusionError::from(ParserError(format!(
Expand Down Expand Up @@ -375,7 +379,11 @@ fn bigint_to_i256(v: &BigInt) -> Option<i256> {
}
}

fn parse_decimal(unsigned_number: &str, negative: bool) -> Result<Expr> {
fn parse_decimal(
unsigned_number: &str,
negative: bool,
trim_trailing_zeros: bool,
) -> Result<Expr> {
let mut dec = BigDecimal::from_str(unsigned_number).map_err(|e| {
DataFusionError::from(ParserError(format!(
"Cannot parse {unsigned_number} as BigDecimal: {e}"
Expand All @@ -384,17 +392,27 @@ fn parse_decimal(unsigned_number: &str, negative: bool) -> Result<Expr> {
if negative {
dec = dec.neg();
}

let digits = dec.digits();
let (int_val, scale) = dec.into_bigint_and_exponent();
let (mut int_val, mut scale) = dec.into_bigint_and_exponent();
if trim_trailing_zeros {
while scale > 0 && (&int_val % 10_u8).is_zero() {
int_val /= 10_u8;
scale -= 1;
}
}
let digits = BigDecimal::new(int_val.clone(), scale).digits();
if scale < i8::MIN as i64 {
return not_impl_err!(
"Decimal scale {} exceeds the minimum supported scale: {}",
scale,
i8::MIN
);
}
let precision = if scale > 0 {
let precision = if trim_trailing_zeros && scale > 0 {
// Exact numeric types include the zero before the decimal point in their
// precision. This makes `0.00100` normalize to DECIMAL(4, 3), matching
// engines such as Snowflake.
std::cmp::max(digits, scale.unsigned_abs() + 1)
} else if scale > 0 {
// arrow-rs requires the precision to include the positive scale.
// See <https://github.com/apache/arrow-rs/blob/123045cc766d42d1eb06ee8bb3f09e39ea995ddc/arrow-array/src/types.rs#L1230>
std::cmp::max(digits, scale.unsigned_abs())
Expand Down Expand Up @@ -509,28 +527,48 @@ mod tests {
),
];
for (input, expect) in cases {
let output = parse_decimal(input, true).unwrap();
let output = parse_decimal(input, true, false).unwrap();
assert_eq!(
output,
Expr::Literal(expect.arithmetic_negate().unwrap(), None)
);

let output = parse_decimal(input, false).unwrap();
let output = parse_decimal(input, false, false).unwrap();
assert_eq!(output, Expr::Literal(expect, None));
}

// scale < i8::MIN
assert_eq!(
parse_decimal("1e129", false).unwrap_err().strip_backtrace(),
parse_decimal("1e129", false, false)
.unwrap_err()
.strip_backtrace(),
"This feature is not implemented: Decimal scale -129 exceeds the minimum supported scale: -128"
);

// Unsupported precision
assert_eq!(
parse_decimal(&"1".repeat(77), false)
parse_decimal(&"1".repeat(77), false, false)
.unwrap_err()
.strip_backtrace(),
"This feature is not implemented: Decimal precision 77 exceeds the maximum supported precision: 76"
);
}

#[test]
fn test_parse_decimal_trims_insignificant_trailing_zeros() {
let cases = [
("10.00", ScalarValue::Decimal128(Some(10), 2, 0)),
("10.10", ScalarValue::Decimal128(Some(101), 3, 1)),
("0.00100", ScalarValue::Decimal128(Some(1), 4, 3)),
("100.0001", ScalarValue::Decimal128(Some(1_000_001), 7, 4)),
("1.2300e2", ScalarValue::Decimal128(Some(123), 3, 0)),
];

for (input, expected) in cases {
assert_eq!(
parse_decimal(input, false, true).unwrap(),
Expr::Literal(expected, None)
);
}
}
}
11 changes: 11 additions & 0 deletions datafusion/sql/src/planner.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,8 @@ use sqlparser::ast::{DataType as SQLDataType, Ident, ObjectName, TableAlias};
pub struct ParserOptions {
/// Whether to parse float as decimal.
pub parse_float_as_decimal: bool,
/// Whether to remove insignificant trailing zeros from decimal literals.
pub trim_decimal_literal_trailing_zeros: bool,
/// Whether to normalize identifiers.
pub enable_ident_normalization: bool,
/// Whether to support varchar with length.
Expand Down Expand Up @@ -73,6 +75,7 @@ impl ParserOptions {
pub fn new() -> Self {
Self {
parse_float_as_decimal: false,
trim_decimal_literal_trailing_zeros: false,
enable_ident_normalization: true,
support_varchar_with_length: true,
map_string_types_to_utf8view: true,
Expand All @@ -98,6 +101,12 @@ impl ParserOptions {
self
}

/// Sets the `trim_decimal_literal_trailing_zeros` option.
pub fn with_trim_decimal_literal_trailing_zeros(mut self, value: bool) -> Self {
self.trim_decimal_literal_trailing_zeros = value;
self
}

/// Sets the `enable_ident_normalization` option.
///
/// # Examples
Expand Down Expand Up @@ -153,6 +162,8 @@ impl From<&SqlParserOptions> for ParserOptions {
fn from(options: &SqlParserOptions) -> Self {
Self {
parse_float_as_decimal: options.parse_float_as_decimal,
trim_decimal_literal_trailing_zeros: options
.trim_decimal_literal_trailing_zeros,
enable_ident_normalization: options.enable_ident_normalization,
support_varchar_with_length: options.support_varchar_with_length,
map_string_types_to_utf8view: options.map_string_types_to_utf8view,
Expand Down
18 changes: 18 additions & 0 deletions datafusion/sql/tests/sql_integration.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -194,6 +194,21 @@ fn parse_decimals_9() {
);
}

#[test]
fn parse_decimals_trim_insignificant_trailing_zeros() {
let sql = "SELECT 10.00, 10.10, 0.00100, 100.0001, 1.2300e2";
let options =
parse_decimals_parser_options().with_trim_decimal_literal_trailing_zeros(true);
let plan = logical_plan_with_options(sql, options).unwrap();
assert_snapshot!(
plan,
@r"
Projection: Decimal128(10,2,0), Decimal128(10.1,3,1), Decimal128(0.001,4,3), Decimal128(100.0001,7,4), Decimal128(123,3,0)
EmptyRelation: rows=1
"
);
}

#[test]
fn parse_ident_normalization_1() {
let sql = "SELECT CHARACTER_LENGTH('str')";
Expand Down Expand Up @@ -4066,6 +4081,7 @@ impl ScalarUDFImpl for DummyUDF {
fn parse_decimals_parser_options() -> ParserOptions {
ParserOptions {
parse_float_as_decimal: true,
trim_decimal_literal_trailing_zeros: false,
enable_ident_normalization: false,
support_varchar_with_length: false,
map_string_types_to_utf8view: true,
Expand All @@ -4078,6 +4094,7 @@ fn parse_decimals_parser_options() -> ParserOptions {
fn ident_normalization_parser_options_no_ident_normalization() -> ParserOptions {
ParserOptions {
parse_float_as_decimal: true,
trim_decimal_literal_trailing_zeros: false,
enable_ident_normalization: false,
support_varchar_with_length: false,
map_string_types_to_utf8view: true,
Expand All @@ -4090,6 +4107,7 @@ fn ident_normalization_parser_options_no_ident_normalization() -> ParserOptions
fn ident_normalization_parser_options_ident_normalization() -> ParserOptions {
ParserOptions {
parse_float_as_decimal: true,
trim_decimal_literal_trailing_zeros: false,
enable_ident_normalization: true,
support_varchar_with_length: false,
map_string_types_to_utf8view: true,
Expand Down
1 change: 1 addition & 0 deletions docs/source/user-guide/configs.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,7 @@ The following configuration settings are available:
| datafusion.explain.analyze_level | dev | Verbosity level for "EXPLAIN ANALYZE". Default is "dev" "summary" shows common metrics for high-level insights. "dev" provides deep operator-level introspection for developers. |
| datafusion.explain.analyze_categories | all | Which metric categories to include in "EXPLAIN ANALYZE" output. Comma-separated list of: "rows", "bytes", "timing", "uncategorized". Use "none" to show plan structure only, or "all" (default) to show everything. Metrics without a declared category are treated as "uncategorized". |
| datafusion.sql_parser.parse_float_as_decimal | false | When set to true, SQL parser will parse float as decimal type |
| datafusion.sql_parser.trim_decimal_literal_trailing_zeros | false | When set to true, insignificant trailing zeros are removed from decimal literals. For example, `10.00` is planned as `DECIMAL(2, 0)` instead of `DECIMAL(4, 2)`. |
| datafusion.sql_parser.enable_ident_normalization | true | When set to true, SQL parser will normalize ident (convert ident to lowercase when not quoted) |
| datafusion.sql_parser.enable_options_value_normalization | false | When set to true, SQL parser will normalize options value (convert value to lowercase). Note that this option is ignored and will be removed in the future. All case-insensitive values are normalized automatically. |
| datafusion.sql_parser.dialect | generic | Configure the SQL dialect used by DataFusion's parser; supported values include: Generic, MySQL, PostgreSQL, Hive, SQLite, Snowflake, Redshift, MsSQL, ClickHouse, BigQuery, Ansi, DuckDB, Databricks, Spark. |
Expand Down
Loading