diff --git a/datafusion/expr-common/src/type_coercion/binary.rs b/datafusion/expr-common/src/type_coercion/binary.rs index c7a73a7c6ce67..23ccf7f81527c 100644 --- a/datafusion/expr-common/src/type_coercion/binary.rs +++ b/datafusion/expr-common/src/type_coercion/binary.rs @@ -768,6 +768,7 @@ fn type_union_resolution_coercion( } _ => binary_numeric_coercion(lhs_type, rhs_type) .or_else(|| list_coercion(lhs_type, rhs_type, type_union_resolution_coercion)) + .or_else(|| map_coercion(lhs_type, rhs_type, type_union_resolution_coercion)) .or_else(|| temporal_coercion_nonstrict_timezone(lhs_type, rhs_type)) .or_else(|| string_coercion(lhs_type, rhs_type)) .or_else(|| null_coercion(lhs_type, rhs_type)) diff --git a/datafusion/expr-common/src/type_coercion/binary/tests/comparison.rs b/datafusion/expr-common/src/type_coercion/binary/tests/comparison.rs index 5871f24e7f039..cfa3bbe189929 100644 --- a/datafusion/expr-common/src/type_coercion/binary/tests/comparison.rs +++ b/datafusion/expr-common/src/type_coercion/binary/tests/comparison.rs @@ -908,6 +908,60 @@ fn test_type_union_coercion_prefers_finer_timestamp_unit() { ); } +/// Tests that `type_union_resolution` unifies Map types by recursing into the +/// key/value types, so a Map whose value type is Null (e.g. `MAP {'k': NULL}`) +/// unifies with a concretely-typed Map in a VALUES list. +/// See . +#[test] +fn test_type_union_resolution_map() { + fn map_type(value_type: DataType) -> DataType { + DataType::Map( + Arc::new(Field::new( + "entries", + DataType::Struct(Fields::from(vec![ + Field::new("key", DataType::Utf8, false), + Field::new("value", value_type, true), + ])), + false, + )), + false, + ) + } + + // Null value type unifies with a concrete value type, in both orders + assert_eq!( + type_union_resolution(&[map_type(DataType::Int64), map_type(DataType::Null)]), + Some(map_type(DataType::Int64)) + ); + assert_eq!( + type_union_resolution(&[map_type(DataType::Null), map_type(DataType::Int64)]), + Some(map_type(DataType::Int64)) + ); + + // Numeric value types widen following the scalar rules + assert_eq!( + type_union_resolution(&[ + map_type(DataType::Int64), + map_type(DataType::Null), + map_type(DataType::Float64), + ]), + Some(map_type(DataType::Float64)) + ); + + // Map cannot unify with a non-Map composite type + assert_eq!( + type_union_resolution(&[ + map_type(DataType::Int64), + DataType::Struct(Fields::from(vec![Field::new( + "key", + DataType::Utf8, + false + )])), + ]), + None + ); +} + /// Tests that comparison operators coerce to numeric when comparing /// numeric and string types. #[test] diff --git a/datafusion/sqllogictest/test_files/map.slt b/datafusion/sqllogictest/test_files/map.slt index f8cbb395cb7f8..486a50f960f9a 100644 --- a/datafusion/sqllogictest/test_files/map.slt +++ b/datafusion/sqllogictest/test_files/map.slt @@ -920,6 +920,10 @@ SELECT map([column1, column1 * 10], ['x','y']) FROM (VALUES (1), (2), (3)) t; # tests for DISTINCT / GROUP BY / aggregation on map columns # https://github.com/apache/datafusion/issues/15428 +# NOTE: the CAST(NULL AS BIGINT) in the VALUES list below predates the fix for +# https://github.com/apache/datafusion/issues/23474 and is no longer required. +# It is kept as-is to document the historical workaround; the un-cast form is +# exercised in the "map NULL value coercion in VALUES" section further below. statement ok CREATE TABLE map_distinct_table AS VALUES (MAP {'k1': 1, 'k2': 2}, 'a', 1), @@ -1048,3 +1052,144 @@ PUT 25 statement ok DROP TABLE map_data; + +# map NULL value coercion in VALUES +# https://github.com/apache/datafusion/issues/23474 +# A bare NULL map value used to fail type unification across a VALUES list +# ("Inconsistent data type across values list") and required an explicit +# CAST(NULL AS ). The map value type now unifies with concrete value +# types following the same rules as scalar VALUES coercion. + +# concrete-typed row first, NULL-valued row second (the issue reproducer) +statement ok +CREATE TABLE map_null_concrete_first AS VALUES + (MAP {'k1': 1, 'k2': 2}), + (MAP {'k1': NULL}); + +# NULL must round-trip as NULL after coercion, not a default value +query ? rowsort +SELECT * FROM map_null_concrete_first; +---- +{k1: 1, k2: 2} +{k1: NULL} + +# the NULL value type is coerced to the concrete value type (Int64) +query T +SELECT arrow_typeof(column1) FROM map_null_concrete_first LIMIT 1; +---- +Map("entries": non-null Struct("key": non-null Utf8, "value": Int64), unsorted) + +statement ok +DROP TABLE map_null_concrete_first; + +# NULL-valued row first, concrete-typed row second (coercion is symmetric) +statement ok +CREATE TABLE map_null_first AS VALUES + (MAP {'k1': NULL}), + (MAP {'k1': 1, 'k2': 2}); + +query ? rowsort +SELECT * FROM map_null_first; +---- +{k1: 1, k2: 2} +{k1: NULL} + +statement ok +DROP TABLE map_null_first; + +# every row has a NULL value: succeeds and the value type stays Null +statement ok +CREATE TABLE map_all_null_values AS VALUES + (MAP {'k': NULL}), + (MAP {'k': NULL}); + +query ? rowsort +SELECT * FROM map_all_null_values; +---- +{k: NULL} +{k: NULL} + +query T +SELECT arrow_typeof(column1) FROM map_all_null_values LIMIT 1; +---- +Map("entries": non-null Struct("key": non-null Utf8, "value": Null), unsorted) + +statement ok +DROP TABLE map_all_null_values; + +# multiple keys where only one value is NULL +query ? rowsort +SELECT * FROM (VALUES + (MAP {'a': 1, 'b': NULL}), + (MAP {'a': 2, 'b': 3})) t(column1); +---- +{a: 1, b: NULL} +{a: 2, b: 3} + +# three rows with the NULL-valued row in the middle +query ? rowsort +SELECT * FROM (VALUES + (MAP {'k': 1}), + (MAP {'k': NULL}), + (MAP {'k': 2})) t(column1); +---- +{k: 1} +{k: 2} +{k: NULL} + +# numeric widening across a NULL-valued row follows the scalar rule +# (Int64 + Float64 -> Float64) +statement ok +CREATE TABLE map_null_widening AS VALUES + (MAP {'k': 1}), + (MAP {'k': NULL}), + (MAP {'k': 1.5}); + +query ? rowsort +SELECT * FROM map_null_widening; +---- +{k: 1.0} +{k: 1.5} +{k: NULL} + +query T +SELECT arrow_typeof(column1) FROM map_null_widening LIMIT 1; +---- +Map("entries": non-null Struct("key": non-null Utf8, "value": Float64), unsorted) + +statement ok +DROP TABLE map_null_widening; + +# incompatible concrete value types with a NULL-valued row in between still +# error; Int64/Utf8 follows the scalar VALUES rule (coerce to the numeric +# type, then fail to cast the non-numeric string) +query error Cast error: Cannot cast string 'hello' to value of Int64 type +SELECT * FROM (VALUES + (MAP {'k': 1}), + (MAP {'k': NULL}), + (MAP {'k': 'hello'})) t(column1); + +# NULL value type unification recurses into nested maps +query ? rowsort +SELECT * FROM (VALUES + (MAP {'outer': MAP {'inner': 1}}), + (MAP {'outer': MAP {'inner': NULL}})) t(column1); +---- +{outer: {inner: 1}} +{outer: {inner: NULL}} + +# INSERT INTO ... VALUES also accepts a NULL map value without a cast +statement ok +CREATE TABLE map_null_insert AS VALUES (MAP {'k1': 1, 'k2': 2}); + +statement ok +INSERT INTO map_null_insert VALUES (MAP {'k1': NULL}); + +query ? rowsort +SELECT * FROM map_null_insert; +---- +{k1: 1, k2: 2} +{k1: NULL} + +statement ok +DROP TABLE map_null_insert;