diff --git a/Cargo.toml b/Cargo.toml index a28b9d1c9f..18f1ba91a8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -26,7 +26,7 @@ debug = true debug = "line-tables-only" [workspace.dependencies] -delta_kernel = { version = "0.5.0", features = ["default-engine"] } +delta_kernel = { version = "0.6.0", features = ["default-engine"] } #delta_kernel = { path = "../delta-kernel-rs/kernel", features = ["sync-engine"] } # arrow @@ -45,16 +45,16 @@ object_store = { version = "0.11" } parquet = { version = "53" } # datafusion -datafusion = { version = "43" } -datafusion-expr = { version = "43" } -datafusion-common = { version = "43" } -datafusion-ffi = { version = "43" } -datafusion-functions = { version = "43" } -datafusion-functions-aggregate = { version = "43" } -datafusion-physical-expr = { version = "43" } -datafusion-physical-plan = { version = "43" } -datafusion-proto = { version = "43" } -datafusion-sql = { version = "43" } +datafusion = { version = "44" } +datafusion-expr = { version = "44" } +datafusion-common = { version = "44" } +datafusion-ffi = { version = "44" } +datafusion-functions = { version = "44" } +datafusion-functions-aggregate = { version = "44" } +datafusion-physical-expr = { version = "44" } +datafusion-physical-plan = { version = "44" } +datafusion-proto = { version = "44" } +datafusion-sql = { version = "44" } # serde serde = { version = "1.0.194", features = ["derive"] } @@ -78,14 +78,14 @@ num_cpus = { version = "1" } # temporary datafusion patches [patch.crates-io] -datafusion = { git = "https://github.com/apache/datafusion.git", rev = "a50ed3488f77743a192d9e8dd9c99f00df659ef1" } -datafusion-common = { git = "https://github.com/apache/datafusion.git", rev = "a50ed3488f77743a192d9e8dd9c99f00df659ef1" } -datafusion-execution = { git = "https://github.com/apache/datafusion.git", rev = "a50ed3488f77743a192d9e8dd9c99f00df659ef1" } -datafusion-expr = { git = "https://github.com/apache/datafusion.git", rev = "a50ed3488f77743a192d9e8dd9c99f00df659ef1" } -datafusion-ffi = { git = "https://github.com/apache/datafusion.git", rev = "a50ed3488f77743a192d9e8dd9c99f00df659ef1" } -datafusion-functions = { git = "https://github.com/apache/datafusion.git", rev = "a50ed3488f77743a192d9e8dd9c99f00df659ef1" } -datafusion-functions-aggregate = { git = "https://github.com/apache/datafusion.git", rev = "a50ed3488f77743a192d9e8dd9c99f00df659ef1" } -datafusion-physical-expr = { git = "https://github.com/apache/datafusion.git", rev = "a50ed3488f77743a192d9e8dd9c99f00df659ef1" } -datafusion-physical-plan = { git = "https://github.com/apache/datafusion.git", rev = "a50ed3488f77743a192d9e8dd9c99f00df659ef1" } -datafusion-proto = { git = "https://github.com/apache/datafusion.git", rev = "a50ed3488f77743a192d9e8dd9c99f00df659ef1" } -datafusion-sql = { git = "https://github.com/apache/datafusion.git", rev = "a50ed3488f77743a192d9e8dd9c99f00df659ef1" } +datafusion = { git = "https://github.com/apache/datafusion.git", rev = "073a3b110852f97ccb7085ce4bfd19473b8a3f4f" } +datafusion-common = { git = "https://github.com/apache/datafusion.git", rev = "073a3b110852f97ccb7085ce4bfd19473b8a3f4f" } +datafusion-execution = { git = "https://github.com/apache/datafusion.git", rev = "073a3b110852f97ccb7085ce4bfd19473b8a3f4f" } +datafusion-expr = { git = "https://github.com/apache/datafusion.git", rev = "073a3b110852f97ccb7085ce4bfd19473b8a3f4f" } +datafusion-ffi = { git = "https://github.com/apache/datafusion.git", rev = "073a3b110852f97ccb7085ce4bfd19473b8a3f4f" } +datafusion-functions = { git = "https://github.com/apache/datafusion.git", rev = "073a3b110852f97ccb7085ce4bfd19473b8a3f4f" } +datafusion-functions-aggregate = { git = "https://github.com/apache/datafusion.git", rev = "073a3b110852f97ccb7085ce4bfd19473b8a3f4f" } +datafusion-physical-expr = { git = "https://github.com/apache/datafusion.git", rev = "073a3b110852f97ccb7085ce4bfd19473b8a3f4f" } +datafusion-physical-plan = { git = "https://github.com/apache/datafusion.git", rev = "073a3b110852f97ccb7085ce4bfd19473b8a3f4f" } +datafusion-proto = { git = "https://github.com/apache/datafusion.git", rev = "073a3b110852f97ccb7085ce4bfd19473b8a3f4f" } +datafusion-sql = { git = "https://github.com/apache/datafusion.git", rev = "073a3b110852f97ccb7085ce4bfd19473b8a3f4f" } diff --git a/crates/core/src/table/state_arrow.rs b/crates/core/src/table/state_arrow.rs index 0258109859..4ea67cd9ef 100644 --- a/crates/core/src/table/state_arrow.rs +++ b/crates/core/src/table/state_arrow.rs @@ -14,7 +14,7 @@ use arrow_array::{ use arrow_cast::cast; use arrow_cast::parse::Parser; use arrow_schema::{DataType, Field, Fields, TimeUnit}; -use delta_kernel::table_features::ColumnMappingMode; +use delta_kernel::table_features::{validate_schema_column_mapping, ColumnMappingMode}; use itertools::Itertools; use super::state::DeltaTableState; @@ -171,6 +171,8 @@ impl DeltaTableState { }) .collect::>(); + validate_schema_column_mapping(self.schema(), column_mapping_mode)?; + let physical_name_to_logical_name = match column_mapping_mode { ColumnMappingMode::None => HashMap::with_capacity(0), // No column mapping, no need for this HashMap ColumnMappingMode::Id | ColumnMappingMode::Name => metadata @@ -184,7 +186,7 @@ impl DeltaTableState { "Invalid partition column {0}", name )))? - .physical_name(column_mapping_mode)? + .physical_name() .to_string(); Ok((physical_name, name.as_str())) })