feat(rust): run cloud parquet reader in default engine (pola-rs#11196)

Author: ritchie46

.github/workflows/lint-rust.yml

@@ -38,8 +38,7 @@ jobs:
           save-if: ${{ github.ref_name == 'main' }}
 
       - name: Run cargo clippy with all features enabled
-        # not all features can combine with each other for nano-arrow
-        run: cargo clippy --workspace --all-targets --exclude nano-arrow --all-features -- -D warnings
+        run: cargo clippy -p polars --all-features -- -D warnings
 
   # Default feature set should compile on the stable toolchain
   clippy-stable:
@@ -59,7 +58,7 @@ jobs:
           save-if: ${{ github.ref_name == 'main' }}
 
       - name: Run cargo clippy
-        run: cargo clippy --workspace --all-targets --exclude nano-arrow -- -D warnings
+        run: cargo clippy -p polars -- -D warnings
 
   rustfmt:
     if: github.ref_name != 'main'

crates/Makefile

@@ -14,11 +14,11 @@ check:  ## Run cargo check with all features
 
 .PHONY: clippy
 clippy:  ## Run clippy with all features
-	cargo clippy --workspace --all-targets --exclude nano-arrow --all-features
+	cargo clippy -p polars --all-features
 
 .PHONY: clippy-default
 clippy-default:  ## Run clippy with default features
-	cargo clippy --workspace --all-targets
+	cargo clippy -p polars
 
 .PHONY: pre-commit
 pre-commit: fmt clippy clippy-default  ## Run autoformatting and linting

crates/nano-arrow/Cargo.toml

@@ -5,7 +5,6 @@ authors = ["Jorge C. Leitao <[email protected]>", "Apache Arrow <dev@arro
 edition.workspace = true
 homepage.workspace = true
 licence = "Apache 2.0 and MIT"
-license.workspace = true
 repository.workspace = true
 description = "Minimal implementation of the Arrow specification forked from arrow2."
 

crates/nano-arrow/src/array/dictionary/typed_iterator.rs

@@ -33,9 +33,9 @@ impl<O: Offset> DictValue for Utf8Array<O> {
         array
             .as_any()
             .downcast_ref::<Self>()
-            .ok_or(Error::InvalidArgumentError(
-                "could not convert array to dictionary value".into(),
-            ))
+            .ok_or_else(|| {
+                Error::InvalidArgumentError("could not convert array to dictionary value".into())
+            })
             .map(|arr| {
                 assert_eq!(
                     arr.null_count(),

crates/polars-core/Cargo.toml

@@ -26,7 +26,6 @@ indexmap = { workspace = true }
 itoap = { version = "1", optional = true, features = ["simd"] }
 ndarray = { version = "0.15", optional = true, default_features = false }
 num-traits = { workspace = true }
-object_store = { workspace = true, optional = true }
 once_cell = { workspace = true }
 rand = { workspace = true, optional = true, features = ["small_rng", "std"] }
 rand_distr = { version = "0.4", optional = true }
@@ -37,7 +36,6 @@ serde = { workspace = true, features = ["derive"], optional = true }
 serde_json = { workspace = true, optional = true }
 smartstring = { workspace = true }
 thiserror = { workspace = true }
-url = { workspace = true, optional = true }
 xxhash-rust = { workspace = true }
 
 [dev-dependencies]
@@ -182,12 +180,6 @@ docs-selection = [
   "algorithm_group_by",
 ]
 
-# Cloud support.
-"async" = ["url", "object_store"]
-"aws" = ["async", "object_store/aws"]
-"azure" = ["async", "object_store/azure"]
-"gcp" = ["async", "object_store/gcp"]
-
 [package.metadata.docs.rs]
 # not all because arrow 4.3 does not compile with simd
 # all-features = true

crates/polars-core/src/chunked_array/from.rs

@@ -1,6 +1,6 @@
 use super::*;
 
-#[allow(clippy::ptr_arg)]
+#[allow(clippy::all)]
 fn from_chunks_list_dtype(chunks: &mut Vec<ArrayRef>, dtype: DataType) -> DataType {
     // ensure we don't get List<null>
     let dtype = if let Some(arr) = chunks.get(0) {

crates/polars-core/src/chunked_array/ops/bit_repr.rs

@@ -24,6 +24,7 @@ fn reinterpret_chunked_array<T: PolarsNumericType, U: PolarsNumericType>(
 
 /// Reinterprets the type of a [`ListChunked`]. T and U must have the same size
 /// and alignment.
+#[cfg(feature = "reinterpret")]
 fn reinterpret_list_chunked<T: PolarsNumericType, U: PolarsNumericType>(
     ca: &ListChunked,
 ) -> ListChunked {

crates/polars-core/src/datatypes/mod.rs

@@ -270,6 +270,3 @@ impl NumericNative for f32 {
 impl NumericNative for f64 {
     type PolarsType = Float64Type;
 }
-
-// Provide options to cloud providers (credentials, region).
-pub type CloudOptions = PlHashMap<String, String>;

crates/polars-core/src/frame/asof_join/mod.rs

@@ -8,6 +8,7 @@ use num_traits::Bounded;
 use serde::{Deserialize, Serialize};
 use smartstring::alias::String as SmartString;
 
+use crate::frame::hash_join::_finish_join;
 use crate::prelude::*;
 use crate::utils::{ensure_sorted_arg, slice_slice};
 

crates/polars-core/src/frame/cross_join.rs

@@ -1,5 +1,6 @@
 use smartstring::alias::String as SmartString;
 
+use crate::frame::hash_join::_finish_join;
 use crate::prelude::*;
 use crate::series::IsSorted;
 use crate::utils::{concat_df_unchecked, slice_offsets, CustomIterTools, NoNull};