upmerge

Cargo.toml

@@ -33,6 +33,7 @@ bytes = "1.8.0"
 datafusion = { version = "42.0.0", features = ["pyarrow", "avro"] }
 datafusion-proto = "42.0.0"
 futures = "0.3"
+glob = "0.3.1"
 log = "0.4"
 object_store = { version = "0.11.1", features = ["aws"] }
 prost = "0.13"

src/shuffle/writer.rs

@@ -24,16 +24,18 @@ use datafusion::arrow::record_batch::RecordBatch;
 use datafusion::arrow::util::pretty::pretty_format_batches;
 use datafusion::common::{DataFusionError, Result, Statistics};
 use datafusion::execution::context::TaskContext;
+use datafusion::execution::RecordBatchStream;
 use datafusion::physical_expr::expressions::UnKnownColumn;
 use datafusion::physical_expr::EquivalenceProperties;
 use datafusion::physical_plan::memory::MemoryStream;
 use datafusion::physical_plan::metrics::{ExecutionPlanMetricsSet, MetricBuilder};
 use datafusion::physical_plan::repartition::BatchPartitioner;
 use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
 use datafusion::physical_plan::{
-    DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties,
+    metrics, DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, PlanProperties,
     SendableRecordBatchStream,
 };
+use datafusion_proto::protobuf::PartitionStats;
 use futures::StreamExt;
 use futures::TryStreamExt;
 use log::debug;
@@ -43,6 +45,7 @@ use std::fmt::Formatter;
 use std::fs::File;
 use std::io::Read;
 use std::path::{Path, PathBuf};
+use std::pin::Pin;
 use std::sync::Arc;
 use std::time::Instant;