Skip to content

Commit 9371dd1

Browse files
committed
Merge branch 'main' into arrow_rs_9683
2 parents e1e1d09 + 0143dfe commit 9371dd1

129 files changed

Lines changed: 7071 additions & 2390 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎datafusion-cli/src/catalog.rs‎

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@
1515
// specific language governing permissions and limitations
1616
// under the License.
1717

18-
use std::any::Any;
1918
use std::sync::{Arc, Weak};
2019

2120
use crate::object_storage::{AwsOptions, GcpOptions, get_object_store};
@@ -50,10 +49,6 @@ impl DynamicObjectStoreCatalog {
5049
}
5150

5251
impl CatalogProviderList for DynamicObjectStoreCatalog {
53-
fn as_any(&self) -> &dyn Any {
54-
self
55-
}
56-
5752
fn register_catalog(
5853
&self,
5954
name: String,
@@ -91,10 +86,6 @@ impl DynamicObjectStoreCatalogProvider {
9186
}
9287

9388
impl CatalogProvider for DynamicObjectStoreCatalogProvider {
94-
fn as_any(&self) -> &dyn Any {
95-
self
96-
}
97-
9889
fn schema_names(&self) -> Vec<String> {
9990
self.inner.schema_names()
10091
}
@@ -134,10 +125,6 @@ impl DynamicObjectStoreSchemaProvider {
134125

135126
#[async_trait]
136127
impl SchemaProvider for DynamicObjectStoreSchemaProvider {
137-
fn as_any(&self) -> &dyn Any {
138-
self
139-
}
140-
141128
fn table_names(&self) -> Vec<String> {
142129
self.inner.table_names()
143130
}

‎datafusion-cli/src/functions.rs‎

Lines changed: 0 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -229,10 +229,6 @@ struct ParquetMetadataTable {
229229

230230
#[async_trait]
231231
impl TableProvider for ParquetMetadataTable {
232-
fn as_any(&self) -> &dyn std::any::Any {
233-
self
234-
}
235-
236232
fn schema(&self) -> SchemaRef {
237233
self.schema.clone()
238234
}
@@ -479,10 +475,6 @@ struct MetadataCacheTable {
479475

480476
#[async_trait]
481477
impl TableProvider for MetadataCacheTable {
482-
fn as_any(&self) -> &dyn std::any::Any {
483-
self
484-
}
485-
486478
fn schema(&self) -> SchemaRef {
487479
self.schema.clone()
488480
}
@@ -598,10 +590,6 @@ struct StatisticsCacheTable {
598590

599591
#[async_trait]
600592
impl TableProvider for StatisticsCacheTable {
601-
fn as_any(&self) -> &dyn std::any::Any {
602-
self
603-
}
604-
605593
fn schema(&self) -> SchemaRef {
606594
self.schema.clone()
607595
}
@@ -734,10 +722,6 @@ struct ListFilesCacheTable {
734722

735723
#[async_trait]
736724
impl TableProvider for ListFilesCacheTable {
737-
fn as_any(&self) -> &dyn std::any::Any {
738-
self
739-
}
740-
741725
fn schema(&self) -> SchemaRef {
742726
self.schema.clone()
743727
}

‎datafusion-examples/examples/custom_data_source/custom_datasource.rs‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717

1818
//! See `main.rs` for how to run it.
1919
20-
use std::any::Any;
2120
use std::collections::{BTreeMap, HashMap};
2221
use std::fmt::{self, Debug, Formatter};
2322
use std::sync::{Arc, Mutex};
@@ -162,10 +161,6 @@ impl Default for CustomDataSource {
162161

163162
#[async_trait]
164163
impl TableProvider for CustomDataSource {
165-
fn as_any(&self) -> &dyn Any {
166-
self
167-
}
168-
169164
fn schema(&self) -> SchemaRef {
170165
SchemaRef::new(Schema::new(vec![
171166
Field::new("id", DataType::UInt8, false),

‎datafusion-examples/examples/custom_data_source/default_column_values.rs‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717

1818
//! See `main.rs` for how to run it.
1919
20-
use std::any::Any;
2120
use std::collections::HashMap;
2221
use std::sync::Arc;
2322

@@ -201,10 +200,6 @@ impl DefaultValueTableProvider {
201200

202201
#[async_trait]
203202
impl TableProvider for DefaultValueTableProvider {
204-
fn as_any(&self) -> &dyn Any {
205-
self
206-
}
207-
208203
fn schema(&self) -> SchemaRef {
209204
Arc::clone(&self.schema)
210205
}

‎datafusion-examples/examples/data_io/catalog.rs‎

Lines changed: 1 addition & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ use datafusion::{
3232
prelude::SessionContext,
3333
};
3434
use std::sync::RwLock;
35-
use std::{any::Any, collections::HashMap, path::Path, sync::Arc};
35+
use std::{collections::HashMap, path::Path, sync::Arc};
3636
use std::{fs::File, io::Write};
3737
use tempfile::TempDir;
3838

@@ -178,10 +178,6 @@ impl DirSchema {
178178

179179
#[async_trait]
180180
impl SchemaProvider for DirSchema {
181-
fn as_any(&self) -> &dyn Any {
182-
self
183-
}
184-
185181
fn table_names(&self) -> Vec<String> {
186182
let tables = self.tables.read().unwrap();
187183
tables.keys().cloned().collect::<Vec<_>>()
@@ -231,10 +227,6 @@ impl DirCatalog {
231227
}
232228

233229
impl CatalogProvider for DirCatalog {
234-
fn as_any(&self) -> &dyn Any {
235-
self
236-
}
237-
238230
fn register_schema(
239231
&self,
240232
name: &str,
@@ -277,10 +269,6 @@ impl CustomCatalogProviderList {
277269
}
278270

279271
impl CatalogProviderList for CustomCatalogProviderList {
280-
fn as_any(&self) -> &dyn Any {
281-
self
282-
}
283-
284272
fn register_catalog(
285273
&self,
286274
name: String,

‎datafusion-examples/examples/data_io/parquet_advanced_index.rs‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717

1818
//! See `main.rs` for how to run it.
1919
20-
use std::any::Any;
2120
use std::collections::{HashMap, HashSet};
2221
use std::fs::File;
2322
use std::ops::Range;
@@ -451,10 +450,6 @@ impl IndexedFile {
451450
/// so that we can query it as a table.
452451
#[async_trait]
453452
impl TableProvider for IndexTableProvider {
454-
fn as_any(&self) -> &dyn Any {
455-
self
456-
}
457-
458453
fn schema(&self) -> SchemaRef {
459454
Arc::clone(&self.indexed_file.schema)
460455
}

‎datafusion-examples/examples/data_io/parquet_embedded_index.rs‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -393,9 +393,6 @@ fn get_key_value<'a>(file_meta_data: &'a FileMetaData, key: &'_ str) -> Option<&
393393
/// Implement TableProvider for DistinctIndexTable, using the distinct index to prune files
394394
#[async_trait]
395395
impl TableProvider for DistinctIndexTable {
396-
fn as_any(&self) -> &dyn std::any::Any {
397-
self
398-
}
399396
fn schema(&self) -> SchemaRef {
400397
self.schema.clone()
401398
}

‎datafusion-examples/examples/data_io/parquet_index.rs‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,6 @@ use datafusion::physical_expr::PhysicalExpr;
4545
use datafusion::physical_optimizer::pruning::PruningPredicate;
4646
use datafusion::physical_plan::ExecutionPlan;
4747
use datafusion::prelude::*;
48-
use std::any::Any;
4948
use std::collections::HashSet;
5049
use std::fmt::Display;
5150
use std::fs;
@@ -208,10 +207,6 @@ impl IndexTableProvider {
208207

209208
#[async_trait]
210209
impl TableProvider for IndexTableProvider {
211-
fn as_any(&self) -> &dyn Any {
212-
self
213-
}
214-
215210
fn schema(&self) -> SchemaRef {
216211
self.index.schema().clone()
217212
}

‎datafusion-examples/examples/data_io/remote_catalog.rs‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,6 @@ use datafusion::physical_plan::ExecutionPlan;
4545
use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
4646
use datafusion::prelude::{DataFrame, SessionContext};
4747
use futures::TryStreamExt;
48-
use std::any::Any;
4948
use std::sync::Arc;
5049

5150
/// Interfacing with a remote catalog (e.g. over a network)
@@ -224,10 +223,6 @@ impl RemoteTable {
224223
/// Implement the DataFusion Catalog API for [`RemoteTable`]
225224
#[async_trait]
226225
impl TableProvider for RemoteTable {
227-
fn as_any(&self) -> &dyn Any {
228-
self
229-
}
230-
231226
fn schema(&self) -> SchemaRef {
232227
self.schema.clone()
233228
}

‎datafusion-examples/examples/relation_planner/pivot_unpivot.rs‎

Lines changed: 53 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -217,6 +217,25 @@ async fn run_examples(ctx: &SessionContext) -> Result<()> {
217217
+--------+
218218
");
219219

220+
// Example 7: PIVOT on a quoted mixed-case column
221+
// Reuses the parsed column expression so quoted identifiers keep their case.
222+
let results = run_example(
223+
ctx,
224+
"Example 7: PIVOT with quoted mixed-case column",
225+
r#"SELECT * FROM point_stats
226+
PIVOT (MAX(max_value) FOR "pointNumber" IN ('16951' AS p16951, '16952' AS p16952)) AS p
227+
ORDER BY ts"#,
228+
)
229+
.await?;
230+
assert_snapshot!(results, @r"
231+
+----------------------+------+--------+--------+
232+
| ts | port | p16951 | p16952 |
233+
+----------------------+------+--------+--------+
234+
| 2024-09-01T10:00:00Z | 2411 | 10 | 20 |
235+
| 2024-09-01T10:01:00Z | 2411 | 30 | 40 |
236+
+----------------------+------+--------+--------+
237+
");
238+
220239
Ok(())
221240
}
222241

@@ -288,6 +307,34 @@ fn register_sample_data(ctx: &SessionContext) -> Result<()> {
288307
])?,
289308
)?;
290309

310+
// point_stats: grouped data with a quoted mixed-case pivot column.
311+
ctx.register_batch(
312+
"point_stats",
313+
RecordBatch::try_from_iter(vec![
314+
(
315+
"ts",
316+
Arc::new(StringArray::from(vec![
317+
"2024-09-01T10:00:00Z",
318+
"2024-09-01T10:00:00Z",
319+
"2024-09-01T10:01:00Z",
320+
"2024-09-01T10:01:00Z",
321+
])) as ArrayRef,
322+
),
323+
(
324+
"pointNumber",
325+
Arc::new(StringArray::from(vec!["16951", "16952", "16951", "16952"])),
326+
),
327+
(
328+
"port",
329+
Arc::new(StringArray::from(vec!["2411", "2411", "2411", "2411"])),
330+
),
331+
(
332+
"max_value",
333+
Arc::new(Int64Array::from(vec![10, 20, 30, 40])),
334+
),
335+
])?,
336+
)?;
337+
291338
Ok(())
292339
}
293340

@@ -415,11 +462,12 @@ fn plan_pivot(
415462
.collect();
416463

417464
let group_by_cols: Vec<Expr> = schema
418-
.fields()
419465
.iter()
420-
.map(|f| f.name().as_str())
421-
.filter(|name| *name != pivot_col_name.as_str() && !agg_input_cols.contains(name))
422-
.map(col)
466+
.filter(|(_, field)| {
467+
let name = field.name();
468+
name != pivot_col_name.as_str() && !agg_input_cols.contains(&name.as_str())
469+
})
470+
.map(Expr::from)
423471
.collect();
424472

425473
// Build CASE expressions for each (aggregate, pivot_value) pair
@@ -434,7 +482,7 @@ fn plan_pivot(
434482

435483
for (value_alias, pivot_value) in &pivot_values {
436484
// CASE pivot_col WHEN pivot_value THEN agg_input END
437-
let case_expr = case(col(&pivot_col_name))
485+
let case_expr = case(pivot_col.clone())
438486
.when(pivot_value.clone(), agg_input.clone())
439487
.end()?;
440488

0 commit comments

Comments
 (0)