Skip to content

Commit f27028e

Browse files
author
ebembi-crdb
committed
feat: Support CSV files with inconsistent column counts
Enable DataFusion to read directories containing CSV files with different numbers of columns by implementing schema union during inference. Changes: - Modified CSV schema inference to create union schema from all files - Extended infer_schema_from_stream to handle varying column counts - Added tests for schema building logic and integration scenarios Requires CsvReadOptions::new().truncated_rows(true) to handle files with fewer columns than the inferred schema. Fixes apache#17516
1 parent bb0c3ff commit f27028e

4 files changed

Lines changed: 274 additions & 17 deletions

File tree

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
//! Test for CSV schema inference with different column counts (GitHub issue #17516)
19+
20+
use datafusion::error::Result;
21+
use datafusion::prelude::*;
22+
use std::fs;
23+
use tempfile::TempDir;
24+
25+
#[tokio::test]
26+
async fn test_csv_schema_inference_different_column_counts() -> Result<()> {
27+
// Create temporary directory for test files
28+
let temp_dir = TempDir::new().expect("Failed to create temp dir");
29+
let temp_path = temp_dir.path();
30+
31+
// Create CSV file 1 with 3 columns (simulating older railway services format)
32+
let csv1_content = r#"service_id,route_type,agency_id
33+
1,bus,agency1
34+
2,rail,agency2
35+
3,bus,agency3
36+
"#;
37+
fs::write(temp_path.join("services_2024.csv"), csv1_content)?;
38+
39+
// Create CSV file 2 with 6 columns (simulating newer railway services format)
40+
let csv2_content = r#"service_id,route_type,agency_id,stop_platform_change,stop_planned_platform,stop_actual_platform
41+
4,rail,agency2,true,Platform A,Platform B
42+
5,bus,agency1,false,Stop 1,Stop 1
43+
6,rail,agency3,true,Platform C,Platform D
44+
"#;
45+
fs::write(temp_path.join("services_2025.csv"), csv2_content)?;
46+
47+
// Create DataFusion context
48+
let ctx = SessionContext::new();
49+
50+
// This should now work (previously would have failed with column count mismatch)
51+
// Enable truncated_rows to handle files with different column counts
52+
let df = ctx
53+
.read_csv(
54+
temp_path.to_str().unwrap(),
55+
CsvReadOptions::new().truncated_rows(true)
56+
)
57+
.await
58+
.expect("Should successfully read CSV directory with different column counts");
59+
60+
// Verify the schema contains all 6 columns (union of both files)
61+
let df_clone = df.clone();
62+
let schema = df_clone.schema();
63+
assert_eq!(schema.fields().len(), 6, "Schema should contain all 6 columns");
64+
65+
// Check that we have all expected columns
66+
let field_names: Vec<&str> = schema.fields().iter().map(|f| f.name().as_str()).collect();
67+
assert!(field_names.contains(&"service_id"));
68+
assert!(field_names.contains(&"route_type"));
69+
assert!(field_names.contains(&"agency_id"));
70+
assert!(field_names.contains(&"stop_platform_change"));
71+
assert!(field_names.contains(&"stop_planned_platform"));
72+
assert!(field_names.contains(&"stop_actual_platform"));
73+
74+
// All fields should be nullable since they don't appear in all files
75+
for field in schema.fields() {
76+
assert!(
77+
field.is_nullable(),
78+
"Field {} should be nullable",
79+
field.name()
80+
);
81+
}
82+
83+
// Verify we can actually read the data
84+
let results = df.collect().await?;
85+
86+
// Calculate total rows across all batches
87+
let total_rows: usize = results.iter().map(|batch| batch.num_rows()).sum();
88+
assert_eq!(total_rows, 6, "Should have 6 total rows across all batches");
89+
90+
// All batches should have 6 columns (the union schema)
91+
for batch in &results {
92+
assert_eq!(batch.num_columns(), 6, "All batches should have 6 columns");
93+
}
94+
95+
// Verify that the union schema is being used correctly
96+
// We should be able to find records from both files
97+
println!("✅ Successfully read {} record batches with {} total rows", results.len(), total_rows);
98+
99+
// Verify schema has all expected columns
100+
for batch in &results {
101+
assert_eq!(batch.schema().fields().len(), 6, "Each batch should use the union schema with 6 fields");
102+
}
103+
104+
println!("✅ Successfully verified CSV schema inference fix!");
105+
println!(" - Read {} files with different column counts (3 vs 6)", temp_dir.path().read_dir().unwrap().count());
106+
println!(" - Inferred schema with {} columns", schema.fields().len());
107+
println!(" - Processed {} total rows", total_rows);
108+
109+
Ok(())
110+
}

‎datafusion/datasource-csv/src/file_format.rs‎

Lines changed: 37 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ use arrow::error::ArrowError;
3131
use datafusion_common::config::{ConfigField, ConfigFileType, CsvOptions};
3232
use datafusion_common::file_options::csv_writer::CsvWriterOptions;
3333
use datafusion_common::{
34-
exec_err, not_impl_err, DataFusionError, GetExt, Result, Statistics,
34+
not_impl_err, DataFusionError, GetExt, Result, Statistics,
3535
DEFAULT_CSV_EXTENSION,
3636
};
3737
use datafusion_common_runtime::SpawnedTask;
@@ -497,7 +497,20 @@ impl FileFormat for CsvFormat {
497497
impl CsvFormat {
498498
/// Return the inferred schema reading up to records_to_read from a
499499
/// stream of delimited chunks returning the inferred schema and the
500-
/// number of lines that were read
500+
/// number of lines that were read.
501+
///
502+
/// This method now supports CSV files with different numbers of columns.
503+
/// The inferred schema will be the union of all columns found across all files.
504+
/// Files with fewer columns will have missing columns filled with null values.
505+
///
506+
/// # Example
507+
///
508+
/// If you have two CSV files:
509+
/// - `file1.csv`: `col1,col2,col3`
510+
/// - `file2.csv`: `col1,col2,col3,col4,col5`
511+
///
512+
/// The inferred schema will contain all 5 columns, with files that don't
513+
/// have columns 4 and 5 having null values for those columns.
501514
pub async fn infer_schema_from_stream(
502515
&self,
503516
state: &dyn Session,
@@ -560,21 +573,28 @@ impl CsvFormat {
560573
})
561574
.unzip();
562575
} else {
563-
if fields.len() != column_type_possibilities.len() {
564-
return exec_err!(
565-
"Encountered unequal lengths between records on CSV file whilst inferring schema. \
566-
Expected {} fields, found {} fields at record {}",
567-
column_type_possibilities.len(),
568-
fields.len(),
569-
record_number + 1
570-
);
576+
// Handle files with different numbers of columns by extending the schema
577+
if fields.len() > column_type_possibilities.len() {
578+
// New columns found - extend our tracking structures
579+
for field in fields.iter().skip(column_type_possibilities.len()) {
580+
column_names.push(field.name().clone());
581+
let mut possibilities = HashSet::new();
582+
if records_read > 0 {
583+
possibilities.insert(field.data_type().clone());
584+
}
585+
column_type_possibilities.push(possibilities);
586+
}
587+
}
588+
589+
// Update type possibilities for columns that exist in this file
590+
// We take the minimum of fields.len() and column_type_possibilities.len()
591+
// to avoid index out of bounds when a file has fewer columns
592+
let max_fields_to_process = fields.len().min(column_type_possibilities.len());
593+
for field_idx in 0..max_fields_to_process {
594+
if let Some(field) = fields.get(field_idx) {
595+
column_type_possibilities[field_idx].insert(field.data_type().clone());
596+
}
571597
}
572-
573-
column_type_possibilities.iter_mut().zip(&fields).for_each(
574-
|(possibilities, field)| {
575-
possibilities.insert(field.data_type().clone());
576-
},
577-
);
578598
}
579599

580600
if records_to_read == 0 {
@@ -587,7 +607,7 @@ impl CsvFormat {
587607
}
588608
}
589609

590-
fn build_schema_helper(names: Vec<String>, types: &[HashSet<DataType>]) -> Schema {
610+
pub(crate) fn build_schema_helper(names: Vec<String>, types: &[HashSet<DataType>]) -> Schema {
591611
let fields = names
592612
.into_iter()
593613
.zip(types)

‎datafusion/datasource-csv/src/mod.rs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,8 @@
2121

2222
pub mod file_format;
2323
pub mod source;
24+
#[cfg(test)]
25+
mod tests;
2426

2527
use std::sync::Arc;
2628

Lines changed: 125 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,125 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#[cfg(test)]
19+
mod tests {
20+
use crate::file_format::build_schema_helper;
21+
use arrow::datatypes::DataType;
22+
use std::collections::HashSet;
23+
24+
#[test]
25+
fn test_build_schema_helper_different_column_counts() {
26+
// Test the core schema building logic with different column counts
27+
let mut column_names = vec!["col1".to_string(), "col2".to_string(), "col3".to_string()];
28+
29+
// Simulate adding two more columns from another file
30+
column_names.push("col4".to_string());
31+
column_names.push("col5".to_string());
32+
33+
let column_type_possibilities = vec![
34+
{
35+
let mut set = HashSet::new();
36+
set.insert(DataType::Int64);
37+
set
38+
},
39+
{
40+
let mut set = HashSet::new();
41+
set.insert(DataType::Utf8);
42+
set
43+
},
44+
{
45+
let mut set = HashSet::new();
46+
set.insert(DataType::Float64);
47+
set
48+
},
49+
{
50+
let mut set = HashSet::new();
51+
set.insert(DataType::Utf8); // col4
52+
set
53+
},
54+
{
55+
let mut set = HashSet::new();
56+
set.insert(DataType::Utf8); // col5
57+
set
58+
},
59+
];
60+
61+
let schema = build_schema_helper(column_names, &column_type_possibilities);
62+
63+
// Verify schema has 5 columns
64+
assert_eq!(schema.fields().len(), 5);
65+
assert_eq!(schema.field(0).name(), "col1");
66+
assert_eq!(schema.field(1).name(), "col2");
67+
assert_eq!(schema.field(2).name(), "col3");
68+
assert_eq!(schema.field(3).name(), "col4");
69+
assert_eq!(schema.field(4).name(), "col5");
70+
71+
// All fields should be nullable
72+
for field in schema.fields() {
73+
assert!(field.is_nullable(), "Field {} should be nullable", field.name());
74+
}
75+
}
76+
77+
#[test]
78+
fn test_build_schema_helper_type_merging() {
79+
// Test type merging logic
80+
let column_names = vec!["col1".to_string(), "col2".to_string()];
81+
82+
let column_type_possibilities = vec![
83+
{
84+
let mut set = HashSet::new();
85+
set.insert(DataType::Int64);
86+
set.insert(DataType::Float64); // Should resolve to Float64
87+
set
88+
},
89+
{
90+
let mut set = HashSet::new();
91+
set.insert(DataType::Utf8); // Should remain Utf8
92+
set
93+
},
94+
];
95+
96+
let schema = build_schema_helper(column_names, &column_type_possibilities);
97+
98+
// col1 should be Float64 due to Int64 + Float64 = Float64
99+
assert_eq!(*schema.field(0).data_type(), DataType::Float64);
100+
101+
// col2 should remain Utf8
102+
assert_eq!(*schema.field(1).data_type(), DataType::Utf8);
103+
}
104+
105+
#[test]
106+
fn test_build_schema_helper_conflicting_types() {
107+
// Test when we have incompatible types - should default to Utf8
108+
let column_names = vec!["col1".to_string()];
109+
110+
let column_type_possibilities = vec![
111+
{
112+
let mut set = HashSet::new();
113+
set.insert(DataType::Boolean);
114+
set.insert(DataType::Int64);
115+
set.insert(DataType::Utf8); // Should resolve to Utf8 due to conflicts
116+
set
117+
},
118+
];
119+
120+
let schema = build_schema_helper(column_names, &column_type_possibilities);
121+
122+
// Should default to Utf8 for conflicting types
123+
assert_eq!(*schema.field(0).data_type(), DataType::Utf8);
124+
}
125+
}

0 commit comments

Comments
 (0)