Skip to content

Commit cbdd34f

Browse files
committed
fix: address duplicate field name review feedback
1 parent 9d30ff9 commit cbdd34f

5 files changed

Lines changed: 34 additions & 11 deletions

File tree

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
id,value,value
2+
1,10,100

‎datafusion/datasource-csv/src/file_format.rs‎

Lines changed: 13 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -397,19 +397,25 @@ impl FileFormat for CsvFormat {
397397
Box::new(err),
398398
)
399399
})?;
400-
ensure_unique_field_names(&schema).map_err(|err| {
401-
DataFusionError::Context(
402-
format!("Error when processing CSV file {}", object.location),
403-
Box::new(err),
404-
)
405-
})?;
406400
records_to_read -= records_read;
407-
schemas.push(schema);
401+
schemas.push((&object.location, schema));
408402
if records_to_read == 0 {
409403
break;
410404
}
411405
}
412406

407+
let mut seen = HashSet::new();
408+
for (location, schema) in &schemas {
409+
ensure_unique_field_names(schema, &mut seen).map_err(|err| {
410+
DataFusionError::Context(
411+
format!("Error when processing CSV file {location}"),
412+
Box::new(err),
413+
)
414+
})?;
415+
}
416+
drop(seen);
417+
418+
let schemas = schemas.into_iter().map(|(_, schema)| schema);
413419
let merged_schema = Schema::try_merge(schemas)?;
414420
Ok(Arc::new(merged_schema))
415421
}

‎datafusion/datasource-parquet/src/file_format.rs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
//! [`ParquetFormat`]: Parquet [`FileFormat`] abstractions
1919
20+
use std::collections::HashSet;
2021
use std::fmt;
2122
use std::fmt::Debug;
2223
use std::ops::Range;
@@ -390,14 +391,16 @@ impl FileFormat for ParquetFormat {
390391
schemas
391392
.sort_unstable_by(|(location1, _), (location2, _)| location1.cmp(location2));
392393

394+
let mut seen = HashSet::new();
393395
for (location, schema) in &schemas {
394-
ensure_unique_field_names(schema).map_err(|err| {
396+
ensure_unique_field_names(schema, &mut seen).map_err(|err| {
395397
DataFusionError::Context(
396398
format!("Error when processing Parquet file {location}"),
397399
Box::new(err),
398400
)
399401
})?;
400402
}
403+
drop(seen);
401404

402405
let schemas = schemas.into_iter().map(|(_, schema)| schema);
403406

‎datafusion/datasource/src/file_format.rs‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -47,9 +47,14 @@ pub const DEFAULT_SCHEMA_INFER_MAX_RECORD: usize = 1000;
4747
/// Rejects an inferred schema that names the same field more than once.
4848
///
4949
/// [`Schema::try_merge`] coalesces fields by name, so callers validate each
50-
/// inferred file schema before merging.
51-
pub fn ensure_unique_field_names(schema: &Schema) -> Result<()> {
52-
let mut seen = HashSet::with_capacity(schema.fields().len());
50+
/// inferred file schema before merging. `seen` is cleared before use so callers
51+
/// can reuse its allocation across schemas.
52+
pub fn ensure_unique_field_names<'a>(
53+
schema: &'a Schema,
54+
seen: &mut HashSet<&'a str>,
55+
) -> Result<()> {
56+
seen.clear();
57+
seen.reserve(schema.fields().len());
5358
for field in schema.fields() {
5459
if !seen.insert(field.name()) {
5560
return schema_err!(SchemaError::DuplicateUnqualifiedField {

‎datafusion/sqllogictest/test_files/csv_files.slt‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,13 @@
1515
# specific language governing permissions and limitations
1616
# under the License.
1717

18+
# duplicate_header_is_rejected
19+
statement error Schema contains duplicate unqualified field name value
20+
CREATE EXTERNAL TABLE csv_with_duplicate_header
21+
STORED AS CSV
22+
LOCATION '../core/tests/data/duplicate_header.csv'
23+
OPTIONS ('format.has_header' 'true');
24+
1825
# create_external_table_with_quote_escape
1926
statement ok
2027
CREATE EXTERNAL TABLE csv_with_quote (

0 commit comments

Comments
 (0)