Skip to content

Commit 9730c86

Browse files
authored
fix(workflow-operator): auto-rename empty CSV column headers (apache#5282)
### What changes were proposed in this PR? A CSV with an empty column header (e.g. a trailing comma `id,name,,age`) produced an `Attribute` with an empty name. After apache#3295 every output port writes via `IcebergTableWriter → Parquet`, where Avro rejects empty names with `IllegalArgumentException: Empty name` at flush time — losing the operator port's entire result. Rename blank header positions to `column-<index>` (the convention used by pandas, Spark, R, and DuckDB) in all three CSV scan operators: `CSVScanSourceOpDesc`, `ParallelCSVScanSourceOpDesc`, and `CSVOldScanSourceOpDesc`. The issue named the first two; the legacy `csvOld` variant is still registered in `LogicalOp` and had the same latent bug. Note: `ParallelCSVScanSourceOpDesc` is currently commented out of `LogicalOp`'s operator registry (so it is not reachable from the UI), but it is fixed here for consistency and so the bug does not resurface if it is re-enabled for experiments. ### Any related issues, documentation, discussions? Closes apache#5279. ### How was this PR tested? Added three cases to `CSVScanSourceOpDescSpec` — one per operator — feeding a CSV with an empty third header (`id,name,,age`) and asserting the inferred schema is `["id", "name", "column-3", "age"]`. `WorkflowOperator` suite is green (13 passed); scalafmt clean. ### Was this PR authored or co-authored using generative AI tooling? Generated-by: Claude Code (claude-opus-4-7)
1 parent 5cc75bc commit 9730c86

4 files changed

Lines changed: 65 additions & 5 deletions

File tree

‎common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDesc.scala‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -120,7 +120,11 @@ class CSVScanSourceOpDesc extends ScanSourceOpDesc {
120120
else (1 to attributeTypeList.length).map(i => "column-" + i).toArray
121121

122122
header.indices.foldLeft(Schema()) { (schema, i) =>
123-
schema.add(header(i), attributeTypeList(i))
123+
// Auto-rename blank header positions to `column-N` so empty CSV headers
124+
// (e.g. a trailing comma) do not propagate empty attribute names to
125+
// downstream Iceberg/Parquet writers, which reject them.
126+
val name = Option(header(i)).filter(_.nonEmpty).getOrElse(s"column-${i + 1}")
127+
schema.add(name, attributeTypeList(i))
124128
}
125129

126130
}

‎common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csv/ParallelCSVScanSourceOpDesc.scala‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -104,10 +104,13 @@ class ParallelCSVScanSourceOpDesc extends ScanSourceOpDesc {
104104

105105
reader.close()
106106

107-
// build schema based on inferred AttributeTypes
107+
// build schema based on inferred AttributeTypes.
108+
// Auto-rename blank header positions to `column-N` so empty CSV headers
109+
// (e.g. a trailing comma) do not propagate empty attribute names to
110+
// downstream Iceberg/Parquet writers, which reject them.
108111
Schema().add(firstRow.indices.map { i =>
109112
new Attribute(
110-
if (hasHeader) firstRow(i) else s"column-${i + 1}",
113+
if (hasHeader && firstRow(i).nonEmpty) firstRow(i) else s"column-${i + 1}",
111114
attributeTypeList(i)
112115
)
113116
})

‎common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/csvOld/CSVOldScanSourceOpDesc.scala‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -102,10 +102,13 @@ class CSVOldScanSourceOpDesc extends ScanSourceOpDesc {
102102

103103
reader.close()
104104

105-
// build schema based on inferred AttributeTypes
105+
// build schema based on inferred AttributeTypes.
106+
// Auto-rename blank header positions to `column-N` so empty CSV headers
107+
// (e.g. a trailing comma) do not propagate empty attribute names to
108+
// downstream Iceberg/Parquet writers, which reject them.
106109
Schema().add(firstRow.indices.map { i =>
107110
new Attribute(
108-
if (hasHeader) firstRow(i) else s"column-${i + 1}",
111+
if (hasHeader && firstRow(i).nonEmpty) firstRow(i) else s"column-${i + 1}",
109112
attributeTypeList(i)
110113
)
111114
})

‎common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/csv/CSVScanSourceOpDescSpec.scala‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,9 +26,13 @@ import org.apache.texera.amber.core.workflow.WorkflowContext.{
2626
DEFAULT_WORKFLOW_ID
2727
}
2828
import org.apache.texera.amber.operator.TestOperators
29+
import org.apache.texera.amber.operator.source.scan.csvOld.CSVOldScanSourceOpDesc
2930
import org.scalatest.BeforeAndAfter
3031
import org.scalatest.flatspec.AnyFlatSpec
3132

33+
import java.nio.charset.StandardCharsets
34+
import java.nio.file.Files
35+
3236
class CSVScanSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter {
3337

3438
var csvScanSourceOpDesc: CSVScanSourceOpDesc = _
@@ -38,6 +42,18 @@ class CSVScanSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter {
3842
parallelCsvScanSourceOpDesc = new ParallelCSVScanSourceOpDesc()
3943
}
4044

45+
// Writes a CSV whose header row has an empty column (the third position),
46+
// e.g. `id,name,,age`, and returns the absolute path.
47+
private def writeCsvWithEmptyHeader(): String = {
48+
val tmpFile = Files.createTempFile("empty-header-", ".csv")
49+
tmpFile.toFile.deleteOnExit()
50+
Files.write(
51+
tmpFile,
52+
"id,name,,age\n1,Alice,x,30\n2,Bob,y,25\n".getBytes(StandardCharsets.UTF_8)
53+
)
54+
tmpFile.toString
55+
}
56+
4157
it should "infer schema from single-line-data csv" in {
4258

4359
parallelCsvScanSourceOpDesc.fileName = Some(TestOperators.CountrySalesSmallCsvPath)
@@ -160,4 +176,38 @@ class CSVScanSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter {
160176
assert(csvScanSourceOpDesc.customDelimiter.contains(","))
161177
}
162178

179+
it should "auto-rename empty CSV column headers to column-N" in {
180+
val path = writeCsvWithEmptyHeader()
181+
csvScanSourceOpDesc.fileName = Some(path)
182+
csvScanSourceOpDesc.customDelimiter = Some(",")
183+
csvScanSourceOpDesc.hasHeader = true
184+
csvScanSourceOpDesc.setResolvedFileName(FileResolver.resolve(path))
185+
186+
val names = csvScanSourceOpDesc.sourceSchema().getAttributes.map(_.getName).toList
187+
assert(names == List("id", "name", "column-3", "age"))
188+
}
189+
190+
it should "auto-rename empty CSV column headers to column-N for parallel CSV" in {
191+
val path = writeCsvWithEmptyHeader()
192+
parallelCsvScanSourceOpDesc.fileName = Some(path)
193+
parallelCsvScanSourceOpDesc.customDelimiter = Some(",")
194+
parallelCsvScanSourceOpDesc.hasHeader = true
195+
parallelCsvScanSourceOpDesc.setResolvedFileName(FileResolver.resolve(path))
196+
197+
val names = parallelCsvScanSourceOpDesc.sourceSchema().getAttributes.map(_.getName).toList
198+
assert(names == List("id", "name", "column-3", "age"))
199+
}
200+
201+
it should "auto-rename empty CSV column headers to column-N for old CSV" in {
202+
val path = writeCsvWithEmptyHeader()
203+
val oldCsvScanSourceOpDesc = new CSVOldScanSourceOpDesc()
204+
oldCsvScanSourceOpDesc.fileName = Some(path)
205+
oldCsvScanSourceOpDesc.customDelimiter = Some(",")
206+
oldCsvScanSourceOpDesc.hasHeader = true
207+
oldCsvScanSourceOpDesc.setResolvedFileName(FileResolver.resolve(path))
208+
209+
val names = oldCsvScanSourceOpDesc.sourceSchema().getAttributes.map(_.getName).toList
210+
assert(names == List("id", "name", "column-3", "age"))
211+
}
212+
163213
}

0 commit comments

Comments
 (0)