Skip to content

Commit 95ab40c

Browse files
kz930claude
andcommitted
fix(operator): name the file each line came from when a file scan was asked to
A file scan with Extract and Include Filename on could not run at all in any attribute type that reads line by line, which is the default. The two halves disagreed about how wide a row is: the schema prepends a `filename` column whenever the flag is set, while the line-by-line branch emitted the value alone, so the first tuple was one field against two columns and could not be built. Include Filename is only offered once Extract is on, so every configuration reaching it is one the panel invites. The branch now pairs each entry with its name, as the single-value branch already did. Both exports follow: their `&& isSingle` was there to mirror the defect, and the spec pinning that a line-mode export carries no filename column now pins the column it does carry. Closes apache#8598 Co-Authored-By: Claude Opus 5 (1M context) <[email protected]>
1 parent d8a9d13 commit 95ab40c

5 files changed

Lines changed: 80 additions & 28 deletions

File tree

‎common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDesc.scala‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -141,10 +141,10 @@ class FileScanOpDesc
141141
(" " * 8, "_fn", "_f.read()", "_f")
142142
}
143143

144-
// Match the platform (FileScanUtils.createTuplesFromFile): its line-by-line
145-
// branch ignores outputFileName and emits only the value, so the filename
146-
// column is added ONLY in single-value mode.
147-
val emitFilename = outputFileName && attributeType.isSingle
144+
// Whatever the flag says, as the platform now reads it: every row carries the
145+
// name of the file its value came from, a line's as much as a whole file's.
146+
// See FileScanUtils.createTuplesFromFile.
147+
val emitFilename = outputFileName
148148

149149
if (attributeType.isSingle) {
150150
if (emitFilename) buf += s"${indent}_rows.append(($nameExpr, $readWhole))"
@@ -172,7 +172,8 @@ class FileScanOpDesc
172172
.fold(s"$lines.readlines()")(o => s"$lines.readlines()[$o:]")
173173
fileScanLimit.fold(dropped)(l => s"$dropped[:${l.max(0)}]")
174174
}
175-
buf += s"${indent}_rows.extend($castExpr for l in $linesExpr)"
175+
val row = if (emitFilename) s"($nameExpr, $castExpr)" else castExpr
176+
buf += s"${indent}_rows.extend($row for l in $linesExpr)"
176177
}
177178

178179
val colLit = pyStringLiteral(col)

‎common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDesc.scala‎

Lines changed: 11 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -115,10 +115,10 @@ class FileScanSourceOpDesc
115115
fileScanLimit.fold(dropped)(l => s"$dropped[:${l.max(0)}]")
116116
}
117117

118-
// Match the platform (FileScanUtils.createTuplesFromFile): its line-by-line
119-
// branch emits only the value, so the filename column is added ONLY in
120-
// single-value mode, whatever the flag says.
121-
val emitFilename = outputFileName && attributeType.isSingle
118+
// Whatever the flag says, as the platform now reads it: every row carries
119+
// the name of the file its value came from, a line's as much as a whole
120+
// file's. See FileScanUtils.createTuplesFromFile.
121+
val emitFilename = outputFileName
122122

123123
if (extract) {
124124
// The engine reads the files INSIDE the archive, one tuple per entry, and
@@ -143,7 +143,8 @@ class FileScanSourceOpDesc
143143
// TextIOWrapper, not splitlines: it ends a line where `open(..., "r")`
144144
// does, which is what the branch below reads with.
145145
val linesExpr = windowed(s"io.TextIOWrapper(_f, encoding=$encLit)")
146-
buf += s" _rows.extend($castExpr for l in $linesExpr)"
146+
val row = if (emitFilename) s"(_name, $castExpr)" else castExpr
147+
buf += s" _rows.extend($row for l in $linesExpr)"
147148
}
148149
if (emitFilename) buf += s"""out1df = pd.DataFrame(_rows, columns=["filename", $colLit])"""
149150
else buf += s"""out1df = pd.DataFrame({$colLit: _rows})"""
@@ -158,8 +159,12 @@ class FileScanSourceOpDesc
158159
buf += s""" out1df = pd.DataFrame($dfCols)"""
159160
} else {
160161
val linesExpr = windowed("_f")
162+
val dfCols =
163+
if (emitFilename)
164+
s"""{"filename": $SourceFilePlaceholder, $colLit: [$castExpr for l in $linesExpr]}"""
165+
else s"""{$colLit: [$castExpr for l in $linesExpr]}"""
161166
buf += s"""with open($SourceFilePlaceholder, "r", encoding=$encLit) as _f:"""
162-
buf += s""" out1df = pd.DataFrame({$colLit: [$castExpr for l in $linesExpr]})"""
167+
buf += s""" out1df = pd.DataFrame($dfCols)"""
163168
}
164169

165170
buf.mkString("\n")

‎common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/file/FileScanUtils.scala‎

Lines changed: 21 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -138,20 +138,27 @@ private[file] object FileScanUtils {
138138
TupleLike(fields.toSeq: _*)
139139
}
140140
} else {
141-
fileEntries.flatMap { entry =>
142-
val lines = new BufferedReader(new InputStreamReader(entry, fileEncoding.getCharset))
143-
.lines()
144-
.iterator()
145-
.asScala
146-
.drop(fileScanOffset.getOrElse(0))
147-
fileScanLimit
148-
.fold(lines)(lines.take)
149-
.map(line =>
150-
TupleLike(attributeType match {
151-
case FileAttributeType.SINGLE_STRING => line
152-
case _ => parseField(line, attributeType.getType)
153-
})
154-
)
141+
// Paired with the entry names the same way the single-value branch is:
142+
// a row carries the name of the file its line came from, which is what
143+
// the schema declares when the filename was asked for. Emitting the value
144+
// alone left a one-field row against a two-column schema, and the tuple
145+
// could not be built at all.
146+
fileEntries.zipAll(filenameIt, null, null).flatMap {
147+
case (entry, entryFileName) =>
148+
val lines = new BufferedReader(new InputStreamReader(entry, fileEncoding.getCharset))
149+
.lines()
150+
.iterator()
151+
.asScala
152+
.drop(fileScanOffset.getOrElse(0))
153+
fileScanLimit
154+
.fold(lines)(lines.take)
155+
.map { line =>
156+
val value = attributeType match {
157+
case FileAttributeType.SINGLE_STRING => line
158+
case _ => parseField(line, attributeType.getType)
159+
}
160+
if (outputFileName) TupleLike(entryFileName, value) else TupleLike(value)
161+
}
155162
}
156163
}
157164

‎common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/file/FileScanOpDescSpec.scala‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -191,11 +191,20 @@ class FileScanOpDescSpec extends AnyFlatSpec with BeforeAndAfter {
191191
assert(withoutName.contains(" _rows.append(_f.read())"))
192192
assert(withoutName.endsWith("""out1df = pd.DataFrame({"line": _rows})"""))
193193

194-
// The platform's line-by-line branch (FileScanUtils.createTuplesFromFile)
195-
// emits only the value, so line mode must drop the filename column too.
194+
// A line carries the name of the file it came from, as the whole file does,
195+
// which is what the platform's line-by-line branch now emits and what the
196+
// schema declares either way. It used to emit the value alone, leaving a
197+
// one-field row against a two-column schema.
196198
fileScanOpDesc.attributeType = FileAttributeType.STRING
197199
fileScanOpDesc.outputFileName = true
198-
assert(!fileScanOpDesc.generateStandaloneCode().contains("filename"))
200+
val lines = fileScanOpDesc.generateStandaloneCode()
201+
assert(lines.contains(""" _rows.extend((_fn, l.rstrip("\n")) for l in _f)"""))
202+
assert(lines.endsWith("""out1df = pd.DataFrame(_rows, columns=["filename", "line"])"""))
203+
204+
fileScanOpDesc.outputFileName = false
205+
val bareLines = fileScanOpDesc.generateStandaloneCode()
206+
assert(bareLines.contains(""" _rows.extend(l.rstrip("\n") for l in _f)"""))
207+
assert(bareLines.endsWith("""out1df = pd.DataFrame({"line": _rows})"""))
199208
}
200209

201210
it should "open binary attribute types in binary mode" in {

‎common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/file/FileScanSourceOpDescSpec.scala‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -369,6 +369,36 @@ class FileScanSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter {
369369
assert(out.trim.endsWith("""[('a.txt', 'first'), ('b.txt', 'second')]"""))
370370
}
371371

372+
// Include Filename is only offered once Extract is on, and reading the entries
373+
// line by line is the default there, so this is the configuration the panel
374+
// invites first. The line-by-line branch carried no name, leaving a one-field
375+
// row against the two-column schema, and the first tuple could not be built.
376+
it should "name the entry each line came from, reading an archive line by line" in {
377+
val python = resolvePython().getOrElse(
378+
cancel("No runnable python executable (udf.conf python.path, python3, python, py)")
379+
)
380+
if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas")
381+
382+
val dir = Files.createTempDirectory("file-scan-archive-lines-")
383+
dir.toFile.deleteOnExit()
384+
val archive = writeArchive(dir, "a.txt" -> "one\ntwo\n", "b.txt" -> "three\n")
385+
386+
val desc = extractingDesc(archive, """"outputFileName":true""")
387+
val engine =
388+
tuplesFromEngine(desc).map(t => (t.getField[String]("filename"), t.getField[String]("line")))
389+
assert(engine == Seq(("a.txt", "one"), ("a.txt", "two"), ("b.txt", "three")))
390+
391+
val out = runStandalone(
392+
python,
393+
dir,
394+
desc,
395+
"""print(list(out1df.itertuples(index=False, name=None)))"""
396+
)._2
397+
assert(
398+
out.trim.endsWith("""[('a.txt', 'one'), ('a.txt', 'two'), ('b.txt', 'three')]""")
399+
)
400+
}
401+
372402
/** A zip at `dir/archive.zip` holding the given entries. */
373403
private def writeArchive(dir: Path, entries: (String, String)*): Path = {
374404
val archive = dir.resolve("archive.zip")

0 commit comments

Comments
 (0)