Skip to content

Commit e9d9824

Browse files
kz930claude
andcommitted
fix(operator): split Text Input lines only where the engine does
The engine splits with linesIterator, which ends a line only at LF, CR and CRLF. The export used splitlines, which also ends one at eight other characters such as U+2028 and NEL, so a row holding one came out as two. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 0dcbb8d commit e9d9824

2 files changed

Lines changed: 69 additions & 6 deletions

File tree

‎common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/text/TextInputSourceOpDesc.scala‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -98,12 +98,18 @@ class TextInputSourceOpDesc
9898
case FileAttributeType.TIMESTAMP => "l"
9999
case _ => "l"
100100
}
101+
// The engine splits with linesIterator, which ends a line only at LF, CR
102+
// and CRLF. splitlines also ends one at eight other characters, U+2028
103+
// and NEL among them, so a row holding one came out as two. A newline at
104+
// the very end closes the last line rather than opening an empty one.
105+
buf += """_lines = _text.replace("\r\n", "\n").replace("\r", "\n").split("\n")"""
106+
buf += """if _lines[-1] == "":"""
107+
buf += " _lines.pop()"
101108
// The slice applies to the raw lines, as the engine drops and takes
102109
// before parsing: a line outside the window is never converted, so an
103110
// unparseable one there costs nothing. Taking after dropping also keeps
104111
// a large limit from overflowing the end index.
105-
val dropped =
106-
fileScanOffset.filter(_ > 0).fold("_text.splitlines()")(o => s"_text.splitlines()[$o:]")
112+
val dropped = fileScanOffset.filter(_ > 0).fold("_lines")(o => s"_lines[$o:]")
107113
val linesExpr = fileScanLimit.fold(dropped)(l => s"$dropped[:${l.max(0)}]")
108114
buf += s"""out1df = pd.DataFrame({$colLit: [$castExpr for l in $linesExpr]})"""
109115
buf ++= TextSourceOpDesc.parseTimestampColumn(attributeType, colLit)

‎common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/text/TextInputSourceOpDescSpec.scala‎

Lines changed: 61 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -310,6 +310,60 @@ class TextInputSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter {
310310
}
311311
}
312312

313+
// linesIterator ends a line only at LF, CR and CRLF, while splitlines also
314+
// ended one at eight other characters, so text holding U+2028 was one row in
315+
// the engine and two in the export.
316+
it should "split the text into the lines the engine splits it into" taggedAs NeedsPythonPackages in {
317+
val python = resolvePython().getOrElse(
318+
cancel("No runnable python executable (udf.conf python.path, python3, python, py)")
319+
)
320+
if (!canImportPandas(python)) cancel(s"'$python' cannot import pandas")
321+
322+
val separators =
323+
Seq(
324+
"\n",
325+
"\r",
326+
"\r\n",
327+
"\u000b",
328+
"\u000c",
329+
"\u001c",
330+
"\u001d",
331+
"\u001e",
332+
"\u0085",
333+
"\u2028",
334+
"\u2029"
335+
)
336+
val inputs = separators.map(sep => s"first${sep}second") ++
337+
Seq("", "first\n", "first\n\n", "\nfirst", "first\r\r\nsecond")
338+
339+
textInputSourceOpDesc.attributeType = FileAttributeType.STRING
340+
inputs.foreach { input =>
341+
textInputSourceOpDesc.textInput = input
342+
val (exitCode, out) =
343+
runStandalone(python, """import json; print(json.dumps(list(out1df["line"])))""")
344+
withClue(s"input ${input.map(c => f"\\u${c.toInt}%04x").mkString}, python said:\n$out\n") {
345+
assert(exitCode == 0)
346+
assert(objectMapper.readValue(out.trim, classOf[Array[String]]).toSeq == linesFromEngine())
347+
}
348+
}
349+
}
350+
351+
/** The configured text read as STRING lines by the executor the engine runs. */
352+
private def linesFromEngine(): Seq[String] = {
353+
val exec = new TextInputSourceOpExec(objectMapper.writeValueAsString(textInputSourceOpDesc))
354+
exec.open()
355+
try {
356+
exec
357+
.produceTuple()
358+
.map(
359+
_.asInstanceOf[SchemaEnforceable]
360+
.enforceSchema(textInputSourceOpDesc.sourceSchema())
361+
.getField[String]("line")
362+
)
363+
.toSeq
364+
} finally exec.close()
365+
}
366+
313367
/** The configured text read as booleans by the executor the engine runs. */
314368
private def booleansFromEngine(): Seq[Boolean] = {
315369
val exec = new TextInputSourceOpExec(objectMapper.writeValueAsString(textInputSourceOpDesc))
@@ -329,10 +383,13 @@ class TextInputSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter {
329383
/**
330384
* The configured operator's exported script, run, printing its one column.
331385
*
332-
* `bool` on each value because a numpy scalar does not repr as Python's own
333-
* True and False.
386+
* `bool` on each value by default because a numpy scalar does not repr as
387+
* Python's own True and False.
334388
*/
335-
private def runStandalone(python: String): (Int, String) = {
389+
private def runStandalone(
390+
python: String,
391+
printLine: String = """print([bool(v) for v in out1df["line"]])"""
392+
): (Int, String) = {
336393
val dir = Files.createTempDirectory("text-input-standalone-")
337394
dir.toFile.deleteOnExit()
338395
val script = dir.resolve("run.py")
@@ -341,7 +398,7 @@ class TextInputSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter {
341398
s"""import pandas as pd
342399
|${textInputSourceOpDesc.standaloneHelpers().mkString("\n\n")}
343400
|${textInputSourceOpDesc.generateStandaloneCode()}
344-
|print([bool(v) for v in out1df["line"]])
401+
|$printLine
345402
|""".stripMargin.getBytes(StandardCharsets.UTF_8)
346403
)
347404

0 commit comments

Comments
 (0)