Skip to content

Commit 8ecdb99

Browse files
eugenegujingclaude
andcommitted
fix(workflow-operator): Text Input offset with an empty limit emits no rows
TextInputSourceOpExec computed its line window as slice(offset, offset + limit.getOrElse(Int.MaxValue)). With an offset set and the limit left empty, the addition overflows Int to a negative bound, which Iterator.slice clamps to 0, so the operator silently emitted zero rows while the workflow reported success. An explicit large limit overflows the same way. This contradicts the Limit property's own description, "Leave empty to read all lines." Replace the slice with drop(offset) + take(limit), the same idiom the CSV, Arrow, and JSONL scan sources already use. Every configuration that previously worked is unchanged. Add regression tests covering offset-without-limit, offset with an Int.MaxValue limit, the offset+limit window, limit-only, offset at and past EOF, a negative offset, and SINGLE_STRING ignoring offset/limit. Closes apache#7346 Co-Authored-By: Claude Fable 5 <[email protected]>
1 parent f46b2a7 commit 8ecdb99

2 files changed

Lines changed: 99 additions & 4 deletions

File tree

‎common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/text/TextInputSourceOpExec.scala‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,12 @@ class TextInputSourceOpExec private[text] (
3434
(if (desc.attributeType.isSingle) {
3535
Iterator(desc.textInput)
3636
} else {
37-
desc.textInput.linesIterator.slice(
38-
desc.fileScanOffset.getOrElse(0),
39-
desc.fileScanOffset.getOrElse(0) + desc.fileScanLimit.getOrElse(Int.MaxValue)
40-
)
37+
// `slice(offset, offset + limit)` overflows Int when the limit is absent
38+
// (it defaults to Int.MaxValue) or large, making `until <= from` and
39+
// silently yielding no rows.
40+
desc.textInput.linesIterator
41+
.drop(desc.fileScanOffset.getOrElse(0))
42+
.take(desc.fileScanLimit.getOrElse(Int.MaxValue))
4143
}).map(line =>
4244
TupleLike(desc.attributeType match {
4345
case FileAttributeType.SINGLE_STRING => line

‎common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/text/TextInputSourceOpDescSpec.scala‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,99 @@ class TextInputSourceOpDescSpec extends AnyFlatSpec with BeforeAndAfter {
171171
textScanSourceOpExec.close()
172172
}
173173

174+
it should "read all lines after the offset when no limit is specified" in {
175+
assert(
176+
linesFrom(offset = Some(5)) == Seq("line6", "line7", "line8", "line9", "line10")
177+
)
178+
}
179+
180+
it should "read all lines after the offset when the limit is Int.MaxValue" in {
181+
assert(
182+
linesFrom(offset = Some(1), limit = Some(Int.MaxValue)) ==
183+
Seq("line2", "line3", "line4", "line5", "line6", "line7", "line8", "line9", "line10")
184+
)
185+
}
186+
187+
it should "read a window of lines when both offset and limit are specified" in {
188+
assert(linesFrom(offset = Some(5), limit = Some(2)) == Seq("line6", "line7"))
189+
}
190+
191+
it should "read the first lines when only a limit is specified" in {
192+
assert(linesFrom(limit = Some(3)) == Seq("line1", "line2", "line3"))
193+
}
194+
195+
it should "produce no tuples when the offset is at or past the end of the input" in {
196+
assert(linesFrom(offset = Some(10)).isEmpty)
197+
assert(linesFrom(offset = Some(99)).isEmpty)
198+
}
199+
200+
it should "treat a negative offset as zero" in {
201+
assert(
202+
linesFrom(offset = Some(-1)) ==
203+
Seq(
204+
"line1",
205+
"line2",
206+
"line3",
207+
"line4",
208+
"line5",
209+
"line6",
210+
"line7",
211+
"line8",
212+
"line9",
213+
"line10"
214+
)
215+
)
216+
}
217+
218+
it should "ignore the offset when reading the input text into a single output tuple" in {
219+
val inputString: String = readFileIntoString(TestOperators.TestTextFilePath)
220+
textInputSourceOpDesc.attributeType = FileAttributeType.SINGLE_STRING
221+
textInputSourceOpDesc.textInput = inputString
222+
textInputSourceOpDesc.fileScanOffset = Option(5)
223+
val textScanSourceOpExec =
224+
new TextInputSourceOpExec(objectMapper.writeValueAsString(textInputSourceOpDesc))
225+
textScanSourceOpExec.open()
226+
val processedTuple: Iterator[Tuple] = textScanSourceOpExec
227+
.produceTuple()
228+
.map(tupleLike =>
229+
tupleLike
230+
.asInstanceOf[SchemaEnforceable]
231+
.enforceSchema(textInputSourceOpDesc.sourceSchema())
232+
)
233+
234+
assert(
235+
processedTuple
236+
.next()
237+
.getField[String]("line")
238+
.equals("line1\nline2\nline3\nline4\nline5\nline6\nline7\nline8\nline9\nline10")
239+
)
240+
assertThrows[java.util.NoSuchElementException](processedTuple.next().getField("line"))
241+
textScanSourceOpExec.close()
242+
}
243+
244+
/**
245+
* Helper function collecting the "line" field of every tuple produced for
246+
* the STRING attribute type with the given offset and limit.
247+
*/
248+
private def linesFrom(offset: Option[Int] = None, limit: Option[Int] = None): Seq[String] = {
249+
textInputSourceOpDesc.attributeType = FileAttributeType.STRING
250+
textInputSourceOpDesc.textInput = readFileIntoString(TestOperators.TestTextFilePath)
251+
textInputSourceOpDesc.fileScanOffset = offset
252+
textInputSourceOpDesc.fileScanLimit = limit
253+
val exec = new TextInputSourceOpExec(objectMapper.writeValueAsString(textInputSourceOpDesc))
254+
exec.open()
255+
try {
256+
exec
257+
.produceTuple()
258+
.map(
259+
_.asInstanceOf[SchemaEnforceable]
260+
.enforceSchema(textInputSourceOpDesc.sourceSchema())
261+
.getField[String]("line")
262+
)
263+
.toSeq
264+
} finally exec.close()
265+
}
266+
174267
/**
175268
* Helper function using UTF-8 encoding to read text file
176269
* into String

0 commit comments

Comments
 (0)