Skip to content

Commit d50d410

Browse files
kz930claude
andcommitted
fix(operator): keep a JSONL file's columns when the offset passes the last line
The schema sampled only the rows inside the window, so an offset past the end left nothing to infer from and the operator declared no columns. An empty window now samples from the top of the file instead. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 0215bb6 commit d50d410

2 files changed

Lines changed: 49 additions & 24 deletions

File tree

‎common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDesc.scala‎

Lines changed: 27 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -258,35 +258,38 @@ class JSONLScanSourceOpDesc extends ScanSourceOpDesc with StandaloneCodeGenerato
258258
"The selected item is a folder or does not exist. Please select an actual .jsonl file from the 'File' dropdown."
259259
)
260260
}
261-
val stream = DocumentFactory.openReadonlyDocument(uri).asInputStream()
262-
val reader = new BufferedReader(new InputStreamReader(stream, fileEncoding.getCharset))
263-
var fieldNames = Set[String]()
264-
265-
val allFields: ArrayBuffer[Map[String, String]] = ArrayBuffer()
266-
267-
val startOffset = windowOffset
268261
// A window of no rows is still a window on this file, and the file's columns
269262
// do not depend on how many of its rows were asked for. Reading the sample
270263
// through the limit left a Limit of 0 nothing to infer from, and the operator
271264
// declared a schema of no columns at all.
272-
val endOffset =
273-
startOffset + windowLimit.filter(_ > 0).getOrElse(INFER_READ_LIMIT).min(INFER_READ_LIMIT)
274-
reader
275-
.lines()
276-
.iterator()
277-
.asScala
278-
.slice(startOffset, endOffset)
279-
.foreach(line => {
280-
val root: JsonNode = objectMapper.readTree(line)
281-
if (root.isObject) {
282-
val fields: Map[String, String] = JSONToMap(root, flatten = flatten)
283-
fieldNames = fieldNames.++(fields.keySet)
284-
allFields += fields
285-
}
286-
})
265+
val sampleSize = windowLimit.filter(_ > 0).getOrElse(INFER_READ_LIMIT).min(INFER_READ_LIMIT)
266+
267+
def readSample(startOffset: Int): ArrayBuffer[Map[String, String]] = {
268+
val stream = DocumentFactory.openReadonlyDocument(uri).asInputStream()
269+
val reader = new BufferedReader(new InputStreamReader(stream, fileEncoding.getCharset))
270+
val sample: ArrayBuffer[Map[String, String]] = ArrayBuffer()
271+
reader
272+
.lines()
273+
.iterator()
274+
.asScala
275+
.slice(startOffset, startOffset + sampleSize)
276+
.foreach(line => {
277+
val root: JsonNode = objectMapper.readTree(line)
278+
if (root.isObject) {
279+
sample += JSONToMap(root, flatten = flatten)
280+
}
281+
})
282+
reader.close()
283+
sample
284+
}
285+
286+
// An offset past the last line empties the window the same way, so the
287+
// sample falls back to the top of the file, which is where CSV always reads it.
288+
val windowSample = readSample(windowOffset)
289+
val allFields =
290+
if (windowSample.isEmpty && windowOffset > 0) readSample(0) else windowSample
287291

288-
val sortedFieldNames = fieldNames.toList.sorted
289-
reader.close()
292+
val sortedFieldNames = allFields.flatMap(_.keySet).toSet.toList.sorted
290293

291294
val attributeTypes = inferSchemaFromRows(allFields.iterator.map(fields => {
292295
val result = ArrayBuffer[Object]()

‎common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/source/scan/json/JSONLScanSourceOpDescSpec.scala‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,28 @@ class JSONLScanSourceOpDescSpec extends AnyFlatSpec with Matchers {
124124
schema.getAttribute("id").getType shouldBe AttributeType.INTEGER
125125
}
126126

127+
// An offset past the last line empties the window the same way, so the sample
128+
// read from it was empty too.
129+
it should "keep the file's columns when the offset passes the last line" in {
130+
val data = Files.createTempFile("jsonl-offset-past-end-", ".jsonl")
131+
data.toFile.deleteOnExit()
132+
Files.write(
133+
data,
134+
"{\"id\":1,\"name\":\"alice\"}\n{\"id\":2,\"name\":\"bob\"}\n".getBytes(
135+
StandardCharsets.UTF_8
136+
)
137+
)
138+
139+
val op = new JSONLScanSourceOpDesc
140+
op.fileName = Some(data.toString)
141+
op.setResolvedFileName(FileResolver.resolve(data.toString))
142+
op.offset = Some(10)
143+
144+
val schema = op.sourceSchema()
145+
schema.getAttributeNames shouldBe List("id", "name")
146+
schema.getAttribute("id").getType shouldBe AttributeType.INTEGER
147+
}
148+
127149
// Jackson parses 1.0000000000000002 exactly; read_json's default parser
128150
// rounds it to 1.0, so a filter on v > 1 would keep a different set of rows.
129151
"JSONLScanSourceOpDesc.generateStandaloneCode" should

0 commit comments

Comments
 (0)