Skip to content

Commit 04d95cb

Browse files
kz930claude
andcommitted
test(verify): read File Scan Source's files from File Scan's handler
The files moved to the curated handlers, which come first, so the two changes no longer need each other. The fixture's resource name goes back to private. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 7dbad1b commit 04d95cb

2 files changed

Lines changed: 10 additions & 75 deletions

File tree

‎workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/CanonicalFixture.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -138,7 +138,7 @@ object CanonicalFixture extends SharedFixture {
138138
// etc.), so a hand-edit that breaks one fails the build. `schema` above stays
139139
// authoritative for column types: JSON has no TIMESTAMP, so the timestamps
140140
// are stored as JDBC strings ("2024-01-01 00:00:00.0") and coerced back here.
141-
private[verify] val fixtureResource = "/verify/canonical_fixture.json"
141+
private val fixtureResource = "/verify/canonical_fixture.json"
142142

143143
override val allRows: Vector[Tuple] = {
144144
val stream = Option(getClass.getResourceAsStream(fixtureResource))

‎workflow-compiling-service/src/test/scala/org/apache/texera/amber/translator/verify/SourceCategoryRunner.scala‎

Lines changed: 9 additions & 74 deletions
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,6 @@ import java.nio.channels.FileChannel
5858
import java.nio.charset.{Charset, StandardCharsets}
5959
import java.nio.file.{Files, Path, StandardOpenOption}
6060
import java.time.ZoneOffset
61-
import java.util.zip.{ZipEntry, ZipOutputStream}
6261
import scala.collection.mutable
6362
import scala.jdk.CollectionConverters._
6463
import scala.util.{Try, Using}
@@ -840,23 +839,19 @@ object TextInputHandler extends SourceHandler {
840839
desc
841840
}
842841

843-
/** Timestamp lines the engine reads its own way: digits past the millisecond,
844-
* which it cuts, an offset, which it moves to the local zone, and a year
845-
* past pandas' nanosecond range.
846-
*/
847-
val TimestampLines: String =
848-
"2024-01-01 00:00:00.123456789\n2024-01-01T00:00:00+05:30\n2500-01-01 00:00:00"
849-
850842
/** A line the window skips is never parsed, so it may be one the type refuses.
851-
* A boolean line is read as the engine reads one, where 1 is true.
843+
* A boolean line is read as the engine reads one, where 1 is true. The
844+
* timestamp lines are the ones File Scan's files carry.
852845
*/
853846
override def scenarios: Seq[(String, Path => LogicalOp)] =
854847
Seq(
855848
"integers past a line skipped" -> (_ =>
856849
typed("not a number\n1\n2\n3", FileAttributeType.INTEGER, Some(1))
857850
),
858851
"booleans" -> (_ => typed("true\nFALSE\n1\n0", FileAttributeType.BOOLEAN, None)),
859-
"timestamps" -> (_ => typed(TimestampLines, FileAttributeType.TIMESTAMP, None))
852+
"timestamps" -> (_ =>
853+
typed(FileScanTransformHandler.TimestampLines, FileAttributeType.TIMESTAMP, None)
854+
)
860855
)
861856
}
862857

@@ -893,72 +888,12 @@ object FileScanSourceHandler extends SourceHandler {
893888
desc
894889
}
895890

896-
private def text(dir: Path, content: String, charset: Charset = StandardCharsets.UTF_8): Path =
897-
Files.write(dir.resolve("sample.txt"), content.getBytes(charset))
898-
899-
/** Two entries and the one macOS adds beside them, which the engine skips. */
900-
private def archive(dir: Path): Path = {
901-
val path = dir.resolve("sample.zip")
902-
Using(new ZipOutputStream(Files.newOutputStream(path))) { out =>
903-
Seq("a.txt" -> "line1\nline2\n", "b.txt" -> "line3\n", "__MACOSX/a.txt" -> "junk\n").foreach {
904-
case (name, content) =>
905-
out.putNextEntry(new ZipEntry(name))
906-
out.write(content.getBytes(StandardCharsets.UTF_8))
907-
out.closeEntry()
908-
}
909-
}.get
910-
path
911-
}
912-
913-
/** Each scenario's file, written into the directory it is given, with the
914-
* attribute type and the settings that read it. Shared with File Scan, which
915-
* reads the same files from the paths its input names.
916-
*
917-
* A line the window skips is never parsed, so it may be one the type refuses.
918-
* A boolean line is read as the engine reads one, where 1 is true. The charset
919-
* the Encoding field names is the one the bytes are in. With Extract on, the
920-
* engine reads the entries inside the archive and names each one by its own
921-
* name, line by line or whole.
891+
/** File Scan's files, read from this operator's own path field. The two
892+
* operators share the engine's reader and differ only in where the path
893+
* comes from.
922894
*/
923-
private[verify] val files: Seq[(String, Path => (Path, FileAttributeType, Seq[String]))] =
924-
Seq(
925-
"integers past a line skipped" -> (dir =>
926-
(
927-
text(dir, "not a number\n1\n2\n3\n"),
928-
FileAttributeType.INTEGER,
929-
Seq(""""fileScanOffset":1""")
930-
)
931-
),
932-
"booleans" -> (dir => (text(dir, "true\nFALSE\n1\n0\n"), FileAttributeType.BOOLEAN, Seq())),
933-
"timestamps" -> (dir =>
934-
(text(dir, TextInputHandler.TimestampLines + "\n"), FileAttributeType.TIMESTAMP, Seq())
935-
),
936-
"UTF-16" -> (dir =>
937-
(
938-
text(dir, "première\ndeuxième\n", StandardCharsets.UTF_16),
939-
FileAttributeType.STRING,
940-
Seq(""""encoding":"UTF_16"""")
941-
)
942-
),
943-
"archive" -> (dir => (archive(dir), FileAttributeType.STRING, Seq(""""extract":true"""))),
944-
"archive, lines named" -> (dir =>
945-
(
946-
archive(dir),
947-
FileAttributeType.STRING,
948-
Seq(""""extract":true""", """"outputFileName":true""")
949-
)
950-
),
951-
"archive, entries named" -> (dir =>
952-
(
953-
archive(dir),
954-
FileAttributeType.SINGLE_STRING,
955-
Seq(""""extract":true""", """"outputFileName":true""")
956-
)
957-
)
958-
)
959-
960895
override def scenarios: Seq[(String, Path => LogicalOp)] =
961-
files.map {
896+
FileScanTransformHandler.files.map {
962897
case (label, write) =>
963898
label -> { (dir: Path) =>
964899
val (file, attributeType, fields) = write(dir)

0 commit comments

Comments
 (0)