@@ -58,7 +58,6 @@ import java.nio.channels.FileChannel
5858import java .nio .charset .{Charset , StandardCharsets }
5959import java .nio .file .{Files , Path , StandardOpenOption }
6060import java .time .ZoneOffset
61- import java .util .zip .{ZipEntry , ZipOutputStream }
6261import scala .collection .mutable
6362import scala .jdk .CollectionConverters ._
6463import scala .util .{Try , Using }
@@ -840,23 +839,19 @@ object TextInputHandler extends SourceHandler {
840839 desc
841840 }
842841
843- /** Timestamp lines the engine reads its own way: digits past the millisecond,
844- * which it cuts, an offset, which it moves to the local zone, and a year
845- * past pandas' nanosecond range.
846- */
847- val TimestampLines : String =
848- " 2024-01-01 00:00:00.123456789\n 2024-01-01T00:00:00+05:30\n 2500-01-01 00:00:00"
849-
850842 /** A line the window skips is never parsed, so it may be one the type refuses.
851- * A boolean line is read as the engine reads one, where 1 is true.
843+ * A boolean line is read as the engine reads one, where 1 is true. The
844+ * timestamp lines are the ones File Scan's files carry.
852845 */
853846 override def scenarios : Seq [(String , Path => LogicalOp )] =
854847 Seq (
855848 " integers past a line skipped" -> (_ =>
856849 typed(" not a number\n 1\n 2\n 3" , FileAttributeType .INTEGER , Some (1 ))
857850 ),
858851 " booleans" -> (_ => typed(" true\n FALSE\n 1\n 0" , FileAttributeType .BOOLEAN , None )),
859- " timestamps" -> (_ => typed(TimestampLines , FileAttributeType .TIMESTAMP , None ))
852+ " timestamps" -> (_ =>
853+ typed(FileScanTransformHandler .TimestampLines , FileAttributeType .TIMESTAMP , None )
854+ )
860855 )
861856}
862857
@@ -893,72 +888,12 @@ object FileScanSourceHandler extends SourceHandler {
893888 desc
894889 }
895890
896- private def text (dir : Path , content : String , charset : Charset = StandardCharsets .UTF_8 ): Path =
897- Files .write(dir.resolve(" sample.txt" ), content.getBytes(charset))
898-
899- /** Two entries and the one macOS adds beside them, which the engine skips. */
900- private def archive (dir : Path ): Path = {
901- val path = dir.resolve(" sample.zip" )
902- Using (new ZipOutputStream (Files .newOutputStream(path))) { out =>
903- Seq (" a.txt" -> " line1\n line2\n " , " b.txt" -> " line3\n " , " __MACOSX/a.txt" -> " junk\n " ).foreach {
904- case (name, content) =>
905- out.putNextEntry(new ZipEntry (name))
906- out.write(content.getBytes(StandardCharsets .UTF_8 ))
907- out.closeEntry()
908- }
909- }.get
910- path
911- }
912-
913- /** Each scenario's file, written into the directory it is given, with the
914- * attribute type and the settings that read it. Shared with File Scan, which
915- * reads the same files from the paths its input names.
916- *
917- * A line the window skips is never parsed, so it may be one the type refuses.
918- * A boolean line is read as the engine reads one, where 1 is true. The charset
919- * the Encoding field names is the one the bytes are in. With Extract on, the
920- * engine reads the entries inside the archive and names each one by its own
921- * name, line by line or whole.
891+ /** File Scan's files, read from this operator's own path field. The two
892+ * operators share the engine's reader and differ only in where the path
893+ * comes from.
922894 */
923- private [verify] val files : Seq [(String , Path => (Path , FileAttributeType , Seq [String ]))] =
924- Seq (
925- " integers past a line skipped" -> (dir =>
926- (
927- text(dir, " not a number\n 1\n 2\n 3\n " ),
928- FileAttributeType .INTEGER ,
929- Seq (""" "fileScanOffset":1""" )
930- )
931- ),
932- " booleans" -> (dir => (text(dir, " true\n FALSE\n 1\n 0\n " ), FileAttributeType .BOOLEAN , Seq ())),
933- " timestamps" -> (dir =>
934- (text(dir, TextInputHandler .TimestampLines + " \n " ), FileAttributeType .TIMESTAMP , Seq ())
935- ),
936- " UTF-16" -> (dir =>
937- (
938- text(dir, " première\n deuxième\n " , StandardCharsets .UTF_16 ),
939- FileAttributeType .STRING ,
940- Seq (""" "encoding":"UTF_16"""" )
941- )
942- ),
943- " archive" -> (dir => (archive(dir), FileAttributeType .STRING , Seq (""" "extract":true""" ))),
944- " archive, lines named" -> (dir =>
945- (
946- archive(dir),
947- FileAttributeType .STRING ,
948- Seq (""" "extract":true""" , """ "outputFileName":true""" )
949- )
950- ),
951- " archive, entries named" -> (dir =>
952- (
953- archive(dir),
954- FileAttributeType .SINGLE_STRING ,
955- Seq (""" "extract":true""" , """ "outputFileName":true""" )
956- )
957- )
958- )
959-
960895 override def scenarios : Seq [(String , Path => LogicalOp )] =
961- files.map {
896+ FileScanTransformHandler . files.map {
962897 case (label, write) =>
963898 label -> { (dir : Path ) =>
964899 val (file, attributeType, fields) = write(dir)
0 commit comments