2424 * <p>Built on the driver and shipped to executors, so the configuration travels with it, or is broadcast to them.
2525 */
2626public final class VortexIo implements Serializable {
27- private static final long serialVersionUID = 1L ;
27+ private static final long serialVersionUID = 2L ;
2828
2929 /**
3030 * Option bounding how many concurrent read upcalls the native reader issues against one file. Each in-flight upcall
@@ -34,6 +34,16 @@ public final class VortexIo implements Serializable {
3434
3535 private static final int DEFAULT_READ_CONCURRENCY = 0 ;
3636
37+ /**
38+ * Maximum gap between merged reads, in bytes. Defaults to 1 MiB; zero merges only adjacent or overlapping reads.
39+ */
40+ public static final String COALESCE_DISTANCE_OPTION = "vortex.read.coalesce.distance" ;
41+
42+ /**
43+ * Maximum span of a merged read, in bytes. Must be positive; defaults to 16 MiB. Individual reads may be larger.
44+ */
45+ public static final String COALESCE_MAX_SIZE_OPTION = "vortex.read.coalesce.maxSize" ;
46+
3747 /**
3848 * Option bounding how many splits of a file the native scan reads at once, in all. Each split in flight holds the
3949 * decoded chunks of every column it overlaps, and every task in an executor runs its own scan, which by default
@@ -51,53 +61,48 @@ public final class VortexIo implements Serializable {
5161 /** The configuration broadcast to every executor once, or null when this instance carries {@link #conf}. */
5262 private final Broadcast <SerializableHadoopConf > broadcastConf ;
5363
54- private final int readConcurrency ;
55- private final int splitConcurrency ;
64+ private final ReadSettings settings ;
65+
66+ private record ReadSettings (int readConcurrency , int splitConcurrency , int coalesceDistance , int coalesceMaxSize )
67+ implements Serializable {
68+ static ReadSettings parse (VortexOptions options ) {
69+ Objects .requireNonNull (options , "options" );
70+ return new ReadSettings (
71+ nonNegative (options , READ_CONCURRENCY_OPTION , DEFAULT_READ_CONCURRENCY ),
72+ nonNegative (options , SPLIT_CONCURRENCY_OPTION , DEFAULT_SPLIT_CONCURRENCY ),
73+ nonNegative (options , COALESCE_DISTANCE_OPTION , NativeReadable .DEFAULT_COALESCE_DISTANCE ),
74+ positive (options , COALESCE_MAX_SIZE_OPTION , NativeReadable .DEFAULT_COALESCE_MAX_SIZE ));
75+ }
76+ }
5677
5778 /** Captures {@code hadoopConf} and the read settings found in the format options. */
5879 public static VortexIo create (VortexOptions options , Configuration hadoopConf ) {
59- Objects . requireNonNull (options , "options" );
80+ ReadSettings settings = ReadSettings . parse (options );
6081 Objects .requireNonNull (hadoopConf , "hadoopConf" );
61- return new VortexIo (
62- new SerializableHadoopConf (hadoopConf ),
63- null ,
64- nonNegative (options , READ_CONCURRENCY_OPTION , DEFAULT_READ_CONCURRENCY ),
65- nonNegative (options , SPLIT_CONCURRENCY_OPTION , DEFAULT_SPLIT_CONCURRENCY ));
82+ return new VortexIo (new SerializableHadoopConf (hadoopConf ), null , settings );
6683 }
6784
6885 /**
6986 * Like {@link #create}, but broadcasts {@code hadoopConf} through {@code context}. Each task that deserializes the
7087 * result then shares the executor's copy, instead of reading the whole configuration back from its own bytes.
7188 */
7289 public static VortexIo broadcast (VortexOptions options , Configuration hadoopConf , SparkContext context ) {
73- Objects . requireNonNull (options , "options" );
90+ ReadSettings settings = ReadSettings . parse (options );
7491 Objects .requireNonNull (hadoopConf , "hadoopConf" );
7592 ClassTag <SerializableHadoopConf > tag = ClassTag$ .MODULE$ .apply (SerializableHadoopConf .class );
76- return new VortexIo (
77- null ,
78- context .broadcast (new SerializableHadoopConf (hadoopConf ), tag ),
79- nonNegative (options , READ_CONCURRENCY_OPTION , DEFAULT_READ_CONCURRENCY ),
80- nonNegative (options , SPLIT_CONCURRENCY_OPTION , DEFAULT_SPLIT_CONCURRENCY ));
93+ return new VortexIo (null , context .broadcast (new SerializableHadoopConf (hadoopConf ), tag ), settings );
8194 }
8295
8396 /** Hadoop I/O over a default configuration, for callers that have no Spark session to draw one from. */
8497 public static VortexIo defaults () {
85- return new VortexIo (
86- new SerializableHadoopConf (new Configuration ()),
87- null ,
88- DEFAULT_READ_CONCURRENCY ,
89- DEFAULT_SPLIT_CONCURRENCY );
98+ return create (VortexOptions .empty (), new Configuration ());
9099 }
91100
92101 private VortexIo (
93- SerializableHadoopConf conf ,
94- Broadcast <SerializableHadoopConf > broadcastConf ,
95- int readConcurrency ,
96- int splitConcurrency ) {
102+ SerializableHadoopConf conf , Broadcast <SerializableHadoopConf > broadcastConf , ReadSettings settings ) {
97103 this .conf = conf ;
98104 this .broadcastConf = broadcastConf ;
99- this .readConcurrency = readConcurrency ;
100- this .splitConcurrency = splitConcurrency ;
105+ this .settings = settings ;
101106 }
102107
103108 private static int nonNegative (VortexOptions options , String key , int defaultValue ) {
@@ -108,18 +113,26 @@ private static int nonNegative(VortexOptions options, String key, int defaultVal
108113 return parsed ;
109114 }
110115
116+ private static int positive (VortexOptions options , String key , int defaultValue ) {
117+ int parsed = options .getInt (key , defaultValue );
118+ if (parsed <= 0 ) {
119+ throw new IllegalArgumentException (String .format (Locale .ROOT , "%s must be > 0, got %d" , key , parsed ));
120+ }
121+ return parsed ;
122+ }
123+
111124 public Configuration hadoopConf () {
112125 return conf != null ? conf .value () : broadcastConf .value ().value ();
113126 }
114127
115128 /** Bound on concurrent read upcalls per file; zero keeps the native default. */
116129 public int readConcurrency () {
117- return readConcurrency ;
130+ return settings . readConcurrency () ;
118131 }
119132
120133 /** The most splits of a file the native scan reads at once; zero keeps the native default. */
121134 public int splitConcurrency () {
122- return splitConcurrency ;
135+ return settings . splitConcurrency () ;
123136 }
124137
125138 /**
@@ -133,6 +146,7 @@ public int splitConcurrency() {
133146 */
134147 public NativeReadable openReadable (VortexFile file ) {
135148 VortexFile .requireVortexExtension (file .path ());
136- return HadoopReadable .open (hadoopConf (), file .path (), file .length ());
149+ return HadoopReadable .open (
150+ hadoopConf (), file .path (), file .length (), settings .coalesceDistance (), settings .coalesceMaxSize ());
137151 }
138152}
0 commit comments