2828import org .apache .hadoop .fs .FSDataInputStream ;
2929import org .apache .hadoop .fs .FileStatus ;
3030import org .apache .hadoop .fs .FileSystem ;
31+ import org .apache .hadoop .fs .FutureDataInputStreamBuilder ;
3132import org .apache .hadoop .fs .Path ;
3233import org .apache .parquet .io .InputFile ;
3334import org .apache .parquet .io .SeekableInputStream ;
@@ -39,6 +40,14 @@ public class HadoopInputFile implements InputFile {
3940 */
4041 private static final String OPENFILE_READ_POLICY_KEY = "fs.option.openfile.read.policy" ;
4142
43+ /**
44+ * openFile() option name for passing a known file length: {@value}.
45+ *
46+ * <p>This is an optional filesystem hint, so retain the string form to support Hadoop versions before the
47+ * constant was introduced.
48+ */
49+ private static final String OPENFILE_LENGTH_KEY = "fs.option.openfile.length" ;
50+
4251 /**
4352 * Read policy when opening parquet files: {@value}.
4453 * <p>Policy-aware stores pick the first policy they recognize in the list.
@@ -53,6 +62,8 @@ public class HadoopInputFile implements InputFile {
5362 private static final String PARQUET_READ_POLICY = "parquet, vector, random, adaptive" ;
5463
5564 private final FileSystem fs ;
65+ private final Path path ;
66+ private final long length ;
5667 private final FileStatus stat ;
5768 private final Configuration conf ;
5869
@@ -61,6 +72,32 @@ public static HadoopInputFile fromPath(Path path, Configuration conf) throws IOE
6172 return new HadoopInputFile (fs , fs .getFileStatus (path ), conf );
6273 }
6374
75+ /**
76+ * Creates an input file using a caller-supplied file length.
77+ *
78+ * <p>The length is trusted and no file status lookup is performed. Callers must provide the exact
79+ * length of the file that will be opened. The file system may defer checking whether the file
80+ * exists until the first read. If the supplied length is incorrect, reads may fail.
81+ *
82+ * <p>When a {@link FileStatus} is available, prefer {@link #fromStatus(FileStatus, Configuration)}.
83+ * A length alone does not preserve file-system-specific identity metadata used when opening a file
84+ * (for example, S3A ETags and version IDs), so initial-open change detection may be weaker.
85+ *
86+ * @param path file path
87+ * @param length exact file length in bytes
88+ * @param conf configuration used to resolve the file system
89+ * @return an input file that uses the supplied length
90+ * @throws IllegalArgumentException if {@code length} is negative
91+ * @throws IOException if the file system cannot be resolved
92+ */
93+ public static HadoopInputFile fromPath (Path path , long length , Configuration conf ) throws IOException {
94+ if (length < 0 ) {
95+ throw new IllegalArgumentException ("Invalid file length: " + length );
96+ }
97+ FileSystem fs = path .getFileSystem (conf );
98+ return new HadoopInputFile (fs , path , length , conf );
99+ }
100+
64101 public static HadoopInputFile fromPathUnchecked (Path path , Configuration conf ) {
65102 try {
66103 return fromPath (path , conf );
@@ -76,28 +113,38 @@ public static HadoopInputFile fromStatus(FileStatus stat, Configuration conf) th
76113
77114 private HadoopInputFile (FileSystem fs , FileStatus stat , Configuration conf ) {
78115 this .fs = fs ;
116+ this .path = stat .getPath ();
117+ this .length = stat .getLen ();
79118 this .stat = stat ;
80119 this .conf = conf ;
81120 }
82121
122+ private HadoopInputFile (FileSystem fs , Path path , long length , Configuration conf ) {
123+ this .fs = fs ;
124+ this .path = path ;
125+ this .length = length ;
126+ this .stat = null ;
127+ this .conf = conf ;
128+ }
129+
83130 public Configuration getConfiguration () {
84131 return conf ;
85132 }
86133
87134 public Path getPath () {
88- return stat . getPath () ;
135+ return path ;
89136 }
90137
91138 @ Override
92139 public long getLength () {
93- return stat . getLen () ;
140+ return length ;
94141 }
95142
96143 /**
97144 * Open the file.
98- * <p>Uses {@code FileSystem.openFile()} so that
99- * the existing FileStatus can be passed down: saves a HEAD request on cloud storage.
100- * and ignored everywhere else .
145+ * <p>Uses {@code FileSystem.openFile()} so that the existing FileStatus, when available, or the
146+ * known file length can be passed down. File systems may use either to avoid a metadata request
147+ * when opening the file .
101148 *
102149 * @return the input stream.
103150 *
@@ -109,20 +156,24 @@ public long getLength() {
109156 public SeekableInputStream newStream () throws IOException {
110157 FSDataInputStream stream ;
111158 try {
112- // this method is async so that implementations may do async HEAD head
159+ // this method is async so that implementations may do async HEAD
113160 // requests, such as S3A/ABFS when a file status is passed down.
114- final CompletableFuture <FSDataInputStream > future = fs .openFile (stat .getPath ())
115- .withFileStatus (stat )
116- .opt (OPENFILE_READ_POLICY_KEY , PARQUET_READ_POLICY )
117- .build ();
161+ FutureDataInputStreamBuilder builder = fs .openFile (path ).opt (OPENFILE_READ_POLICY_KEY , PARQUET_READ_POLICY );
162+ if (stat != null ) {
163+ builder .withFileStatus (stat );
164+ } else {
165+ builder .opt (OPENFILE_LENGTH_KEY , Long .toString (length ));
166+ }
167+ final CompletableFuture <FSDataInputStream > future = builder .build ();
118168 stream = awaitFuture (future );
119169 } catch (RuntimeException e ) {
120170 // S3A < 3.3.5 would raise illegal path exception if the openFile path didn't
121171 // equal the path in the FileStatus; Hive virtual FS could create this condition.
122- // As the path to open is derived from stat.getPath(), this condition seems
123- // near-impossible to create -but is handled here for due diligence.
172+ // When a status is supplied, the path to open is derived from stat.getPath(),
173+ // so this condition seems near-impossible to create -but is handled here
174+ // for due diligence.
124175 try {
125- stream = fs .open (stat . getPath () );
176+ stream = fs .open (path );
126177 } catch (IOException | RuntimeException ex ) {
127178 // failure on this attempt attaches the failure of the openFile() call
128179 // so the stack trace is preserved.
@@ -136,6 +187,6 @@ public SeekableInputStream newStream() throws IOException {
136187
137188 @ Override
138189 public String toString () {
139- return stat . getPath () .toString ();
190+ return path .toString ();
140191 }
141192}
0 commit comments