Skip to content

Commit 34e586c

Browse files
authored
apacheGH-3856: Allow callers to provide known Hadoop file lengths (apache#3678)
1 parent c972bd0 commit 34e586c

2 files changed

Lines changed: 124 additions & 14 deletions

File tree

‎parquet-hadoop/src/main/java/org/apache/parquet/hadoop/util/HadoopInputFile.java‎

Lines changed: 65 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import org.apache.hadoop.fs.FSDataInputStream;
2929
import org.apache.hadoop.fs.FileStatus;
3030
import org.apache.hadoop.fs.FileSystem;
31+
import org.apache.hadoop.fs.FutureDataInputStreamBuilder;
3132
import org.apache.hadoop.fs.Path;
3233
import org.apache.parquet.io.InputFile;
3334
import org.apache.parquet.io.SeekableInputStream;
@@ -39,6 +40,14 @@ public class HadoopInputFile implements InputFile {
3940
*/
4041
private static final String OPENFILE_READ_POLICY_KEY = "fs.option.openfile.read.policy";
4142

43+
/**
44+
* openFile() option name for passing a known file length: {@value}.
45+
*
46+
* <p>This is an optional filesystem hint, so retain the string form to support Hadoop versions before the
47+
* constant was introduced.
48+
*/
49+
private static final String OPENFILE_LENGTH_KEY = "fs.option.openfile.length";
50+
4251
/**
4352
* Read policy when opening parquet files: {@value}.
4453
* <p>Policy-aware stores pick the first policy they recognize in the list.
@@ -53,6 +62,8 @@ public class HadoopInputFile implements InputFile {
5362
private static final String PARQUET_READ_POLICY = "parquet, vector, random, adaptive";
5463

5564
private final FileSystem fs;
65+
private final Path path;
66+
private final long length;
5667
private final FileStatus stat;
5768
private final Configuration conf;
5869

@@ -61,6 +72,32 @@ public static HadoopInputFile fromPath(Path path, Configuration conf) throws IOE
6172
return new HadoopInputFile(fs, fs.getFileStatus(path), conf);
6273
}
6374

75+
/**
76+
* Creates an input file using a caller-supplied file length.
77+
*
78+
* <p>The length is trusted and no file status lookup is performed. Callers must provide the exact
79+
* length of the file that will be opened. The file system may defer checking whether the file
80+
* exists until the first read. If the supplied length is incorrect, reads may fail.
81+
*
82+
* <p>When a {@link FileStatus} is available, prefer {@link #fromStatus(FileStatus, Configuration)}.
83+
* A length alone does not preserve file-system-specific identity metadata used when opening a file
84+
* (for example, S3A ETags and version IDs), so initial-open change detection may be weaker.
85+
*
86+
* @param path file path
87+
* @param length exact file length in bytes
88+
* @param conf configuration used to resolve the file system
89+
* @return an input file that uses the supplied length
90+
* @throws IllegalArgumentException if {@code length} is negative
91+
* @throws IOException if the file system cannot be resolved
92+
*/
93+
public static HadoopInputFile fromPath(Path path, long length, Configuration conf) throws IOException {
94+
if (length < 0) {
95+
throw new IllegalArgumentException("Invalid file length: " + length);
96+
}
97+
FileSystem fs = path.getFileSystem(conf);
98+
return new HadoopInputFile(fs, path, length, conf);
99+
}
100+
64101
public static HadoopInputFile fromPathUnchecked(Path path, Configuration conf) {
65102
try {
66103
return fromPath(path, conf);
@@ -76,28 +113,38 @@ public static HadoopInputFile fromStatus(FileStatus stat, Configuration conf) th
76113

77114
private HadoopInputFile(FileSystem fs, FileStatus stat, Configuration conf) {
78115
this.fs = fs;
116+
this.path = stat.getPath();
117+
this.length = stat.getLen();
79118
this.stat = stat;
80119
this.conf = conf;
81120
}
82121

122+
private HadoopInputFile(FileSystem fs, Path path, long length, Configuration conf) {
123+
this.fs = fs;
124+
this.path = path;
125+
this.length = length;
126+
this.stat = null;
127+
this.conf = conf;
128+
}
129+
83130
public Configuration getConfiguration() {
84131
return conf;
85132
}
86133

87134
public Path getPath() {
88-
return stat.getPath();
135+
return path;
89136
}
90137

91138
@Override
92139
public long getLength() {
93-
return stat.getLen();
140+
return length;
94141
}
95142

96143
/**
97144
* Open the file.
98-
* <p>Uses {@code FileSystem.openFile()} so that
99-
* the existing FileStatus can be passed down: saves a HEAD request on cloud storage.
100-
* and ignored everywhere else.
145+
* <p>Uses {@code FileSystem.openFile()} so that the existing FileStatus, when available, or the
146+
* known file length can be passed down. File systems may use either to avoid a metadata request
147+
* when opening the file.
101148
*
102149
* @return the input stream.
103150
*
@@ -109,20 +156,24 @@ public long getLength() {
109156
public SeekableInputStream newStream() throws IOException {
110157
FSDataInputStream stream;
111158
try {
112-
// this method is async so that implementations may do async HEAD head
159+
// this method is async so that implementations may do async HEAD
113160
// requests, such as S3A/ABFS when a file status is passed down.
114-
final CompletableFuture<FSDataInputStream> future = fs.openFile(stat.getPath())
115-
.withFileStatus(stat)
116-
.opt(OPENFILE_READ_POLICY_KEY, PARQUET_READ_POLICY)
117-
.build();
161+
FutureDataInputStreamBuilder builder = fs.openFile(path).opt(OPENFILE_READ_POLICY_KEY, PARQUET_READ_POLICY);
162+
if (stat != null) {
163+
builder.withFileStatus(stat);
164+
} else {
165+
builder.opt(OPENFILE_LENGTH_KEY, Long.toString(length));
166+
}
167+
final CompletableFuture<FSDataInputStream> future = builder.build();
118168
stream = awaitFuture(future);
119169
} catch (RuntimeException e) {
120170
// S3A < 3.3.5 would raise illegal path exception if the openFile path didn't
121171
// equal the path in the FileStatus; Hive virtual FS could create this condition.
122-
// As the path to open is derived from stat.getPath(), this condition seems
123-
// near-impossible to create -but is handled here for due diligence.
172+
// When a status is supplied, the path to open is derived from stat.getPath(),
173+
// so this condition seems near-impossible to create -but is handled here
174+
// for due diligence.
124175
try {
125-
stream = fs.open(stat.getPath());
176+
stream = fs.open(path);
126177
} catch (IOException | RuntimeException ex) {
127178
// failure on this attempt attaches the failure of the openFile() call
128179
// so the stack trace is preserved.
@@ -136,6 +187,6 @@ public SeekableInputStream newStream() throws IOException {
136187

137188
@Override
138189
public String toString() {
139-
return stat.getPath().toString();
190+
return path.toString();
140191
}
141192
}

‎parquet-hadoop/src/test/java/org/apache/parquet/hadoop/util/TestHadoopOpenFile.java‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
import org.junit.jupiter.api.AfterEach;
4545
import org.junit.jupiter.api.BeforeEach;
4646
import org.junit.jupiter.api.Test;
47+
import org.junit.jupiter.api.io.TempDir;
4748
import org.mockito.Mockito;
4849

4950
/**
@@ -58,7 +59,13 @@
5859
*/
5960
public class TestHadoopOpenFile {
6061

62+
private static final String OPENFILE_LENGTH_KEY = "fs.option.openfile.length";
63+
6164
private static final int FIRST = MockHadoopInputStream.TEST_ARRAY[0];
65+
66+
@TempDir
67+
private java.nio.file.Path tempDir;
68+
6269
private URI fsUri;
6370
private FileStatus status;
6471
private Path path;
@@ -101,10 +108,58 @@ public void testOpenFileGoodPath() throws Throwable {
101108
// this looks up the FS binding via the status file path.
102109
openAndRead(fileFromStatus());
103110

111+
assertThat(opener.getFileStatus()).as("file status").isSameAs(status);
112+
104113
// The fallback call of open(path) never took place.
105114
Mockito.verify(mockFS, never()).open(path);
106115
}
107116

117+
/**
118+
* A caller-supplied length avoids a status lookup and is passed to openFile().
119+
*/
120+
@Test
121+
public void testOpenFileWithKnownLength() throws Throwable {
122+
final FileSystem mockFS = prepareMockFS();
123+
final FSDataInputStream in = new FSDataInputStream(new MockHadoopInputStream());
124+
final StubOpenFileBuilder opener = new StubOpenFileBuilder(mockFS, path, CompletableFuture.completedFuture(in));
125+
doReturn(opener).when(mockFS).openFile(path);
126+
127+
final HadoopInputFile inputFile = HadoopInputFile.fromPath(path, status.getLen(), conf);
128+
assertThat(inputFile.getPath()).as("path").isEqualTo(path);
129+
assertThat(inputFile.getLength()).as("length").isEqualTo(status.getLen());
130+
openAndRead(inputFile);
131+
132+
Mockito.verify(mockFS, never()).getFileStatus(path);
133+
assertThat(opener.getFileStatus()).as("file status").isNull();
134+
assertThat(opener.getOptions().get(OPENFILE_LENGTH_KEY))
135+
.as("openFile length")
136+
.isEqualTo(Long.toString(status.getLen()));
137+
}
138+
139+
@Test
140+
public void testRejectsNegativeKnownLength() {
141+
assertThatThrownBy(() -> HadoopInputFile.fromPath(path, -1, conf))
142+
.isInstanceOf(IllegalArgumentException.class)
143+
.hasMessage("Invalid file length: -1");
144+
}
145+
146+
/**
147+
* Opening a nonexistent file with a caller-supplied length must fail either
148+
* when opening the stream or on the first read.
149+
*/
150+
@Test
151+
public void testKnownLengthMissingFile() throws IOException {
152+
final Path missingPath = new Path(new Path(tempDir.toUri()), "missing.parquet");
153+
final HadoopInputFile inputFile = HadoopInputFile.fromPath(missingPath, 1, conf);
154+
155+
assertThatThrownBy(() -> {
156+
try (SeekableInputStream stream = inputFile.newStream()) {
157+
stream.read();
158+
}
159+
})
160+
.isInstanceOf(FileNotFoundException.class);
161+
}
162+
108163
/**
109164
* The openFile() call raises a RuntimeException which it is caught and the
110165
* classic open() call invoked.
@@ -275,6 +330,10 @@ private StubOpenFileBuilder(
275330
this.result = result;
276331
}
277332

333+
private FileStatus getFileStatus() {
334+
return getStatus();
335+
}
336+
278337
@Override
279338
public CompletableFuture<FSDataInputStream> build()
280339
throws IllegalArgumentException, UnsupportedOperationException {

0 commit comments

Comments
 (0)