Skip to content

Commit 07b4de6

Browse files
morningmanCopilot
andcommitted
[refactor](fe) Implement Phase 1 FS interface refactoring: new FileSystem API and value objects (#61908)
### What problem does this PR solve? Issue Number: #61860 Problem Summary: The existing FileSystem interface uses Status-based return values, bare String paths, and Hadoop-dependent RemoteFile objects throughout the FE codebase, making it hard to test and impossible to isolate from Hadoop at the module boundary. Phase 1 introduces the new clean IOException-based FileSystem API with typed Location value objects, while preserving full backward compatibility via LegacyFileSystemApi. ### Release note None ### Check List (For Author) - Test: Manual build verification (./build.sh --fe) passes with zero errors - Behavior changed: No (all existing code paths preserved via LegacyFileSystemApi) - Does this need documentation: No New files: - Location.java: immutable URI value object replacing bare String paths - FileEntry.java: immutable file/dir descriptor replacing Hadoop-dependent RemoteFile - FileIterator.java: lazy Closeable iterator interface for directory listing - LegacyFileSystemApi.java: @deprecated copy of old FileSystem interface (Status-based) - LegacyFileSystemAdapter.java: abstract bridge implementing new FileSystem via legacy* methods - LegacyToNewFsAdapter.java: wraps any LegacyFileSystemApi as new FileSystem - MemoryFileSystem.java: in-memory FileSystem for unit testing Modified files: - FileSystem.java: replaced with new clean IOException-based interface - PersistentFileSystem, LocalDfsFileSystem, SwitchingFileSystem: implements LegacyFileSystemApi - FileSystemProvider, FileSystemLookup: return LegacyFileSystemApi - DorisInputFile, DorisOutputFile: add location() method; deprecate path() - HdfsInputFile, HdfsOutputFile, HdfsInputStream: use Location instead of ParsedPath - ParsedPath: @deprecated + toLocation() conversion method - RemoteFile: @deprecated + toFileEntry()/fromFileEntry() conversion methods - RemoteFiles, RemoteFileRemoteIterator: @deprecated - All callers updated to use LegacyFileSystemApi type Co-authored-by: Copilot <[email protected]>
1 parent 1841d5b commit 07b4de6

35 files changed

Lines changed: 1510 additions & 176 deletions

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/AcidUtil.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
import org.apache.doris.datasource.hive.AcidInfo.DeleteDeltaInfo;
2323
import org.apache.doris.datasource.hive.HiveExternalMetaCache.FileCacheValue;
2424
import org.apache.doris.datasource.property.storage.StorageProperties;
25-
import org.apache.doris.fs.FileSystem;
25+
import org.apache.doris.fs.LegacyFileSystemApi;
2626
import org.apache.doris.fs.remote.RemoteFile;
2727

2828
import lombok.EqualsAndHashCode;
@@ -132,7 +132,7 @@ public int compareTo(ParsedDelta other) {
132132
}
133133

134134

135-
private static boolean isValidMetaDataFile(FileSystem fileSystem, String baseDir)
135+
private static boolean isValidMetaDataFile(LegacyFileSystemApi fileSystem, String baseDir)
136136
throws IOException {
137137
String fileLocation = baseDir + "_metadata_acid";
138138
Status status = fileSystem.exists(fileLocation);
@@ -162,7 +162,7 @@ private static boolean isValidMetaDataFile(FileSystem fileSystem, String baseDir
162162
return true;
163163
}
164164

165-
private static boolean isValidBase(FileSystem fileSystem, String baseDir,
165+
private static boolean isValidBase(LegacyFileSystemApi fileSystem, String baseDir,
166166
ParsedBase base, ValidWriteIdList writeIdList) throws IOException {
167167
if (base.writeId == Long.MIN_VALUE) {
168168
//Ref: https://issues.apache.org/jira/browse/HIVE-13369
@@ -239,7 +239,7 @@ public boolean accept(String fileName) {
239239
//Since the hive3 library cannot read the hive4 transaction table normally, and there are many problems
240240
// when using the Hive 4 library directly, this method is implemented.
241241
//Ref: hive/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java#getAcidState
242-
public static FileCacheValue getAcidState(FileSystem fileSystem, HivePartition partition,
242+
public static FileCacheValue getAcidState(LegacyFileSystemApi fileSystem, HivePartition partition,
243243
Map<String, String> txnValidIds, Map<StorageProperties.Type, StorageProperties> storagePropertiesMap,
244244
boolean isFullAcid) throws Exception {
245245

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,9 @@
2828
import org.apache.doris.datasource.NameMapping;
2929
import org.apache.doris.datasource.statistics.CommonStatistics;
3030
import org.apache.doris.foundation.util.PathUtils;
31-
import org.apache.doris.fs.FileSystem;
3231
import org.apache.doris.fs.FileSystemProvider;
3332
import org.apache.doris.fs.FileSystemUtil;
33+
import org.apache.doris.fs.LegacyFileSystemApi;
3434
import org.apache.doris.fs.remote.MultipartUploadCapable;
3535
import org.apache.doris.fs.remote.RemoteFile;
3636
import org.apache.doris.fs.remote.S3FileSystem;
@@ -90,7 +90,7 @@
9090
public class HMSTransaction implements Transaction {
9191
private static final Logger LOG = LogManager.getLogger(HMSTransaction.class);
9292
private final HiveMetadataOps hiveOps;
93-
private final FileSystem fs;
93+
private final LegacyFileSystemApi fs;
9494
private Optional<SummaryProfile> summaryProfile = Optional.empty();
9595
private String queryId;
9696
private boolean isOverwrite = false;

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/fileio/DelegateFileIO.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,8 @@
1919

2020
import org.apache.doris.backup.Status;
2121
import org.apache.doris.datasource.property.storage.StorageProperties;
22-
import org.apache.doris.fs.FileSystem;
2322
import org.apache.doris.fs.FileSystemFactory;
23+
import org.apache.doris.fs.LegacyFileSystemApi;
2424
import org.apache.doris.fs.io.ParsedPath;
2525

2626
import com.google.common.collect.Iterables;
@@ -52,7 +52,7 @@ public class DelegateFileIO implements SupportsBulkOperations {
5252
/**
5353
* The underlying Doris file system used for file operations.
5454
*/
55-
private FileSystem fileSystem;
55+
private LegacyFileSystemApi fileSystem;
5656

5757
/**
5858
* Default constructor.
@@ -66,7 +66,7 @@ public DelegateFileIO() {
6666
*
6767
* @param fileSystem the Doris file system to delegate operations to
6868
*/
69-
public DelegateFileIO(FileSystem fileSystem) {
69+
public DelegateFileIO(LegacyFileSystemApi fileSystem) {
7070
this.fileSystem = Objects.requireNonNull(fileSystem, "fileSystem is null");
7171
}
7272

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/fileio/DelegateOutputFile.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717

1818
package org.apache.doris.datasource.iceberg.fileio;
1919

20-
import org.apache.doris.fs.FileSystem;
20+
import org.apache.doris.fs.LegacyFileSystemApi;
2121
import org.apache.doris.fs.io.DorisOutputFile;
2222
import org.apache.doris.fs.io.ParsedPath;
2323

@@ -40,7 +40,7 @@ public class DelegateOutputFile implements OutputFile {
4040
/**
4141
* The underlying Doris file system used for file operations.
4242
*/
43-
private final FileSystem fileSystem;
43+
private final LegacyFileSystemApi fileSystem;
4444
/**
4545
* The DorisOutputFile instance representing the output file.
4646
*/
@@ -52,7 +52,7 @@ public class DelegateOutputFile implements OutputFile {
5252
* @param fileSystem the Doris file system to delegate operations to
5353
* @param path the ParsedPath representing the file location
5454
*/
55-
public DelegateOutputFile(FileSystem fileSystem, ParsedPath path) {
55+
public DelegateOutputFile(LegacyFileSystemApi fileSystem, ParsedPath path) {
5656
this.fileSystem = Objects.requireNonNull(fileSystem, "fileSystem is null");
5757
this.outputFile = fileSystem.newOutputFile(path);
5858
}

‎fe/fe-core/src/main/java/org/apache/doris/fs/DirectoryLister.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,6 @@
2424
import org.apache.doris.fs.remote.RemoteFile;
2525

2626
public interface DirectoryLister {
27-
RemoteIterator<RemoteFile> listFiles(FileSystem fs, boolean recursive, TableIf table, String location)
27+
RemoteIterator<RemoteFile> listFiles(LegacyFileSystemApi fs, boolean recursive, TableIf table, String location)
2828
throws FileSystemIOException;
2929
}
Lines changed: 213 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,213 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.fs;
19+
20+
import java.util.Collections;
21+
import java.util.List;
22+
import java.util.Objects;
23+
24+
/**
25+
* Immutable record describing a file or directory in a storage system.
26+
* <p>
27+
* Replaces {@link org.apache.doris.fs.remote.RemoteFile} with zero Hadoop dependency.
28+
* Block location information uses {@link BlockInfo} instead of {@code BlockLocation[]}.
29+
*/
30+
public final class FileEntry {
31+
32+
private final Location location;
33+
private final boolean directory;
34+
private final long length;
35+
private final long blockSize;
36+
private final long modificationTime;
37+
private final List<BlockInfo> blocks;
38+
39+
private FileEntry(Builder builder) {
40+
this.location = Objects.requireNonNull(builder.location, "location");
41+
this.directory = builder.directory;
42+
this.length = builder.length;
43+
this.blockSize = builder.blockSize;
44+
this.modificationTime = builder.modificationTime;
45+
this.blocks = builder.blocks == null
46+
? Collections.emptyList()
47+
: Collections.unmodifiableList(builder.blocks);
48+
}
49+
50+
/** Full location (URI) of this file or directory. */
51+
public Location location() {
52+
return location;
53+
}
54+
55+
/** Returns the last component of the path (file or directory name). */
56+
public String name() {
57+
return location.name();
58+
}
59+
60+
public boolean isDirectory() {
61+
return directory;
62+
}
63+
64+
public boolean isFile() {
65+
return !directory;
66+
}
67+
68+
/** File size in bytes. -1 if unknown. */
69+
public long length() {
70+
return length;
71+
}
72+
73+
/** Underlying block size of the storage (e.g., HDFS 128 MB). 0 for object storage. */
74+
public long blockSize() {
75+
return blockSize;
76+
}
77+
78+
/** Last-modified time in milliseconds since epoch. */
79+
public long modificationTime() {
80+
return modificationTime;
81+
}
82+
83+
/** Block location hints for split planning. Empty list for object storage. */
84+
public List<BlockInfo> blocks() {
85+
return blocks;
86+
}
87+
88+
@Override
89+
public String toString() {
90+
return "FileEntry{location=" + location
91+
+ ", directory=" + directory
92+
+ ", length=" + length
93+
+ ", modificationTime=" + modificationTime + "}";
94+
}
95+
96+
@Override
97+
public boolean equals(Object o) {
98+
if (this == o) {
99+
return true;
100+
}
101+
if (!(o instanceof FileEntry)) {
102+
return false;
103+
}
104+
FileEntry that = (FileEntry) o;
105+
return directory == that.directory
106+
&& length == that.length
107+
&& modificationTime == that.modificationTime
108+
&& location.equals(that.location);
109+
}
110+
111+
@Override
112+
public int hashCode() {
113+
return Objects.hash(location, directory, length, modificationTime);
114+
}
115+
116+
public static Builder builder(Location location) {
117+
return new Builder(location);
118+
}
119+
120+
/** Convenience factory for simple file entries (no block info). */
121+
public static FileEntry file(Location location, long length, long modificationTime) {
122+
return builder(location).length(length).modificationTime(modificationTime).build();
123+
}
124+
125+
/** Convenience factory for directory entries. */
126+
public static FileEntry directory(Location location, long modificationTime) {
127+
return builder(location).directory(true).modificationTime(modificationTime).build();
128+
}
129+
130+
public static final class Builder {
131+
private final Location location;
132+
private boolean directory = false;
133+
private long length = -1L;
134+
private long blockSize = 0L;
135+
private long modificationTime = 0L;
136+
private List<BlockInfo> blocks;
137+
138+
private Builder(Location location) {
139+
this.location = Objects.requireNonNull(location);
140+
}
141+
142+
public Builder directory(boolean isDirectory) {
143+
this.directory = isDirectory;
144+
return this;
145+
}
146+
147+
public Builder length(long fileLength) {
148+
this.length = fileLength;
149+
return this;
150+
}
151+
152+
public Builder blockSize(long bSize) {
153+
this.blockSize = bSize;
154+
return this;
155+
}
156+
157+
public Builder modificationTime(long modTime) {
158+
this.modificationTime = modTime;
159+
return this;
160+
}
161+
162+
public Builder blocks(List<BlockInfo> blockList) {
163+
this.blocks = blockList;
164+
return this;
165+
}
166+
167+
public Builder addBlock(BlockInfo block) {
168+
if (this.blocks == null) {
169+
this.blocks = new java.util.ArrayList<>();
170+
}
171+
this.blocks.add(block);
172+
return this;
173+
}
174+
175+
public FileEntry build() {
176+
return new FileEntry(this);
177+
}
178+
}
179+
180+
/**
181+
* Lightweight replacement for {@code org.apache.hadoop.fs.BlockLocation}.
182+
* Contains only the data needed by Doris split planning.
183+
*/
184+
public static final class BlockInfo {
185+
private final long offset;
186+
private final long length;
187+
private final List<String> hosts;
188+
189+
public BlockInfo(long offset, long length, List<String> hosts) {
190+
this.offset = offset;
191+
this.length = length;
192+
this.hosts = hosts == null ? Collections.emptyList() : Collections.unmodifiableList(hosts);
193+
}
194+
195+
public long offset() {
196+
return offset;
197+
}
198+
199+
public long length() {
200+
return length;
201+
}
202+
203+
/** Data-local host names for this block (for scheduling affinity). */
204+
public List<String> hosts() {
205+
return hosts;
206+
}
207+
208+
@Override
209+
public String toString() {
210+
return "BlockInfo{offset=" + offset + ", length=" + length + ", hosts=" + hosts + "}";
211+
}
212+
}
213+
}

0 commit comments

Comments
 (0)