Skip to content

Commit 2cd3e9c

Browse files
morningmanCopilot
andauthored
[refactor](fs-spi) Migrate cloud.storage Remote* hierarchy to fs.obj module (#61911)
### What problem does this PR solve? Issue Number: #61860 Problem Summary: Eliminates duplicate object storage abstraction in the Cloud mode codebase. Previously, cloud.storage.RemoteBase and its subclasses (S3Remote, OssRemote, CosRemote, ObsRemote, BosRemote, AzureRemote, etc.) provided object storage operations independently from the fs.obj module. This change migrates all functionality into fs.obj and removes the entire Remote* hierarchy, so all Cloud Stage/Copy operations now go through ObjFileSystem. Key changes: - Add OssObjStorage, CosObjStorage, ObsObjStorage, BosObjStorage as new ObjStorage implementations with STS token, presigned URL, list, and head support - Extend AzureObjStorage and S3ObjStorage with getStsToken, getPresignedUrl, listObjectsWithPrefix, headObjectWithMeta - Add ObjFileSystem passthrough methods: getStsToken, getPresignedUrl, listObjectsWithPrefix, headObjectWithMeta, deleteObjectsByKeys - Extract RemoteBase.ObjectInfo inner class to standalone cloud.storage.ObjectInfo - Rewrite ObjectInfoAdapter: no RemoteBase dependency; uses ObjFileSystem for STS in ARN flow - Migrate all callers (StageUtil, CopyLoadPendingTask, CleanCopyJobTask, CopyIntoAction, CreateStageCommand, CopyIntoInfo) to ObjFileSystem - Delete RemoteBase, S3Remote, OssRemote, CosRemote, ObsRemote, BosRemote, TosRemote, AzureRemote, DefaultRemote, MockRemote - Rewrite CopyLoadPendingTaskTest and StageUtilTest to use MockUp<ObjFileSystem> - Fix Maven build cache causing test execution to be skipped (run-fe-ut.sh) Co-authored-by: Copilot <[email protected]>
1 parent c83cf7c commit 2cd3e9c

42 files changed

Lines changed: 1560 additions & 1845 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,13 +51,13 @@
5151
import org.apache.doris.cloud.proto.Cloud.MetaServiceCode;
5252
import org.apache.doris.cloud.proto.Cloud.ObjectFilePB;
5353
import org.apache.doris.cloud.rpc.MetaServiceProxy;
54-
import org.apache.doris.cloud.storage.ObjectFile;
5554
import org.apache.doris.cloud.system.CloudSystemInfoService;
5655
import org.apache.doris.common.Config;
5756
import org.apache.doris.common.DdlException;
5857
import org.apache.doris.common.MetaNotFoundException;
5958
import org.apache.doris.common.util.ColumnsUtil;
6059
import org.apache.doris.datasource.InternalCatalog;
60+
import org.apache.doris.fs.obj.ObjectFile;
6161
import org.apache.doris.proto.OlapCommon;
6262
import org.apache.doris.proto.OlapFile;
6363
import org.apache.doris.proto.OlapFile.EncryptionAlgorithmPB;

‎fe/fe-core/src/main/java/org/apache/doris/cloud/load/CleanCopyJobTask.java‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,12 @@
2222
import org.apache.doris.cloud.proto.Cloud.FinishCopyRequest.Action;
2323
import org.apache.doris.cloud.proto.Cloud.StagePB;
2424
import org.apache.doris.cloud.proto.Cloud.StagePB.StageType;
25-
import org.apache.doris.cloud.storage.RemoteBase;
26-
import org.apache.doris.cloud.storage.RemoteBase.ObjectInfo;
25+
import org.apache.doris.cloud.storage.ObjectInfo;
26+
import org.apache.doris.cloud.storage.ObjectInfoAdapter;
2727
import org.apache.doris.common.Config;
28+
import org.apache.doris.datasource.property.storage.StorageProperties;
29+
import org.apache.doris.fs.FileSystemFactory;
30+
import org.apache.doris.fs.remote.ObjFileSystem;
2831

2932
import org.apache.logging.log4j.LogManager;
3033
import org.apache.logging.log4j.Logger;
@@ -33,7 +36,7 @@
3336

3437
public class CleanCopyJobTask {
3538
private static final Logger LOG = LogManager.getLogger(CleanCopyJobTask.class);
36-
private RemoteBase.ObjectInfo objectInfo;
39+
private ObjectInfo objectInfo;
3740
private String stageId;
3841
private StagePB.StageType stageType;
3942
private long tableId;
@@ -54,19 +57,16 @@ public void execute() {
5457
if (!Config.cloud_delete_loaded_internal_stage_files) {
5558
return;
5659
}
57-
RemoteBase remote = null;
5860
try {
59-
remote = RemoteBase.newInstance(objectInfo);
60-
remote.deleteObjects(loadFiles);
61+
StorageProperties storageProps = ObjectInfoAdapter.toStorageProperties(objectInfo);
62+
ObjFileSystem fs = (ObjFileSystem) FileSystemFactory.get(storageProps);
63+
fs.deleteObjectsByKeys(objectInfo.getBucket(), loadFiles);
6164
((CloudInternalCatalog) Env.getCurrentInternalCatalog())
6265
.finishCopy(stageId, stageType, tableId, copyId, 0, Action.REMOVE);
6366
} catch (Throwable e) {
6467
LOG.warn("Failed delete internal stage files={}", loadFiles, e);
6568
} finally {
6669
loadFiles = null;
67-
if (remote != null) {
68-
remote.close();
69-
}
7070
}
7171
}
7272
}

‎fe/fe-core/src/main/java/org/apache/doris/cloud/load/CopyJob.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
import org.apache.doris.cloud.proto.Cloud.StagePB;
2828
import org.apache.doris.cloud.proto.Cloud.StagePB.StageType;
2929
import org.apache.doris.cloud.stage.StageUtil;
30-
import org.apache.doris.cloud.storage.RemoteBase.ObjectInfo;
30+
import org.apache.doris.cloud.storage.ObjectInfo;
3131
import org.apache.doris.common.Config;
3232
import org.apache.doris.common.DdlException;
3333
import org.apache.doris.common.MetaNotFoundException;

‎fe/fe-core/src/main/java/org/apache/doris/cloud/load/CopyLoadPendingTask.java‎

Lines changed: 16 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -25,17 +25,20 @@
2525
import org.apache.doris.cloud.proto.Cloud.CopyJobPB.JobStatus;
2626
import org.apache.doris.cloud.proto.Cloud.ObjectFilePB;
2727
import org.apache.doris.cloud.stage.StageUtil;
28-
import org.apache.doris.cloud.storage.ListObjectsResult;
29-
import org.apache.doris.cloud.storage.ObjectFile;
30-
import org.apache.doris.cloud.storage.RemoteBase;
31-
import org.apache.doris.cloud.storage.RemoteBase.ObjectInfo;
28+
import org.apache.doris.cloud.storage.ObjectInfo;
29+
import org.apache.doris.cloud.storage.ObjectInfoAdapter;
3230
import org.apache.doris.common.Config;
3331
import org.apache.doris.common.DdlException;
3432
import org.apache.doris.common.Pair;
3533
import org.apache.doris.common.UserException;
3634
import org.apache.doris.common.util.BrokerUtil;
3735
import org.apache.doris.common.util.LogBuilder;
3836
import org.apache.doris.common.util.LogKey;
37+
import org.apache.doris.datasource.property.storage.StorageProperties;
38+
import org.apache.doris.fs.FileSystemFactory;
39+
import org.apache.doris.fs.obj.ListObjectsResult;
40+
import org.apache.doris.fs.obj.ObjectFile;
41+
import org.apache.doris.fs.remote.ObjFileSystem;
3942
import org.apache.doris.load.BrokerFileGroup;
4043
import org.apache.doris.load.BrokerFileGroupAggInfo.FileGroupAggKey;
4144
import org.apache.doris.load.loadv2.BrokerLoadPendingTask;
@@ -258,7 +261,8 @@ protected void listAndFilterFiles(ObjectInfo objectInfo, String pattern, String
258261
matchedFileNum = 0;
259262
loadedFileNum = 0;
260263
reachLimitStr = "";
261-
RemoteBase remote = RemoteBase.newInstance(objectInfo);
264+
StorageProperties storageProps = ObjectInfoAdapter.toStorageProperties(objectInfo);
265+
ObjFileSystem fs = (ObjFileSystem) FileSystemFactory.get(storageProps);
262266
Set<String> loadedFileSet = copiedFiles.stream().map(f -> StageUtil.getFileInfoUniqueId(f))
263267
.collect(Collectors.toSet());
264268
try {
@@ -268,7 +272,8 @@ protected void listAndFilterFiles(ObjectInfo objectInfo, String pattern, String
268272
String continuationToken = null;
269273
boolean finish = false;
270274
while (!finish) {
271-
ListObjectsResult listObjectsResult = remote.listObjects(continuationToken);
275+
ListObjectsResult listObjectsResult = fs.listObjectsWithPrefix(
276+
objectInfo.getPrefix(), "", continuationToken);
272277
listFileNum += listObjectsResult.getObjectInfoList().size();
273278
long costSeconds = (System.currentTimeMillis() - startTimestamp) / 1000;
274279
if (costSeconds >= 3600 || listFileNum >= 1000000) {
@@ -321,7 +326,7 @@ protected void listAndFilterFiles(ObjectInfo objectInfo, String pattern, String
321326
continuationToken = listObjectsResult.getContinuationToken();
322327
}
323328
} finally {
324-
remote.close();
329+
// ObjFileSystem has no close/cleanup to perform
325330
}
326331
}
327332

@@ -337,12 +342,13 @@ private List<Pair<TBrokerFileStatus, ObjectFilePB>> getCopyFilesWhenReplay(Strin
337342
throw new DdlException("Copy job is not in loading status, status=" + copyJobPB.getJobStatus());
338343
}
339344
isBeginCopyDone = true;
340-
RemoteBase remote = null;
341345
try {
342-
remote = RemoteBase.newInstance(objectInfo);
346+
StorageProperties storageProps = ObjectInfoAdapter.toStorageProperties(objectInfo);
347+
ObjFileSystem fs = (ObjFileSystem) FileSystemFactory.get(storageProps);
343348
List<Pair<TBrokerFileStatus, ObjectFilePB>> fileStatuses = Lists.newArrayList();
344349
for (ObjectFilePB objectFile : copyJobPB.getObjectFilesList()) {
345-
List<ObjectFile> files = remote.headObject(objectFile.getRelativePath()).getObjectInfoList();
350+
List<ObjectFile> files = fs.headObjectWithMeta(
351+
objectInfo.getPrefix(), objectFile.getRelativePath()).getObjectInfoList();
346352
TBrokerFileStatus brokerFileStatus = null;
347353
for (ObjectFile file : files) {
348354
if (file.getRelativePath().equals(objectFile.getRelativePath()) && file.getEtag()
@@ -360,8 +366,6 @@ private List<Pair<TBrokerFileStatus, ObjectFilePB>> getCopyFilesWhenReplay(Strin
360366
return fileStatuses;
361367
} catch (Exception e) {
362368
throw new DdlException(e.getMessage());
363-
} finally {
364-
remote.close();
365369
}
366370
}
367371
}

‎fe/fe-core/src/main/java/org/apache/doris/cloud/stage/StageUtil.java‎

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -24,15 +24,18 @@
2424
import org.apache.doris.cloud.proto.Cloud.ObjectFilePB;
2525
import org.apache.doris.cloud.proto.Cloud.StagePB;
2626
import org.apache.doris.cloud.proto.Cloud.StagePB.StageType;
27-
import org.apache.doris.cloud.storage.ListObjectsResult;
28-
import org.apache.doris.cloud.storage.ObjectFile;
29-
import org.apache.doris.cloud.storage.RemoteBase;
30-
import org.apache.doris.cloud.storage.RemoteBase.ObjectInfo;
27+
import org.apache.doris.cloud.storage.ObjectInfo;
28+
import org.apache.doris.cloud.storage.ObjectInfoAdapter;
3129
import org.apache.doris.common.AnalysisException;
3230
import org.apache.doris.common.Config;
3331
import org.apache.doris.common.DdlException;
3432
import org.apache.doris.common.Pair;
3533
import org.apache.doris.common.UserException;
34+
import org.apache.doris.datasource.property.storage.StorageProperties;
35+
import org.apache.doris.fs.FileSystemFactory;
36+
import org.apache.doris.fs.obj.ListObjectsResult;
37+
import org.apache.doris.fs.obj.ObjectFile;
38+
import org.apache.doris.fs.remote.ObjFileSystem;
3639
import org.apache.doris.mysql.privilege.PrivPredicate;
3740
import org.apache.doris.qe.ConnectContext;
3841
import org.apache.doris.thrift.TBrokerFileStatus;
@@ -149,7 +152,8 @@ public static Triple<Integer, Integer, String> listAndFilterFilesV2(ObjectInfo o
149152
int matchedFileNum = 0;
150153
int loadedFileNum = 0;
151154
String reachLimitStr = "";
152-
RemoteBase remote = RemoteBase.newInstance(objectInfo);
155+
StorageProperties storageProps = ObjectInfoAdapter.toStorageProperties(objectInfo);
156+
ObjFileSystem fs = (ObjFileSystem) FileSystemFactory.get(storageProps);
153157

154158
List<Pair<String, Boolean>> globs = analyzeGlob(copyId, pattern);
155159
LOG.info("Input copy into glob={}, analyzed={}", pattern, globs);
@@ -164,10 +168,11 @@ public static Triple<Integer, Integer, String> listAndFilterFilesV2(ObjectInfo o
164168
ListObjectsResult listObjectsResult;
165169
if (glob.second) {
166170
// list objects with sub prefix
167-
listObjectsResult = remote.listObjects(glob.first, continuationToken);
171+
listObjectsResult = fs.listObjectsWithPrefix(objectInfo.getPrefix(), glob.first,
172+
continuationToken);
168173
} else {
169174
// head object
170-
listObjectsResult = remote.headObject(glob.first);
175+
listObjectsResult = fs.headObjectWithMeta(objectInfo.getPrefix(), glob.first);
171176
}
172177
listFileNum += listObjectsResult.getObjectInfoList().size();
173178
long costSeconds = (System.currentTimeMillis() - startTimestamp) / 1000;
@@ -210,7 +215,7 @@ public static Triple<Integer, Integer, String> listAndFilterFilesV2(ObjectInfo o
210215
matchPatternFiles.clear();
211216
}
212217
} finally {
213-
remote.close();
218+
// ObjFileSystem has no close/cleanup to perform
214219
}
215220
return Triple.of(matchedFileNum, loadedFileNum, reachLimitStr);
216221
}
@@ -447,8 +452,8 @@ private static List<Pair<TBrokerFileStatus, ObjectFilePB>> generateFiles(List<Ob
447452
}
448453

449454
public static List<String> parseLoadFiles(List<String> loadFiles, String bucket, String stagePrefix) {
450-
if (!Config.cloud_delete_loaded_internal_stage_files || loadFiles == null || !RemoteBase.checkStagePrefix(
451-
stagePrefix)) {
455+
if (!Config.cloud_delete_loaded_internal_stage_files || loadFiles == null
456+
|| !ObjectInfoAdapter.checkStagePrefix(stagePrefix)) {
452457
return null;
453458
}
454459
String prefix = "s3://" + bucket + "/";

0 commit comments

Comments
 (0)