Skip to content

Commit 0911a80

Browse files
morningmanclaude
andcommitted
[fix](tvf) Keep file modification time in TVF file listings
### What problem does this PR solve? Issue Number: None Related PR: #62023, #66684 Problem Summary: Since #62023 moved TVF file listing onto the filesystem SPI, `ExternalFileTableValuedFunction#parseFile` builds each `TBrokerFileStatus` from a `FileEntry` without its modification time, although the S3, HDFS, Azure and local listings all fill `FileEntry#modificationTime`. Every `s3()` / `hdfs()` TVF range therefore reaches BE with mtime 0. BE keys cached blocks of external files by `path:mtime`. With the file cache on, which becomes the default, an object overwritten in place under the same key keeps serving the blocks cached from its old content. branch-4.x does not have this problem: its listing goes through `BrokerUtil.parseFile`, which copies the listed modification time. This sets the modification time again. ### Release note None ### Check List (For Author) - Test: Unit Test - `run-fe-ut.sh --run` on `ExternalFileTableValuedFunctionTest` (new `testListedFilesCarryTheirModificationTime`), `TVFScanNodeTest`, `SessionVariablesTest`, `VariableMgrTest`: 40 tests, 0 failures - `mvn checkstyle:check` on fe-core: 0 violations - Behavior changed: No (TVF ranges carry the listed modification time, as on branch-4.x) - Does this need documentation: No Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent 747dd54 commit 0911a80

2 files changed

Lines changed: 68 additions & 2 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -184,8 +184,12 @@ protected void parseFile() throws AnalysisException {
184184
entries = fs.listFiles(Location.of(path));
185185
}
186186
for (FileEntry e : entries) {
187-
fileStatuses.add(new TBrokerFileStatus(
188-
e.location().uri(), e.isDirectory(), e.length(), !e.isDirectory()));
187+
TBrokerFileStatus status = new TBrokerFileStatus(
188+
e.location().uri(), e.isDirectory(), e.length(), !e.isDirectory());
189+
// BE keys cached blocks of external files by path and modification time, so a
190+
// file overwritten in place must arrive with its new mtime, not as mtime 0.
191+
status.setModificationTime(e.modificationTime());
192+
fileStatuses.add(status);
189193
}
190194
} catch (IOException e) {
191195
throw new UserException("list files failed for path " + path + ": " + e.getMessage(), e);

‎fe/fe-core/src/test/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunctionTest.java‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
package org.apache.doris.tablefunction;
1919

20+
import org.apache.doris.analysis.BrokerDesc;
2021
import org.apache.doris.catalog.Column;
2122
import org.apache.doris.catalog.PrimitiveType;
2223
import org.apache.doris.catalog.StructField;
@@ -27,16 +28,24 @@
2728
import org.apache.doris.common.Pair;
2829
import org.apache.doris.common.util.FileFormatConstants;
2930
import org.apache.doris.common.util.FileFormatUtils;
31+
import org.apache.doris.filesystem.FileEntry;
32+
import org.apache.doris.filesystem.FileSystem;
33+
import org.apache.doris.filesystem.GlobListing;
34+
import org.apache.doris.filesystem.Location;
35+
import org.apache.doris.fs.FileSystemFactory;
3036
import org.apache.doris.proto.Types.PScalarType;
3137
import org.apache.doris.proto.Types.PStructField;
3238
import org.apache.doris.proto.Types.PTypeNode;
39+
import org.apache.doris.thrift.TBrokerFileStatus;
40+
import org.apache.doris.thrift.TFileType;
3341
import org.apache.doris.thrift.TPrimitiveType;
3442
import org.apache.doris.thrift.TTypeNodeType;
3543

3644
import com.google.common.collect.Lists;
3745
import com.google.common.collect.Maps;
3846
import org.junit.jupiter.api.Assertions;
3947
import org.junit.jupiter.api.Test;
48+
import org.mockito.MockedStatic;
4049
import org.mockito.Mockito;
4150

4251
import java.lang.reflect.Method;
@@ -45,6 +54,59 @@
4554
import java.util.Map;
4655

4756
public class ExternalFileTableValuedFunctionTest {
57+
@Test
58+
public void testListedFilesCarryTheirModificationTime() throws Exception {
59+
// BE keys cached blocks of an external file by path and modification time. Listing files
60+
// without their mtime ships every TVF range as mtime 0, so an object overwritten in place
61+
// would keep serving the blocks cached from its old content.
62+
FileSystem fs = Mockito.mock(FileSystem.class);
63+
Mockito.when(fs.globListWithLimit(Mockito.any(Location.class), Mockito.anyString(),
64+
Mockito.anyLong(), Mockito.anyLong()))
65+
.thenReturn(new GlobListing(Arrays.asList(
66+
new FileEntry(Location.of("s3://bucket/dir/a.csv"), 10L, false, 1_700_000_000_123L, null),
67+
new FileEntry(Location.of("s3://bucket/dir/b.csv"), 20L, false, 1_700_000_000_456L, null)),
68+
"bucket", "dir/", "dir/b.csv"));
69+
BrokerDesc brokerDesc = Mockito.mock(BrokerDesc.class);
70+
ListingTvf tvf = new ListingTvf(brokerDesc);
71+
try (MockedStatic<FileSystemFactory> factory = Mockito.mockStatic(FileSystemFactory.class)) {
72+
factory.when(() -> FileSystemFactory.getFileSystem(brokerDesc)).thenReturn(fs);
73+
tvf.parseFile();
74+
}
75+
76+
List<TBrokerFileStatus> statuses = tvf.getFileStatuses();
77+
Assertions.assertEquals(2, statuses.size());
78+
Assertions.assertEquals(1_700_000_000_123L, statuses.get(0).getModificationTime());
79+
Assertions.assertEquals(1_700_000_000_456L, statuses.get(1).getModificationTime());
80+
}
81+
82+
private static final class ListingTvf extends ExternalFileTableValuedFunction {
83+
private final BrokerDesc brokerDesc;
84+
85+
private ListingTvf(BrokerDesc brokerDesc) {
86+
this.brokerDesc = brokerDesc;
87+
}
88+
89+
@Override
90+
public TFileType getTFileType() {
91+
return TFileType.FILE_S3;
92+
}
93+
94+
@Override
95+
public String getFilePath() {
96+
return "s3://bucket/dir/*.csv";
97+
}
98+
99+
@Override
100+
public BrokerDesc getBrokerDesc() {
101+
return brokerDesc;
102+
}
103+
104+
@Override
105+
public String getTableName() {
106+
return "ListingTvf";
107+
}
108+
}
109+
48110
@Test
49111
public void testFileSchemaPreservesNestedFieldSpelling() throws Exception {
50112
ExternalFileTableValuedFunction tvf = Mockito.mock(

0 commit comments

Comments
 (0)