|
20 | 20 | import org.apache.doris.common.AnalysisException; |
21 | 21 | import org.apache.doris.common.FeConstants; |
22 | 22 | import org.apache.doris.common.util.S3Util; |
| 23 | +import org.apache.doris.datasource.lance.metadata.LanceMetadataLoader; |
23 | 24 | import org.apache.doris.fs.FileSystemFactory; |
24 | 25 | import org.apache.doris.nereids.StatementContext; |
25 | 26 | import org.apache.doris.nereids.trees.expressions.Properties; |
|
42 | 43 |
|
43 | 44 | import java.util.Arrays; |
44 | 45 | import java.util.Collections; |
| 46 | +import java.util.HashMap; |
45 | 47 | import java.util.Map; |
46 | 48 | import java.util.concurrent.CompletableFuture; |
47 | 49 |
|
@@ -80,6 +82,95 @@ public void testLanceRejectsLocalStorageBeforeOpeningDataset() { |
80 | 82 | Assertions.assertTrue(exception.getMessage().contains("requires remote storage")); |
81 | 83 | } |
82 | 84 |
|
| 85 | + @Test |
| 86 | + public void testLanceRejectsBrokerLocalUrisBeforeOpeningDataset() { |
| 87 | + FeConstants.runningUnitTest = false; |
| 88 | + try (MockedStatic<LanceMetadataLoader> loader = Mockito.mockStatic(LanceMetadataLoader.class)) { |
| 89 | + for (String uri : Arrays.asList("dataset.lance", "/tmp/dataset.lance", "../dataset.lance", |
| 90 | + "//localhost/tmp/dataset.lance", "file:///tmp/dataset.lance", "file:/tmp/dataset.lance", |
| 91 | + "file:dataset.lance", "FILE:///tmp/dataset.lance", "file://localhost/tmp/dataset.lance", |
| 92 | + "file-object-store://ignored/tmp/dataset.lance", "file+uring://ignored/tmp/dataset.lance", |
| 93 | + "FILE-OBJECT-STORE://ignored/tmp/dataset.lance", "FILE+URING://ignored/tmp/dataset.lance", |
| 94 | + "c://ignored/tmp/dataset.lance", "memory://dataset.lance", "shared-memory://dataset.lance", |
| 95 | + "s3://bucket:bad/../../dataset.lance", "s3://bucket:65536/../../dataset.lance", |
| 96 | + "s3://:9000/dataset.lance", "s3://user@/dataset.lance", |
| 97 | + "s3://[::1%25eth0]/dataset.lance", "https://bucket:bad/../../dataset.lance", |
| 98 | + "", "s3:/bucket/dataset.lance", "s3://", "s3://bucket/bad path")) { |
| 99 | + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, |
| 100 | + () -> new S3TableValuedFunction(ImmutableMap.of( |
| 101 | + "format", "lance", "uri", uri, "fs.broker.support", "true")), uri); |
| 102 | + Assertions.assertTrue(exception.getMessage().contains("requires remote storage"), uri); |
| 103 | + } |
| 104 | + loader.verifyNoInteractions(); |
| 105 | + } |
| 106 | + } |
| 107 | + |
| 108 | + @Test |
| 109 | + public void testLanceRejectsGuessedBrokerFileUrisBeforeOpeningDataset() { |
| 110 | + FeConstants.runningUnitTest = false; |
| 111 | + try (MockedStatic<LanceMetadataLoader> loader = Mockito.mockStatic(LanceMetadataLoader.class)) { |
| 112 | + for (String uri : Arrays.asList("file:///tmp/dataset.lance", "file://localhost/tmp/dataset.lance")) { |
| 113 | + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, |
| 114 | + () -> new S3TableValuedFunction(ImmutableMap.of( |
| 115 | + "format", "lance", "uri", uri, "broker.name", "test"))); |
| 116 | + Assertions.assertTrue(exception.getMessage().contains("requires remote storage")); |
| 117 | + } |
| 118 | + loader.verifyNoInteractions(); |
| 119 | + } |
| 120 | + } |
| 121 | + |
| 122 | + @Test |
| 123 | + public void testLanceRejectsBrokerWithoutUriBeforeOpeningDataset() { |
| 124 | + FeConstants.runningUnitTest = false; |
| 125 | + try (MockedStatic<LanceMetadataLoader> loader = Mockito.mockStatic(LanceMetadataLoader.class)) { |
| 126 | + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, |
| 127 | + () -> new S3TableValuedFunction(ImmutableMap.of("format", "lance", "fs.broker.support", "true"))); |
| 128 | + Assertions.assertTrue(exception.getMessage().contains("requires remote storage")); |
| 129 | + loader.verifyNoInteractions(); |
| 130 | + } |
| 131 | + } |
| 132 | + |
| 133 | + @Test |
| 134 | + public void testLanceRemoteUrisReachMetadataLoader() throws Exception { |
| 135 | + FeConstants.runningUnitTest = false; |
| 136 | + for (String uri : Arrays.asList("s3://bucket/dataset.lance", "s3+ddb://bucket/dataset.lance", |
| 137 | + "oss://bucket/dataset.lance", "gs://bucket/dataset.lance", "az://container/dataset.lance", |
| 138 | + "abfss://[email protected]/dataset.lance", |
| 139 | + "gs://bucket_with_underscore/dataset.lance", "s3://bucket:9000/dataset.lance", |
| 140 | + "goosefs://bucket/dataset.lance", "goosefs:///dataset.lance", "goosefs:/dataset.lance", |
| 141 | + "cos://bucket/dataset.lance", |
| 142 | + "hf://datasets/repo/data.lance", |
| 143 | + "tos://bucket/dataset.lance", "S3://bucket/dataset.lance")) { |
| 144 | + assertLanceMetadataLoad(ImmutableMap.of( |
| 145 | + "format", "lance", "uri", uri, "fs.broker.support", "true"), uri); |
| 146 | + } |
| 147 | + } |
| 148 | + |
| 149 | + @Test |
| 150 | + public void testLanceHttpStorageUrisAreNormalizedBeforeValidation() throws Exception { |
| 151 | + FeConstants.runningUnitTest = false; |
| 152 | + Map<String, String> s3Properties = new HashMap<>(PROPERTIES); |
| 153 | + s3Properties.put("format", "lance"); |
| 154 | + s3Properties.put("uri", "https://bucket.s3.us-west-2.amazonaws.com/dataset.lance"); |
| 155 | + assertLanceMetadataLoad(s3Properties, "s3://bucket/dataset.lance"); |
| 156 | + assertLanceMetadataLoad(ImmutableMap.of( |
| 157 | + "format", "lance", "uri", "https://bucket.oss-cn-hangzhou.aliyuncs.com/dataset.lance", |
| 158 | + "oss.endpoint", "oss-cn-hangzhou.aliyuncs.com", "oss.access_key", "ak", "oss.secret_key", "sk"), |
| 159 | + "s3://bucket/dataset.lance"); |
| 160 | + } |
| 161 | + |
| 162 | + private void assertLanceMetadataLoad(Map<String, String> properties, String uri) throws Exception { |
| 163 | + try (MockedStatic<LanceMetadataLoader> loader = Mockito.mockStatic(LanceMetadataLoader.class)) { |
| 164 | + IllegalStateException reachedLoader = new IllegalStateException("remote metadata loader reached"); |
| 165 | + loader.when(() -> LanceMetadataLoader.loadLatestForTvf(Mockito.eq(uri), Mockito.anyList())) |
| 166 | + .thenThrow(reachedLoader); |
| 167 | + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, |
| 168 | + () -> new S3TableValuedFunction(properties)); |
| 169 | + Assertions.assertSame(reachedLoader, exception.getCause(), uri); |
| 170 | + loader.verify(() -> LanceMetadataLoader.loadLatestForTvf(Mockito.eq(uri), Mockito.anyList())); |
| 171 | + } |
| 172 | + } |
| 173 | + |
83 | 174 | @Test |
84 | 175 | public void testExactFilesKeepLiteralPathsAndSkipListing() throws Exception { |
85 | 176 | FeConstants.runningUnitTest = false; |
|
0 commit comments