Skip to content

Commit 8375559

Browse files
authored
[refactor](lance) improve lance catalog implementation in FE (#66581)
### What problem does this PR solve? Problem Summary: refactor some code including: - Adds dedicated properties for filesystem and REST catalogs. - Uses the Lance `tableExists` API for table existence checks. - Refactors metadata loading and snapshot version resolution. - Improves method naming and documentation for readability. ### Release note None ### Check List (For Author) - Test <!-- At least one of them must be included. --> - [ ] Regression test - [ ] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [x] No need to test or manual test. Explain why: - [x] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason <!-- Add your reason? --> - Behavior changed: - [ ] No. - [ ] Yes. <!-- Explain the behavior change --> - Does this need documentation? - [ ] No. - [ ] Yes. <!-- Add document PR link here. eg: apache/doris-website#1214 --> ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label <!-- Add branch pick label that this PR should merge into -->
1 parent fdeff73 commit 8375559

18 files changed

Lines changed: 965 additions & 378 deletions

‎docker/thirdparties/docker-compose/iceberg/scripts/lance_rest_server.py‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,16 @@ def _decode_identifier(identifier: str) -> tuple[str, ...]:
7272
return tuple(part for part in identifier.split(DELIMITER) if part)
7373

7474

75+
def _namespace_exists(namespace: tuple[str, ...]) -> bool:
76+
if not namespace:
77+
return True
78+
return any(
79+
len(identifier) > len(namespace)
80+
and identifier[: len(namespace)] == namespace
81+
for identifier in TABLES
82+
)
83+
84+
7585
class LanceRestHandler(BaseHTTPRequestHandler):
7686
server_version = "DorisLanceRestFixture/1.0"
7787

@@ -147,6 +157,18 @@ def do_POST(self) -> None:
147157
)
148158
return
149159

160+
table_exists_match = re.fullmatch(r"/v1/table/(.+)/exists", path)
161+
if table_exists_match:
162+
identifier = _decode_identifier(table_exists_match.group(1))
163+
if identifier in TABLES:
164+
self._write_empty(200)
165+
return
166+
if not _namespace_exists(identifier[:-1]):
167+
self._write_json(404, {"error": "namespace not found", "code": 1})
168+
return
169+
self._write_json(404, {"error": "table not found", "code": 4})
170+
return
171+
150172
self._write_json(404, {"error": "not found", "code": 4})
151173

152174
def _authorized(self) -> bool:
@@ -168,6 +190,11 @@ def _write_json(self, status: int, body: dict) -> None:
168190
self.end_headers()
169191
self.wfile.write(response)
170192

193+
def _write_empty(self, status: int) -> None:
194+
self.send_response(status)
195+
self.send_header("Content-Length", "0")
196+
self.end_headers()
197+
171198
def log_message(self, message: str, *args: object) -> None:
172199
print(f"{self.address_string()} - {message % args}", flush=True)
173200

‎fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalCatalog.java‎

Lines changed: 86 additions & 254 deletions
Large diffs are not rendered by default.

‎fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceMetadataLoader.java‎

Lines changed: 29 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@
2121
import org.apache.arrow.memory.RootAllocator;
2222
import org.lance.Dataset;
2323
import org.lance.Fragment;
24-
import org.lance.ReadOptions;
2524

2625
import java.util.ArrayList;
2726
import java.util.List;
@@ -31,54 +30,58 @@
3130
/** Loads one fixed Lance dataset snapshot through the Lance Java SDK. */
3231
public final class LanceMetadataLoader {
3332
private static final long ALLOCATOR_LIMIT = 256L * 1024 * 1024;
34-
private static final long METADATA_CACHE_SIZE = 64L * 1024 * 1024;
3533

3634
private LanceMetadataLoader() {
3735
}
3836

3937
/**
40-
* Load metadata for a directly-addressed dataset, using Doris backend storage properties.
41-
* This overload owns a short-lived allocator and is suitable for an S3 TVF.
38+
* Loads the latest metadata for a directly addressed dataset and owns a short-lived allocator.
39+
*
40+
* <p>Called by {@link org.apache.doris.tablefunction.S3TableValuedFunction} when reading a
41+
* Lance dataset through an S3 TVF.
4242
*/
43-
public static LanceTableMetadata load(String datasetUri, Map<String, String> backendStorageOptions)
43+
public static LanceTableMetadata loadLatestForTvf(
44+
String datasetUri, Map<String, String> backendStorageOptions)
4445
throws Exception {
4546
try (BufferAllocator allocator = new RootAllocator(ALLOCATOR_LIMIT)) {
46-
return load(datasetUri, LanceStorageOptions.forJavaSdk(backendStorageOptions),
47+
return loadLatest(datasetUri, LanceStorageOptions.forJavaSdk(backendStorageOptions),
4748
backendStorageOptions, allocator);
4849
}
4950
}
5051

5152
/**
52-
* Load metadata with an allocator and Java storage options already owned by a catalog.
53-
* Schema, version and fragments are all read from the same opened dataset snapshot.
53+
* Loads the latest metadata with the allocator and storage options owned by a catalog.
54+
*
55+
* <p>Called by
56+
* {@link LanceExternalCatalog#loadTableMetadata(String, String, java.util.Optional)} when no
57+
* time-travel version is requested. Schema, version, and fragments are read from the same
58+
* opened dataset snapshot.
5459
*/
55-
public static LanceTableMetadata load(String datasetUri, Map<String, String> javaStorageOptions,
60+
public static LanceTableMetadata loadLatest(String datasetUri, Map<String, String> javaStorageOptions,
5661
Map<String, String> backendStorageOptions, BufferAllocator allocator) throws Exception {
57-
return load(datasetUri, javaStorageOptions, backendStorageOptions, OptionalLong.empty(), allocator);
62+
return loadInternal(
63+
datasetUri, javaStorageOptions, backendStorageOptions, OptionalLong.empty(), allocator);
5864
}
5965

60-
/** Load metadata from an explicitly selected Lance version. */
61-
public static LanceTableMetadata load(String datasetUri, Map<String, String> javaStorageOptions,
66+
/**
67+
* Loads metadata from an explicitly selected Lance version.
68+
*
69+
* <p>Called by
70+
* {@link LanceExternalCatalog#loadTableMetadata(String, String, java.util.Optional)} for both
71+
* {@code FOR VERSION AS OF} and the version resolved from {@code FOR TIME AS OF}.
72+
*/
73+
public static LanceTableMetadata loadVersion(String datasetUri, Map<String, String> javaStorageOptions,
6274
Map<String, String> backendStorageOptions, long version, BufferAllocator allocator) throws Exception {
63-
return load(datasetUri, javaStorageOptions, backendStorageOptions, OptionalLong.of(version), allocator);
75+
return loadInternal(
76+
datasetUri, javaStorageOptions, backendStorageOptions, OptionalLong.of(version), allocator);
6477
}
6578

66-
/** Resolve the latest Lance version whose commit time does not exceed the requested timestamp. */
67-
public static long resolveVersionAtOrBefore(String datasetUri, Map<String, String> javaStorageOptions,
68-
long timestampMillis, BufferAllocator allocator) throws Exception {
69-
ReadOptions readOptions = readOptions(javaStorageOptions, OptionalLong.empty());
70-
try (Dataset dataset = Dataset.open().allocator(allocator).uri(datasetUri)
71-
.readOptions(readOptions).build()) {
72-
return LanceSnapshotResolver.versionAtOrBefore(dataset.listVersions(), timestampMillis);
73-
}
74-
}
75-
76-
private static LanceTableMetadata load(String datasetUri, Map<String, String> javaStorageOptions,
79+
/** Shared implementation for the latest-version and explicit-version public entry points. */
80+
private static LanceTableMetadata loadInternal(String datasetUri, Map<String, String> javaStorageOptions,
7781
Map<String, String> backendStorageOptions, OptionalLong version,
7882
BufferAllocator allocator) throws Exception {
79-
ReadOptions readOptions = readOptions(javaStorageOptions, version);
8083
try (Dataset dataset = Dataset.open().allocator(allocator).uri(datasetUri)
81-
.readOptions(readOptions).build()) {
84+
.readOptions(LanceReadOptions.build(javaStorageOptions, version)).build()) {
8285
long resolvedVersion = dataset.version();
8386
List<LanceTableMetadata.LanceFragmentInfo> fragments = new ArrayList<>();
8487
for (Fragment fragment : dataset.getFragments()) {
@@ -89,15 +92,4 @@ private static LanceTableMetadata load(String datasetUri, Map<String, String> ja
8992
backendStorageOptions);
9093
}
9194
}
92-
93-
private static ReadOptions readOptions(Map<String, String> javaStorageOptions, OptionalLong version) {
94-
ReadOptions.Builder builder = new ReadOptions.Builder()
95-
.setStorageOptions(javaStorageOptions)
96-
.setIndexCacheSizeBytes(0)
97-
.setMetadataCacheSizeBytes(METADATA_CACHE_SIZE);
98-
if (version.isPresent()) {
99-
builder.setVersion(version.getAsLong());
100-
}
101-
return builder.build();
102-
}
10395
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceNamespaceName.java‎

Lines changed: 61 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -23,14 +23,32 @@
2323
import java.util.Collections;
2424
import java.util.List;
2525

26-
/** Reversible mapping between a hierarchical Lance namespace and one Doris database name. */
26+
/**
27+
* Converts between a hierarchical Lance namespace and the flat database name exposed by Doris.
28+
*
29+
* <p>The input namespace is relative to the optional parent namespace configured on the catalog.
30+
* An empty relative namespace maps to {@code rootDatabase}. Other namespace components are joined
31+
* with {@code delimiter}; delimiter occurrences and backslashes inside a component are escaped so
32+
* the conversion remains reversible. A one-level namespace equal to {@code rootDatabase} receives
33+
* an additional leading escape to distinguish it from the empty root namespace.
34+
*
35+
* <p>This class only converts names. It does not access the remote Lance namespace service.
36+
*/
2737
final class LanceNamespaceName {
2838
private static final char ESCAPE = '\\';
2939

3040
private LanceNamespaceName() {
3141
}
3242

33-
static String encode(List<String> relativeNamespace, String delimiter, String rootDatabase) {
43+
/**
44+
* Converts relative Lance namespace components to one Doris database name.
45+
*
46+
* <p>For example, with {@code "."} as the delimiter,
47+
* {@code [company.analytics, raw]} becomes {@code company\.analytics.raw}.
48+
* An empty namespace becomes {@code rootDatabase}.
49+
*/
50+
static String namespaceToDorisDatabaseName(
51+
List<String> relativeNamespace, String delimiter, String rootDatabase) {
3452
if (relativeNamespace.isEmpty()) {
3553
return rootDatabase;
3654
}
@@ -39,32 +57,58 @@ static String encode(List<String> relativeNamespace, String delimiter, String ro
3957
if (i > 0) {
4058
result.append(delimiter);
4159
}
42-
result.append(escape(relativeNamespace.get(i), delimiter));
60+
result.append(escapeNamespaceComponent(relativeNamespace.get(i), delimiter));
4361
}
4462
// The local root database represents the empty Lance namespace. Escape
4563
// the rare, colliding one-level namespace so the mapping stays reversible.
4664
return rootDatabase.contentEquals(result) ? ESCAPE + result.toString() : result.toString();
4765
}
4866

49-
static List<String> decode(String database, String delimiter, String rootDatabase) throws DdlException {
50-
if (rootDatabase.equals(database)) {
67+
/** Escapes backslashes and delimiters occurring inside one namespace component. */
68+
private static String escapeNamespaceComponent(String component, String delimiter) {
69+
return component.replace("\\", "\\\\").replace(delimiter, "\\" + delimiter);
70+
}
71+
72+
/**
73+
* Restores the relative Lance namespace represented by one Doris database name.
74+
*
75+
* <p>For example, with {@code "."} as the delimiter,
76+
* {@code company\.analytics.raw} becomes {@code [company.analytics, raw]}.
77+
* A database name equal to {@code rootDatabase} becomes an empty namespace.
78+
*/
79+
static List<String> dorisDatabaseNameToNamespace(
80+
String databaseName, String delimiter, String rootDatabase) throws DdlException {
81+
if (rootDatabase.equals(databaseName)) {
5182
return Collections.emptyList();
5283
}
53-
if (database.length() > 1 && database.charAt(0) == ESCAPE
54-
&& rootDatabase.equals(database.substring(1))) {
55-
return splitEscaped(database.substring(1), delimiter);
84+
if (databaseName.length() > 1 && databaseName.charAt(0) == ESCAPE
85+
&& rootDatabase.equals(databaseName.substring(1))) {
86+
return parseEscapedNamespace(databaseName.substring(1), delimiter);
5687
}
57-
return splitEscaped(database, delimiter);
88+
return parseEscapedNamespace(databaseName, delimiter);
5889
}
5990

60-
static List<String> parseParent(String value, String delimiter) throws DdlException {
91+
/**
92+
* Parses the escaped parent namespace configured in the catalog properties.
93+
*
94+
* <p>For example, with {@code "."} as the delimiter,
95+
* {@code company.analytics} becomes {@code [company, analytics]}.
96+
*/
97+
static List<String> parseParentNamespace(String value, String delimiter) throws DdlException {
6198
if (value == null || value.isEmpty()) {
6299
return Collections.emptyList();
63100
}
64-
return splitEscaped(value, delimiter);
101+
return parseEscapedNamespace(value, delimiter);
65102
}
66103

67-
private static List<String> splitEscaped(String value, String delimiter) throws DdlException {
104+
/**
105+
* Splits an escaped namespace and rejects invalid escapes or empty components.
106+
*
107+
* <p>For example, with {@code "."} as the delimiter, {@code company\.analytics.raw}
108+
* is parsed as {@code [company.analytics, raw]}.
109+
*/
110+
private static List<String> parseEscapedNamespace(String value, String delimiter)
111+
throws DdlException {
68112
List<String> result = new ArrayList<>();
69113
StringBuilder current = new StringBuilder();
70114
for (int i = 0; i < value.length();) {
@@ -80,22 +124,19 @@ private static List<String> splitEscaped(String value, String delimiter) throws
80124
throw new DdlException("Invalid escape in Lance namespace '" + value + "'");
81125
}
82126
} else if (value.startsWith(delimiter, i)) {
83-
addComponent(result, current, value);
127+
addNamespaceComponent(result, current, value);
84128
i += delimiter.length();
85129
} else {
86130
current.append(value.charAt(i++));
87131
}
88132
}
89-
addComponent(result, current, value);
133+
addNamespaceComponent(result, current, value);
90134
return result;
91135
}
92136

93-
private static String escape(String component, String delimiter) {
94-
return component.replace("\\", "\\\\").replace(delimiter, "\\" + delimiter);
95-
}
96-
97-
private static void addComponent(List<String> result, StringBuilder current, String encoded)
98-
throws DdlException {
137+
/** Adds one parsed component while enforcing that Lance namespace components are non-empty. */
138+
private static void addNamespaceComponent(
139+
List<String> result, StringBuilder current, String encoded) throws DdlException {
99140
if (current.length() == 0) {
100141
throw new DdlException("Empty Lance namespace component in '" + encoded + "'");
101142
}
Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.datasource.lance;
19+
20+
import org.lance.ReadOptions;
21+
22+
import java.util.Map;
23+
import java.util.OptionalLong;
24+
25+
/** Builds consistently configured SDK read options for Lance dataset metadata access. */
26+
final class LanceReadOptions {
27+
private static final long METADATA_CACHE_SIZE = 64L * 1024 * 1024;
28+
29+
private LanceReadOptions() {
30+
}
31+
32+
/** Called by metadata loading and timestamp-based snapshot resolution. */
33+
static ReadOptions build(Map<String, String> javaStorageOptions, OptionalLong version) {
34+
ReadOptions.Builder builder = new ReadOptions.Builder()
35+
.setStorageOptions(javaStorageOptions)
36+
.setIndexCacheSizeBytes(0)
37+
.setMetadataCacheSizeBytes(METADATA_CACHE_SIZE);
38+
if (version.isPresent()) {
39+
builder.setVersion(version.getAsLong());
40+
}
41+
return builder.build();
42+
}
43+
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceSnapshotResolver.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,11 +17,15 @@
1717

1818
package org.apache.doris.datasource.lance;
1919

20+
import org.apache.arrow.memory.BufferAllocator;
21+
import org.lance.Dataset;
2022
import org.lance.Version;
2123

2224
import java.time.Instant;
2325
import java.util.Comparator;
2426
import java.util.List;
27+
import java.util.Map;
28+
import java.util.OptionalLong;
2529

2630
/** Resolves Doris time-travel selectors to immutable Lance version IDs. */
2731
final class LanceSnapshotResolver {
@@ -43,6 +47,22 @@ static long parseVersion(String value) {
4347
return version;
4448
}
4549

50+
/**
51+
* Gets the latest Lance version whose commit time does not exceed the requested timestamp.
52+
*
53+
* <p>Called by
54+
* {@link LanceExternalCatalog#loadTableMetadata(String, String, java.util.Optional)} to resolve
55+
* a {@code FOR TIME AS OF} clause before loading the selected metadata snapshot.
56+
*/
57+
static long getVersionAtOrBefore(String datasetUri, Map<String, String> javaStorageOptions,
58+
long timestampMillis, BufferAllocator allocator) throws Exception {
59+
try (Dataset dataset = Dataset.open().allocator(allocator).uri(datasetUri)
60+
.readOptions(LanceReadOptions.build(javaStorageOptions, OptionalLong.empty())).build()) {
61+
return versionAtOrBefore(dataset.listVersions(), timestampMillis);
62+
}
63+
}
64+
65+
/** Selects a version from the version list fetched by {@link #getVersionAtOrBefore}. */
4666
static long versionAtOrBefore(List<Version> versions, long timestampMillis) {
4767
Instant requestedTime = Instant.ofEpochMilli(timestampMillis);
4868
return versions.stream()

0 commit comments

Comments
 (0)