Skip to content

Commit 8e0eda2

Browse files
committed
[fix](lance) Keep managed reads on the store and manifest the BE reads
- Count credentials in the managed store identity when the store never refreshes them: an OpenDAL store (use_opendal), which lance-io builds from the options once, or an expires_at_millis that does not parse as an unsigned 64-bit integer. Count the table location as well, since a namespace may vend credentials for the table's own prefix. - Reject a namespace-recorded finalized manifest outside the version's canonical path, which the BE opens by URI and number. The table root is derived from the location as lance-io derives it. - When a read pinned to the namespace's newest version finds the table in another store or location, or cannot open that version, list the newest version again and reopen once at it. - Compare FOR TIME AS OF with the full commit instant, which Lance records below a millisecond.
1 parent 78cb959 commit 8e0eda2

12 files changed

Lines changed: 696 additions & 74 deletions

File tree

‎docker/thirdparties/docker-compose/iceberg/scripts/lance_build_time_travel.py‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@
1818
"""Generate the Lance time-travel regression fixture.
1919
2020
time_travel.lance is a root table of the preinstalled Directory catalog next to
21-
all_types.lance. Every other fixture is compacted to a single version, so this is the one
22-
dataset whose history survives: three commits, none of them cleaned up.
21+
all_types.lance. Unlike the other fixtures, most of which are compacted to a single version,
22+
it keeps a history for FOR TIME AS OF, tags and branches: three commits, none of them cleaned up.
2323
2424
version 1 create row_id 1..3 tag column "v1" Lance tag v1
2525
version 2 append row_id 4..6 tag column "v2" Lance tag v2
@@ -45,7 +45,8 @@
4545
--storage-option endpoint=http://127.0.0.1:19000 --storage-option access_key_id=admin \
4646
--storage-option secret_access_key=password --storage-option region=us-east-1 \
4747
--storage-option allow_http=true
48-
# then sync tree/ and _refs/branches/ from that location into preinstalled_data/lance/time_travel.lance/
48+
# then sync tree/, _refs/branches/ and _refs/tags/rel.json from that location into
49+
# preinstalled_data/lance/time_travel.lance/
4950
5051
check() verifies the branch files and that they point at that URI without opening the branch,
5152
which is not possible offline.

‎docker/thirdparties/docker-compose/iceberg/scripts/lance_fixture_requirements.txt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,8 @@
2525
# registers runtime tables into the same __manifest
2626
# lance-java 12.0.0 Doris FE Directory Namespace client - fe/pom.xml
2727
#
28-
# The writer is pinned to the Lance generation the BE actually reads. It used to be pinned
28+
# The writer stays on Lance 7, the generation the BE read when this pin was set; the BE's
29+
# lance-rs 11 reads what it writes, which the regression run verifies. It used to be pinned
2930
# to the oldest reader instead, on the theory that an older writer is readable by every
3031
# reader above; that stopped being possible once the fixture had to carry non-Float32
3132
# indexes. Measured on pylance 4.0.1 (Lance 4): IVF_HNSW_FLAT + float64 and

‎fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceCatalogClient.java‎

Lines changed: 84 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -300,14 +300,14 @@ private <T> T readTableSnapshot(String dbName, String tableName, LanceRefSelecto
300300
OptionalLong direct = directMainVersion(state, metrics);
301301
if (direct.isPresent() || isLatestMain(selector)) {
302302
state.version = direct;
303-
try (Dataset dataset = openDataset(allocator, state, direct, metrics)) {
303+
try (Dataset dataset = openDataset(allocator, state, direct, isLatestMain(selector), metrics)) {
304304
result = reader.read(dataset, state.access, metrics);
305305
}
306306
} else {
307307
OptionalLong mainVersion = state.access.isManagedVersioning()
308308
? OptionalLong.of(recordedLatestVersion(state, Optional.empty(), metrics))
309309
: OptionalLong.empty();
310-
try (Dataset main = openDataset(allocator, state, mainVersion, metrics)) {
310+
try (Dataset main = openDataset(allocator, state, mainVersion, true, metrics)) {
311311
result = readFromLatest(main, state, reader, metrics);
312312
}
313313
}
@@ -455,15 +455,23 @@ private static long tagVersion(Dataset main, String tag, ReadState state) {
455455
}
456456
}
457457

458+
/**
459+
* A dataset URI without its query and trailing slash. The query may carry credentials, which a
460+
* namespace can vend anew on every describe.
461+
*/
462+
private static String location(String uri) {
463+
return StringUtils.removeEnd(StringUtils.substringBefore(uri, "?"), "/");
464+
}
465+
458466
/**
459467
* The branch a dataset checked out from the table root is on, from its root directory: the
460468
* table root for main, {@code <root>/tree/<branch>} otherwise. Lance inserts the branch path
461469
* before a URI's query string, so the query is compared apart. A URI that is neither is an
462470
* error rather than main, which would hand the BE the wrong chain.
463471
*/
464472
static Optional<String> branchOf(String checkedOutUri, String tableUri) {
465-
String root = StringUtils.removeEnd(StringUtils.substringBefore(tableUri, "?"), "/");
466-
String uri = StringUtils.removeEnd(StringUtils.substringBefore(checkedOutUri, "?"), "/");
473+
String root = location(tableUri);
474+
String uri = location(checkedOutUri);
467475
if (uri.equals(root)) {
468476
return Optional.empty();
469477
}
@@ -503,6 +511,16 @@ private long recordedLatestVersion(ReadState state, Optional<String> branch, Lan
503511
.mapToLong(Long::longValue).max()
504512
: metrics.measure(Stage.VERSION_RESOLVE,
505513
() -> namespaceClient.latestManagedVersion(state.access, branch));
514+
return requireRecorded(latest, state, branch);
515+
}
516+
517+
/** The newest version the namespace records now for the main chain, without the lists this read made. */
518+
private long recordedLatestVersion(ReadState state, LanceTableAccess access, LanceMetadataMetrics metrics) {
519+
return requireRecorded(metrics.measure(Stage.VERSION_RESOLVE,
520+
() -> namespaceClient.latestManagedVersion(access, Optional.empty())), state, Optional.empty());
521+
}
522+
523+
private static long requireRecorded(OptionalLong latest, ReadState state, Optional<String> branch) {
506524
if (!latest.isPresent()) {
507525
throw new LanceUserFacingException("Lance namespace lists no versions for " + state.tableName
508526
+ branch.map(name -> "@" + name).orElse(""));
@@ -676,8 +694,8 @@ private static LanceUserFacingException historyRemoved(long version, String requ
676694

677695
/**
678696
* Parses a {@code FOR TIME AS OF} value in the session time zone. Second and millisecond
679-
* precision are accepted; commit times are compared at millisecond precision, the precision a
680-
* namespace reports them in, so a timestamp in the millisecond a commit lands in selects it.
697+
* precision are accepted. Commit times are compared in full, so a commit later within the
698+
* requested millisecond is not selected.
681699
*/
682700
private static long parseTimeTravelTimestamp(String value) {
683701
long timestamp = TimeUtils.timeStringToLong(value, TimeUtils.getTimeZone());
@@ -717,29 +735,51 @@ private static boolean isVersionNotFound(Throwable throwable) {
717735
* access this read ends up with, and must open what the FE planned. If the SDK did not open
718736
* with exactly that access's options, the dataset is opened once more with them. A namespace
719737
* that returns a relative location cannot be read in this mode.
738+
*
739+
* @param latest whether {@code version} is the newest version the namespace listed before the
740+
* SDK's describe. If the SDK opened another store or location, or could not open that
741+
* version, the namespace may have moved the table in between, and the number may name
742+
* another version there or none. The newest version is then listed again, and the dataset
743+
* opened once more if it changed.
720744
*/
721-
private Dataset openDataset(BufferAllocator allocator, ReadState state, OptionalLong version,
745+
private Dataset openDataset(BufferAllocator allocator, ReadState state, OptionalLong version, boolean latest,
722746
LanceMetadataMetrics metrics) {
723747
if (state.access.isManagedVersioning()) {
724748
LanceTableAccess access = state.access;
749+
OptionalLong pinned = version;
725750
for (int attempt = 0; ; attempt++) {
726-
ReadOptions readOptions = LanceReadOptions.forSharedSession(access.getStorageOptions(), version,
751+
ReadOptions readOptions = LanceReadOptions.forSharedSession(access.getStorageOptions(), pinned,
727752
session);
728753
LanceTableAccess requested = access;
729754
LanceSdkNamespace sdkNamespace = namespaceClient.sdkNamespace(requested);
730755
state.sdkNamespace = sdkNamespace;
731-
Dataset dataset = metrics.measure(Stage.DATASET_OPEN, () -> namespaceClient.openManagedDataset(
732-
allocator, requested, readOptions, session, sdkNamespace));
756+
Dataset dataset;
757+
try {
758+
dataset = metrics.measure(Stage.DATASET_OPEN, () -> namespaceClient.openManagedDataset(
759+
allocator, requested, readOptions, session, sdkNamespace));
760+
} catch (Exception e) {
761+
OptionalLong head = latest && attempt == 0
762+
? headAfterFailedOpen(state, requested, pinned, e, metrics) : pinned;
763+
if (head.equals(pinned)) {
764+
throw e;
765+
}
766+
pinned = repin(state, head);
767+
continue;
768+
}
733769
boolean opensAsBuilt;
770+
OptionalLong head = pinned;
734771
try {
735772
Map<String, String> openedOptions = dataset.getInitialStorageOptions();
736773
access = namespaceClient.accessOpenedBySdk(requested, dataset.uri(), openedOptions);
737774
opensAsBuilt = LanceNamespaceClient.opensAs(access, dataset.uri(), openedOptions);
775+
if (latest && !sameStore(requested, access)) {
776+
head = OptionalLong.of(recordedLatestVersion(state, access, metrics));
777+
}
738778
} catch (RuntimeException e) {
739779
dataset.close();
740780
throw e;
741781
}
742-
if (opensAsBuilt) {
782+
if (opensAsBuilt && head.equals(pinned)) {
743783
state.access = access;
744784
return dataset;
745785
}
@@ -748,6 +788,7 @@ private Dataset openDataset(BufferAllocator allocator, ReadState state, Optional
748788
throw new LanceUserFacingException("Lance namespace changed the location or storage options of "
749789
+ state.tableName + " while it was being opened; retry the query");
750790
}
791+
pinned = repin(state, head);
751792
}
752793
}
753794
LanceTableAccess access = state.access;
@@ -756,6 +797,38 @@ private Dataset openDataset(BufferAllocator allocator, ReadState state, Optional
756797
.readOptions(readOptions).build());
757798
}
758799

800+
/**
801+
* The newest version the namespace records now, after an open of the newest version it listed
802+
* before failed, or {@code pinned} if that cannot be told.
803+
*/
804+
private OptionalLong headAfterFailedOpen(ReadState state, LanceTableAccess access, OptionalLong pinned,
805+
Exception failure, LanceMetadataMetrics metrics) {
806+
try {
807+
return OptionalLong.of(recordedLatestVersion(state, access, metrics));
808+
} catch (RuntimeException e) {
809+
failure.addSuppressed(e);
810+
return pinned;
811+
}
812+
}
813+
814+
/** Pins a read to {@code head}; a plain latest read also reports it as the version it reads. */
815+
private static OptionalLong repin(ReadState state, OptionalLong head) {
816+
if (state.version.isPresent()) {
817+
state.version = head;
818+
}
819+
return head;
820+
}
821+
822+
/**
823+
* Whether two accesses reach the same store at the same location. Credentials are left out: a
824+
* namespace may vend new ones on every describe.
825+
*/
826+
static boolean sameStore(LanceTableAccess access, LanceTableAccess other) {
827+
return location(access.getDatasetUri()).equals(location(other.getDatasetUri()))
828+
&& LanceSdkNamespace.withoutCredentials(access.getStorageOptions())
829+
.equals(LanceSdkNamespace.withoutCredentials(other.getStorageOptions()));
830+
}
831+
759832
@FunctionalInterface
760833
private interface SnapshotReader<T> {
761834
T read(Dataset dataset, LanceTableAccess access, LanceMetadataMetrics metrics);

0 commit comments

Comments
 (0)