6161import java .util .List ;
6262import java .util .Locale ;
6363import java .util .Map ;
64- import java .util .NavigableMap ;
6564import java .util .NavigableSet ;
6665import java .util .Objects ;
6766import java .util .Optional ;
6867import java .util .OptionalLong ;
69- import java .util .TreeMap ;
7068import java .util .TreeSet ;
7169import java .util .function .BiFunction ;
7270import java .util .regex .Pattern ;
@@ -291,23 +289,6 @@ private LanceTableMetadata loadQueryMetadata(String dbName, String tableName,
291289 */
292290 private <T > T readTableSnapshot (String dbName , String tableName , LanceRefSelector selector ,
293291 SnapshotReader <T > reader ) {
294- try {
295- return readTableSnapshotOnce (dbName , tableName , selector , reader );
296- } catch (StaleTableAccessException e ) {
297- // The cached access predates a location change the SDK has already seen. Read once
298- // more with a fresh access, so the FE plans and the BE reads the same location.
299- namespaceClient .invalidateTableAccess (dbName , tableName );
300- try {
301- return readTableSnapshotOnce (dbName , tableName , selector , reader );
302- } catch (StaleTableAccessException again ) {
303- throw new RuntimeException ("Lance namespace reported a different location for " + dbName + "."
304- + tableName + " while it was being opened; retry the query" );
305- }
306- }
307- }
308-
309- private <T > T readTableSnapshotOnce (String dbName , String tableName , LanceRefSelector selector ,
310- SnapshotReader <T > reader ) {
311292 ReadState state = new ReadState (selector , dbName + "." + tableName );
312293 LanceMetadataMetrics metrics = LanceMetadataMetrics .startMetadataRead ();
313294 try {
@@ -319,14 +300,14 @@ private <T> T readTableSnapshotOnce(String dbName, String tableName, LanceRefSel
319300 OptionalLong direct = directMainVersion (state , metrics );
320301 if (direct .isPresent () || isLatestMain (selector )) {
321302 state .version = direct ;
322- try (Dataset dataset = openDataset (allocator , state . access , direct , metrics )) {
303+ try (Dataset dataset = openDataset (allocator , state , direct , metrics )) {
323304 result = reader .read (dataset , state .access , metrics );
324305 }
325306 } else {
326307 OptionalLong mainVersion = state .access .isManagedVersioning ()
327308 ? OptionalLong .of (recordedLatestVersion (state , Optional .empty (), metrics ))
328309 : OptionalLong .empty ();
329- try (Dataset main = openDataset (allocator , state . access , mainVersion , metrics )) {
310+ try (Dataset main = openDataset (allocator , state , mainVersion , metrics )) {
330311 result = readFromLatest (main , state , reader , metrics );
331312 }
332313 }
@@ -335,8 +316,6 @@ private <T> T readTableSnapshotOnce(String dbName, String tableName, LanceRefSel
335316 return result ;
336317 } catch (LanceUserFacingException e ) {
337318 throw new RuntimeException (e .getMessage (), e );
338- } catch (StaleTableAccessException e ) {
339- throw e ;
340319 } catch (Exception e ) {
341320 LanceTableAccess access = state .access ;
342321 String uri = access == null ? null : access .getDatasetUri ();
@@ -508,13 +487,6 @@ private LanceUserFacingException(String message) {
508487 }
509488 }
510489
511- /** The SDK opened a managed table at another location than the resolved access names. */
512- private static final class StaleTableAccessException extends RuntimeException {
513- private StaleTableAccessException () {
514- super ("Lance table location changed since its access was resolved" );
515- }
516- }
517-
518490 /**
519491 * The newest version the namespace records for a managed chain. Doris asks for it itself:
520492 * opening "latest" through the SDK falls back to the newest manifest in storage when the
@@ -636,14 +608,10 @@ private List<TableVersion> namespaceVersions(ReadState state, LanceTableAccess a
636608
637609 /**
638610 * Resolves {@code FOR TIME AS OF} to a version on the chain {@code latest} is checked out on,
639- * from the commit times the manifests record, as Lance resolves {@code asof}. A managed table
640- * only selects among the versions its namespace records.
641- *
642- * <p>The version next to the selection must still exist: if the version after it, or with
643- * nothing listed at or before the time the one before the oldest listed, was removed by
644- * cleanup, the table's state at that time is unknown and the query fails instead of reading
645- * an older snapshot. A recorded version missing from the storage listing because its manifest
646- * is still staged is checked out there, which finalizes it and yields its commit time.
611+ * from the commit times the manifests record, over the history {@link LanceSnapshotResolver}
612+ * describes. A managed table only selects among the versions its namespace records; one
613+ * missing from the storage listing because its manifest is still staged is checked out, which
614+ * finalizes it and yields its commit time.
647615 */
648616 private long resolveVersionAtOrBefore (Dataset latest , LanceTableAccess access , long timestamp ,
649617 String requestedText , ReadState state , LanceMetadataMetrics metrics ) {
@@ -652,64 +620,18 @@ private long resolveVersionAtOrBefore(Dataset latest, LanceTableAccess access, l
652620 .filter (Objects ::nonNull ).collect (Collectors .toCollection (TreeSet ::new ))
653621 : null ;
654622 long version = metrics .measure (Stage .VERSION_RESOLVE , () -> {
655- NavigableMap <Long , Version > listed = new TreeMap <>();
656- for (Version candidate : latest .listVersions ()) {
657- if (recorded == null || recorded .contains (candidate .getId ())) {
658- listed .put (candidate .getId (), candidate );
659- }
660- }
661- long selected ;
662623 try {
663- selected = LanceSnapshotResolver .versionAtOrBefore (listed .values (), timestamp , requestedText );
664- } catch (LanceSnapshotResolver .NoVersionAtOrBeforeException e ) {
665- if (recorded == null ) {
666- throw e ;
667- }
668- // Recorded versions older than the oldest listed one may still be staged. Walking
669- // back, every version passed was committed after the time, so the first one at or
670- // before it is the answer.
671- Long older = listed .isEmpty () ? recorded .last () : recorded .lower (listed .firstKey ());
672- while (older != null ) {
673- Version olderVersion = recordedVersion (latest , access , older );
674- if (olderVersion == null ) {
675- throw historyRemoved (older , requestedText , state );
676- }
677- if (LanceSnapshotResolver .commitMillis (olderVersion ) <= timestamp ) {
678- return older ;
679- }
680- older = recorded .lower (older );
681- }
682- throw e ;
683- }
684- // Lance numbers a chain's commits consecutively, so a storage chain is every number
685- // between the oldest and newest listed manifest; a managed chain is what is recorded.
686- Long next = nextVersion (selected , recorded , listed );
687- while (next != null ) {
688- Version nextVersion = listed .containsKey (next ) ? listed .get (next )
689- : recorded == null ? null : recordedVersion (latest , access , next );
690- if (nextVersion == null ) {
691- throw historyRemoved (next , requestedText , state );
692- }
693- if (LanceSnapshotResolver .commitMillis (nextVersion ) > timestamp ) {
694- break ;
695- }
696- selected = next ;
697- next = nextVersion (next , recorded , listed );
624+ return LanceSnapshotResolver .versionAtOrBefore (latest .listVersions (), recorded ,
625+ id -> recordedVersion (latest , access , id ), timestamp , requestedText );
626+ } catch (LanceSnapshotResolver .HistoryRemovedException e ) {
627+ throw historyRemoved (e .getVersion (), requestedText , state );
698628 }
699- return selected ;
700629 });
701630 LOG .debug ("Resolved Lance FOR TIME AS OF '{}' to version {} from manifest commit times" , requestedText ,
702631 version );
703632 return version ;
704633 }
705634
706- private static Long nextVersion (long version , NavigableSet <Long > recorded , NavigableMap <Long , Version > listed ) {
707- if (recorded != null ) {
708- return recorded .higher (version );
709- }
710- return version < listed .lastKey () ? version + 1 : null ;
711- }
712-
713635 /**
714636 * A namespace-recorded version checked out through the namespace, or null if it is gone. A
715637 * still-staged manifest is finalized by the checkout.
@@ -769,21 +691,46 @@ private static boolean isVersionNotFound(Throwable throwable) {
769691 || (lower .contains ("not found" ) && lower .contains ("_versions/" ));
770692 }
771693
772- private Dataset openDataset (BufferAllocator allocator , LanceTableAccess access , OptionalLong version ,
694+ /**
695+ * Opens the main chain of the table {@code state} resolved. For a managed table the SDK
696+ * describes the table again and opens the location and storage options that describe returns,
697+ * so {@code state.access} is replaced by the access for what it opened: the BE reads with the
698+ * access this read ends up with, and must open what the FE planned. If the SDK did not open
699+ * with exactly that access's options, the dataset is opened once more with them. A namespace
700+ * that returns a relative location cannot be read in this mode.
701+ */
702+ private Dataset openDataset (BufferAllocator allocator , ReadState state , OptionalLong version ,
773703 LanceMetadataMetrics metrics ) {
774- ReadOptions readOptions = LanceReadOptions .forSharedSession (access .getSdkStorageOptions (), version , session );
775- if (access .isManagedVersioning ()) {
776- // The SDK re-describes the table and opens the location the namespace returns, so a
777- // namespace that returns a relative location cannot be read in this mode. The BE
778- // opens the access's location, so both must still agree after that second describe.
779- Dataset dataset = metrics .measure (Stage .DATASET_OPEN ,
780- () -> namespaceClient .openManagedDataset (allocator , access , readOptions , session ));
781- if (!StringUtils .removeEnd (dataset .uri (), "/" ).equals (StringUtils .removeEnd (access .getDatasetUri (), "/" ))) {
704+ if (state .access .isManagedVersioning ()) {
705+ LanceTableAccess access = state .access ;
706+ for (int attempt = 0 ; ; attempt ++) {
707+ ReadOptions readOptions = LanceReadOptions .forSharedSession (access .getSdkStorageOptions (), version ,
708+ session );
709+ LanceTableAccess requested = access ;
710+ Dataset dataset = metrics .measure (Stage .DATASET_OPEN ,
711+ () -> namespaceClient .openManagedDataset (allocator , requested , readOptions , session ));
712+ boolean opensAsBuilt ;
713+ try {
714+ Map <String , String > openedOptions = dataset .getInitialStorageOptions ();
715+ access = namespaceClient .accessOpenedBySdk (requested , dataset .uri (), openedOptions );
716+ opensAsBuilt = LanceNamespaceClient .opensAs (access , dataset .uri (), openedOptions );
717+ } catch (RuntimeException e ) {
718+ dataset .close ();
719+ throw e ;
720+ }
721+ if (opensAsBuilt ) {
722+ state .access = access ;
723+ return dataset ;
724+ }
782725 dataset .close ();
783- throw new StaleTableAccessException ();
726+ if (attempt > 0 ) {
727+ throw new LanceUserFacingException ("Lance namespace changed the location or storage options of "
728+ + state .tableName + " while it was being opened; retry the query" );
729+ }
784730 }
785- return dataset ;
786731 }
732+ LanceTableAccess access = state .access ;
733+ ReadOptions readOptions = LanceReadOptions .forSharedSession (access .getSdkStorageOptions (), version , session );
787734 return metrics .measure (Stage .DATASET_OPEN , () -> Dataset .open ().allocator (allocator ).uri (access .getDatasetUri ())
788735 .readOptions (readOptions ).build ());
789736 }
0 commit comments