@@ -300,14 +300,14 @@ private <T> T readTableSnapshot(String dbName, String tableName, LanceRefSelecto
300300 OptionalLong direct = directMainVersion (state , metrics );
301301 if (direct .isPresent () || isLatestMain (selector )) {
302302 state .version = direct ;
303- try (Dataset dataset = openDataset (allocator , state , direct , metrics )) {
303+ try (Dataset dataset = openDataset (allocator , state , direct , isLatestMain ( selector ), metrics )) {
304304 result = reader .read (dataset , state .access , metrics );
305305 }
306306 } else {
307307 OptionalLong mainVersion = state .access .isManagedVersioning ()
308308 ? OptionalLong .of (recordedLatestVersion (state , Optional .empty (), metrics ))
309309 : OptionalLong .empty ();
310- try (Dataset main = openDataset (allocator , state , mainVersion , metrics )) {
310+ try (Dataset main = openDataset (allocator , state , mainVersion , true , metrics )) {
311311 result = readFromLatest (main , state , reader , metrics );
312312 }
313313 }
@@ -455,15 +455,23 @@ private static long tagVersion(Dataset main, String tag, ReadState state) {
455455 }
456456 }
457457
458+ /**
459+ * A dataset URI without its query and trailing slash. The query may carry credentials, which a
460+ * namespace can vend anew on every describe.
461+ */
462+ private static String location (String uri ) {
463+ return StringUtils .removeEnd (StringUtils .substringBefore (uri , "?" ), "/" );
464+ }
465+
458466 /**
459467 * The branch a dataset checked out from the table root is on, from its root directory: the
460468 * table root for main, {@code <root>/tree/<branch>} otherwise. Lance inserts the branch path
461469 * before a URI's query string, so the query is compared apart. A URI that is neither is an
462470 * error rather than main, which would hand the BE the wrong chain.
463471 */
464472 static Optional <String > branchOf (String checkedOutUri , String tableUri ) {
465- String root = StringUtils . removeEnd ( StringUtils . substringBefore ( tableUri , "?" ), "/" );
466- String uri = StringUtils . removeEnd ( StringUtils . substringBefore ( checkedOutUri , "?" ), "/" );
473+ String root = location ( tableUri );
474+ String uri = location ( checkedOutUri );
467475 if (uri .equals (root )) {
468476 return Optional .empty ();
469477 }
@@ -503,6 +511,16 @@ private long recordedLatestVersion(ReadState state, Optional<String> branch, Lan
503511 .mapToLong (Long ::longValue ).max ()
504512 : metrics .measure (Stage .VERSION_RESOLVE ,
505513 () -> namespaceClient .latestManagedVersion (state .access , branch ));
514+ return requireRecorded (latest , state , branch );
515+ }
516+
517+ /** The newest version the namespace records now for the main chain, without the lists this read made. */
518+ private long recordedLatestVersion (ReadState state , LanceTableAccess access , LanceMetadataMetrics metrics ) {
519+ return requireRecorded (metrics .measure (Stage .VERSION_RESOLVE ,
520+ () -> namespaceClient .latestManagedVersion (access , Optional .empty ())), state , Optional .empty ());
521+ }
522+
523+ private static long requireRecorded (OptionalLong latest , ReadState state , Optional <String > branch ) {
506524 if (!latest .isPresent ()) {
507525 throw new LanceUserFacingException ("Lance namespace lists no versions for " + state .tableName
508526 + branch .map (name -> "@" + name ).orElse ("" ));
@@ -676,8 +694,8 @@ private static LanceUserFacingException historyRemoved(long version, String requ
676694
677695 /**
678696 * Parses a {@code FOR TIME AS OF} value in the session time zone. Second and millisecond
679- * precision are accepted; commit times are compared at millisecond precision, the precision a
680- * namespace reports them in, so a timestamp in the millisecond a commit lands in selects it .
697+ * precision are accepted. Commit times are compared in full, so a commit later within the
698+ * requested millisecond is not selected .
681699 */
682700 private static long parseTimeTravelTimestamp (String value ) {
683701 long timestamp = TimeUtils .timeStringToLong (value , TimeUtils .getTimeZone ());
@@ -717,29 +735,51 @@ private static boolean isVersionNotFound(Throwable throwable) {
717735 * access this read ends up with, and must open what the FE planned. If the SDK did not open
718736 * with exactly that access's options, the dataset is opened once more with them. A namespace
719737 * that returns a relative location cannot be read in this mode.
738+ *
739+ * @param latest whether {@code version} is the newest version the namespace listed before the
740+ * SDK's describe. If the SDK opened another store or location, or could not open that
741+ * version, the namespace may have moved the table in between, and the number may name
742+ * another version there or none. The newest version is then listed again, and the dataset
743+ * opened once more if it changed.
720744 */
721- private Dataset openDataset (BufferAllocator allocator , ReadState state , OptionalLong version ,
745+ private Dataset openDataset (BufferAllocator allocator , ReadState state , OptionalLong version , boolean latest ,
722746 LanceMetadataMetrics metrics ) {
723747 if (state .access .isManagedVersioning ()) {
724748 LanceTableAccess access = state .access ;
749+ OptionalLong pinned = version ;
725750 for (int attempt = 0 ; ; attempt ++) {
726- ReadOptions readOptions = LanceReadOptions .forSharedSession (access .getStorageOptions (), version ,
751+ ReadOptions readOptions = LanceReadOptions .forSharedSession (access .getStorageOptions (), pinned ,
727752 session );
728753 LanceTableAccess requested = access ;
729754 LanceSdkNamespace sdkNamespace = namespaceClient .sdkNamespace (requested );
730755 state .sdkNamespace = sdkNamespace ;
731- Dataset dataset = metrics .measure (Stage .DATASET_OPEN , () -> namespaceClient .openManagedDataset (
732- allocator , requested , readOptions , session , sdkNamespace ));
756+ Dataset dataset ;
757+ try {
758+ dataset = metrics .measure (Stage .DATASET_OPEN , () -> namespaceClient .openManagedDataset (
759+ allocator , requested , readOptions , session , sdkNamespace ));
760+ } catch (Exception e ) {
761+ OptionalLong head = latest && attempt == 0
762+ ? headAfterFailedOpen (state , requested , pinned , e , metrics ) : pinned ;
763+ if (head .equals (pinned )) {
764+ throw e ;
765+ }
766+ pinned = repin (state , head );
767+ continue ;
768+ }
733769 boolean opensAsBuilt ;
770+ OptionalLong head = pinned ;
734771 try {
735772 Map <String , String > openedOptions = dataset .getInitialStorageOptions ();
736773 access = namespaceClient .accessOpenedBySdk (requested , dataset .uri (), openedOptions );
737774 opensAsBuilt = LanceNamespaceClient .opensAs (access , dataset .uri (), openedOptions );
775+ if (latest && !sameStore (requested , access )) {
776+ head = OptionalLong .of (recordedLatestVersion (state , access , metrics ));
777+ }
738778 } catch (RuntimeException e ) {
739779 dataset .close ();
740780 throw e ;
741781 }
742- if (opensAsBuilt ) {
782+ if (opensAsBuilt && head . equals ( pinned ) ) {
743783 state .access = access ;
744784 return dataset ;
745785 }
@@ -748,6 +788,7 @@ private Dataset openDataset(BufferAllocator allocator, ReadState state, Optional
748788 throw new LanceUserFacingException ("Lance namespace changed the location or storage options of "
749789 + state .tableName + " while it was being opened; retry the query" );
750790 }
791+ pinned = repin (state , head );
751792 }
752793 }
753794 LanceTableAccess access = state .access ;
@@ -756,6 +797,38 @@ private Dataset openDataset(BufferAllocator allocator, ReadState state, Optional
756797 .readOptions (readOptions ).build ());
757798 }
758799
800+ /**
801+ * The newest version the namespace records now, after an open of the newest version it listed
802+ * before failed, or {@code pinned} if that cannot be told.
803+ */
804+ private OptionalLong headAfterFailedOpen (ReadState state , LanceTableAccess access , OptionalLong pinned ,
805+ Exception failure , LanceMetadataMetrics metrics ) {
806+ try {
807+ return OptionalLong .of (recordedLatestVersion (state , access , metrics ));
808+ } catch (RuntimeException e ) {
809+ failure .addSuppressed (e );
810+ return pinned ;
811+ }
812+ }
813+
814+ /** Pins a read to {@code head}; a plain latest read also reports it as the version it reads. */
815+ private static OptionalLong repin (ReadState state , OptionalLong head ) {
816+ if (state .version .isPresent ()) {
817+ state .version = head ;
818+ }
819+ return head ;
820+ }
821+
822+ /**
823+ * Whether two accesses reach the same store at the same location. Credentials are left out: a
824+ * namespace may vend new ones on every describe.
825+ */
826+ static boolean sameStore (LanceTableAccess access , LanceTableAccess other ) {
827+ return location (access .getDatasetUri ()).equals (location (other .getDatasetUri ()))
828+ && LanceSdkNamespace .withoutCredentials (access .getStorageOptions ())
829+ .equals (LanceSdkNamespace .withoutCredentials (other .getStorageOptions ()));
830+ }
831+
759832 @ FunctionalInterface
760833 private interface SnapshotReader <T > {
761834 T read (Dataset dataset , LanceTableAccess access , LanceMetadataMetrics metrics );
0 commit comments