|
5 | 5 |
|
6 | 6 | import com.google.common.base.Preconditions; |
7 | 7 | import dev.vortex.VortexCleaner; |
| 8 | +import dev.vortex.io.NativeReadable; |
8 | 9 | import dev.vortex.jni.NativeDataSource; |
9 | 10 | import java.util.Arrays; |
10 | 11 | import java.util.Collections; |
@@ -71,6 +72,51 @@ public static DataSource open(Session session, List<String> uris, Map<String, St |
71 | 72 | return new DataSource(session, pointer); |
72 | 73 | } |
73 | 74 |
|
| 75 | + /** Open a single file through a caller-provided byte source. See {@link #open(Session, List, int)}. */ |
| 76 | + public static DataSource open(Session session, NativeReadable readable) { |
| 77 | + return open(session, List.of(readable), 0); |
| 78 | + } |
| 79 | + |
| 80 | + /** Open files through caller-provided byte sources with the default read concurrency. */ |
| 81 | + public static DataSource open(Session session, List<NativeReadable> readables) { |
| 82 | + return open(session, readables, 0); |
| 83 | + } |
| 84 | + |
| 85 | + /** |
| 86 | + * Open one or more files through caller-provided byte sources instead of native storage clients. Every read the |
| 87 | + * scan performs becomes an upcall into the corresponding {@link NativeReadable}, so this is the integration point |
| 88 | + * for external I/O abstractions (for example Iceberg's {@code FileIO}). Each file is identified by |
| 89 | + * {@link NativeReadable#name()}, which must be unique within {@code readables}. |
| 90 | + * |
| 91 | + * <p>The readables must remain open until this data source and all scans created from it are closed; native code |
| 92 | + * never closes them. |
| 93 | + * |
| 94 | + * @param session open session |
| 95 | + * @param readables byte sources |
| 96 | + * @param readConcurrency maximum in-flight {@code readFully} calls across all files of this data source; {@code 0} |
| 97 | + * selects the default. Each in-flight read occupies one native thread and typically one stream on its readable. |
| 98 | + */ |
| 99 | + public static DataSource open(Session session, List<NativeReadable> readables, int readConcurrency) { |
| 100 | + Objects.requireNonNull(session, "session"); |
| 101 | + Objects.requireNonNull(readables, "readables"); |
| 102 | + Preconditions.checkArgument(!readables.isEmpty(), "at least one readable is required"); |
| 103 | + Preconditions.checkArgument(readConcurrency >= 0, "readConcurrency must not be negative"); |
| 104 | + Object[] readableArray = readables.toArray(); |
| 105 | + String[] names = new String[readableArray.length]; |
| 106 | + long[] lengths = new long[readableArray.length]; |
| 107 | + for (int i = 0; i < readableArray.length; i++) { |
| 108 | + Preconditions.checkArgument(readableArray[i] != null, "readables must not contain null values"); |
| 109 | + NativeReadable readable = readables.get(i); |
| 110 | + names[i] = readable.name(); |
| 111 | + Preconditions.checkArgument(names[i] != null, "readable at index %s returned a null name", i); |
| 112 | + lengths[i] = readable.length(); |
| 113 | + Preconditions.checkArgument(lengths[i] >= 0, "readable for %s reported negative length", names[i]); |
| 114 | + } |
| 115 | + long pointer = |
| 116 | + NativeDataSource.openFiles(session.nativePointer(), readableArray, names, lengths, readConcurrency); |
| 117 | + return new DataSource(session, pointer); |
| 118 | + } |
| 119 | + |
74 | 120 | /** Arrow schema of the data source (and of scans produced from it). */ |
75 | 121 | public Schema arrowSchema(BufferAllocator allocator) { |
76 | 122 | try (ArrowSchema schema = ArrowSchema.allocateNew(allocator)) { |
|
0 commit comments