3030import org .apache .paimon .catalog .CatalogContext ;
3131import org .apache .paimon .catalog .CatalogFactory ;
3232import org .apache .paimon .catalog .Identifier ;
33- import org .apache .paimon .data .BinaryRow ;
3433import org .apache .paimon .data .GenericRow ;
3534import org .apache .paimon .data .InternalRow ;
36- import org .apache .paimon .io .BundleRecords ;
3735import org .apache .paimon .options .Options ;
3836import org .apache .paimon .table .Table ;
39- import org .apache .paimon .table .sink .BatchTableWrite ;
4037import org .apache .paimon .table .sink .CommitMessage ;
4138import org .slf4j .Logger ;
4239import org .slf4j .LoggerFactory ;
4340
4441import java .io .InputStream ;
45- import java .lang .reflect .Constructor ;
4642import java .nio .ByteBuffer ;
4743import java .util .ArrayList ;
48- import java .util .Iterator ;
4944import java .util .LinkedHashMap ;
5045import java .util .List ;
5146import java .util .Map ;
5752 * Called from C++ (VPaimonTableWriter) via JNI. Data path:
5853 *
5954 * C++ Block → Arrow IPC Stream → direct memory address
60- * → PaimonJniWriter.write(address, length )
55+ * → PaimonJniWriter.write(directBuffer )
6156 * → ArrowStreamReader → VectorSchemaRoot
6257 * → PaimonArrowConverter (column-major typed extraction)
6358 * → group by writer.getPartition(row) + writer.getBucket(row)
@@ -80,16 +75,14 @@ public class PaimonJniWriter {
8075 private final ClassLoader classLoader ;
8176 private final PaimonArrowConverter arrowConverter = new PaimonArrowConverter ();
8277 private final PaimonCommitCodec commitCodec = new PaimonCommitCodec ();
83- private final PaimonWriterLifecycle writerLifecycle = new PaimonWriterLifecycle ();
78+ private final PaimonSdkWriter sdkWriter = new PaimonSdkWriter ();
8479
8580 private String tableLocation ;
8681 private PreExecutionAuthenticator preExecutionAuthenticator ;
8782
8883 private PaimonWriteSchema writeSchema ;
8984 private long commitIdentifier ;
9085
91- private static volatile Constructor <?> directByteBufferConstructor ;
92-
9386 public PaimonJniWriter () {
9487 this .allocator = new RootAllocator (Long .MAX_VALUE );
9588 this .classLoader = this .getClass ().getClassLoader ();
@@ -131,7 +124,7 @@ public void open(String tableLocation, Map<String, String> options,
131124 options .getOrDefault (KEY_COMMIT_IDENTIFIER , "-1" ));
132125
133126 this .writeSchema = PaimonWriteSchema .create (table .rowType (), columnNames );
134- writerLifecycle .open (table , options , writeSchema .writeType ());
127+ sdkWriter .open (table , options , writeSchema .writeType ());
135128 return null ;
136129 } catch (Throwable t ) {
137130 throw new RuntimeException ("PaimonJniWriter open failed: location="
@@ -143,13 +136,11 @@ public void open(String tableLocation, Map<String, String> options,
143136 /**
144137 * Write a batch of rows. Called once per Arrow IPC buffer from C++.
145138 *
146- * @param address native memory address of the Arrow IPC Stream bytes
147- * @param length number of bytes
139+ * @param directBuffer direct view of the native Arrow IPC Stream bytes
148140 */
149- public void write (long address , int length ) throws Exception {
141+ public void write (ByteBuffer directBuffer ) throws Exception {
150142 preExecutionAuthenticator .execute (() -> {
151143 try {
152- ByteBuffer directBuffer = getDirectBuffer (address , length );
153144 try (ArrowStreamReader reader = new ArrowStreamReader (
154145 new DirectBufInputStream (directBuffer ), allocator )) {
155146 VectorSchemaRoot root = reader .getVectorSchemaRoot ();
@@ -159,8 +150,8 @@ public void write(long address, int length) throws Exception {
159150 }
160151 return null ;
161152 } catch (Throwable t ) {
162- throw new RuntimeException ("PaimonJniWriter write failed: addr ="
163- + address + ", len=" + length , t );
153+ throw new RuntimeException ("PaimonJniWriter write failed: bytes ="
154+ + directBuffer . capacity () , t );
164155 }
165156 });
166157 }
@@ -171,12 +162,9 @@ public void write(long address, int length) throws Exception {
171162 * @return byte[][] each element is a DPCM-framed serialized CommitMessage chunk
172163 */
173164 public byte [][] prepareCommit () throws Exception {
174- if (!writerLifecycle .isOpen ()) {
175- throw new IllegalStateException ("Paimon writer is not open" );
176- }
177165 return preExecutionAuthenticator .execute (() -> {
178166 try {
179- List <CommitMessage > messages = writerLifecycle .prepareCommit (commitIdentifier );
167+ List <CommitMessage > messages = sdkWriter .prepareCommit (commitIdentifier );
180168 if (messages .isEmpty ()) {
181169 LOG .info ("PaimonJniWriter prepareCommit: empty, location={}" , tableLocation );
182170 return new byte [0 ][];
@@ -197,11 +185,11 @@ public void abort() throws Exception {
197185 try {
198186 if (preExecutionAuthenticator != null ) {
199187 preExecutionAuthenticator .execute (() -> {
200- writerLifecycle .close ();
188+ sdkWriter .close ();
201189 return null ;
202190 });
203191 } else {
204- writerLifecycle .close ();
192+ sdkWriter .close ();
205193 }
206194 } catch (Exception e ) {
207195 LOG .error ("PaimonJniWriter abort failed" , e );
@@ -264,11 +252,9 @@ private void writeBatch(VectorSchemaRoot root) throws Exception {
264252 }
265253 int colCount = root .getFieldVectors ().size ();
266254 Object [][] columnValues = arrowConverter .convert (root , writeSchema .targetTypes ());
267- BatchTableWrite writer = writerLifecycle .writer ();
268-
269255 // Group rows by (partition, bucket) using the Paimon SDK's routing methods,
270256 // then write each group via writeBundle for batch-level I/O optimization.
271- Map <PartitionBucketKey , List <InternalRow >> groups = new LinkedHashMap <>();
257+ Map <PaimonSdkWriter . RoutingKey , List <InternalRow >> groups = new LinkedHashMap <>();
272258 for (int r = 0 ; r < rowCount ; r ++) {
273259 GenericRow row = new GenericRow (colCount );
274260 for (int c = 0 ; c < colCount ; c ++) {
@@ -277,15 +263,13 @@ private void writeBatch(VectorSchemaRoot root) throws Exception {
277263 // withWriteType controls the file row layout, but the SDK routing
278264 // extractors still read fields in table schema order.
279265 InternalRow routingRow = writeSchema .routingRow (row , columnValues , r );
280- BinaryRow partition = writer .getPartition (routingRow );
281- int bucket = writer .getBucket (routingRow );
282- groups .computeIfAbsent (new PartitionBucketKey (partition , bucket ),
266+ PaimonSdkWriter .RoutingKey routingKey = sdkWriter .route (routingRow );
267+ groups .computeIfAbsent (routingKey ,
283268 k -> new ArrayList <>()).add (row );
284269 }
285270
286- for (Map .Entry <PartitionBucketKey , List <InternalRow >> entry : groups .entrySet ()) {
287- PartitionBucketKey key = entry .getKey ();
288- writer .writeBundle (key .partition , key .bucket , new ListBundleRecords (entry .getValue ()));
271+ for (Map .Entry <PaimonSdkWriter .RoutingKey , List <InternalRow >> entry : groups .entrySet ()) {
272+ sdkWriter .writeBundle (entry .getKey (), entry .getValue ());
289273 }
290274 }
291275
@@ -295,7 +279,7 @@ private void writeBatch(VectorSchemaRoot root) throws Exception {
295279
296280 private void closeResources () throws Exception {
297281 try {
298- writerLifecycle .close ();
282+ sdkWriter .close ();
299283 } finally {
300284 writeSchema = null ;
301285 allocator .close ();
@@ -306,17 +290,6 @@ private void closeResources() throws Exception {
306290 // Utilities
307291 // ────────────────────────────────────────────────────────────
308292
309- private static ByteBuffer getDirectBuffer (long address , int length ) throws Exception {
310- Constructor <?> ctor = directByteBufferConstructor ;
311- if (ctor == null ) {
312- Class <?> cls = Class .forName ("java.nio.DirectByteBuffer" );
313- ctor = cls .getDeclaredConstructor (long .class , int .class );
314- ctor .setAccessible (true );
315- directByteBufferConstructor = ctor ;
316- }
317- return (ByteBuffer ) ctor .newInstance (address , length );
318- }
319-
320293 private static Catalog createCatalog (Map <String , String > paimonOpts ,
321294 Map <String , String > hadoopOpts ) {
322295 Options options = new Options ();
@@ -359,59 +332,6 @@ private static void configureLogLevels() {
359332 }
360333 }
361334
362- // ────────────────────────────────────────────────────────────
363- // Bundle helpers
364- // ────────────────────────────────────────────────────────────
365-
366- /** Compound key for grouping rows by (partition, bucket). */
367- static final class PartitionBucketKey {
368- final BinaryRow partition ;
369- final int bucket ;
370-
371- PartitionBucketKey (BinaryRow partition , int bucket ) {
372- // Paimon projections reuse their BinaryRow result across records.
373- // Keep an immutable snapshot because this object is also a map key.
374- this .partition = partition .copy ();
375- this .bucket = bucket ;
376- }
377-
378- @ Override
379- public int hashCode () {
380- return partition .hashCode () * 31 + bucket ;
381- }
382-
383- @ Override
384- public boolean equals (Object obj ) {
385- if (this == obj ) {
386- return true ;
387- }
388- if (!(obj instanceof PartitionBucketKey )) {
389- return false ;
390- }
391- PartitionBucketKey other = (PartitionBucketKey ) obj ;
392- return bucket == other .bucket && partition .equals (other .partition );
393- }
394- }
395-
396- /** BundleRecords backed by a List of pre-built InternalRows. */
397- private static final class ListBundleRecords implements BundleRecords {
398- private final List <InternalRow > rows ;
399-
400- ListBundleRecords (List <InternalRow > rows ) {
401- this .rows = rows ;
402- }
403-
404- @ Override
405- public long rowCount () {
406- return rows .size ();
407- }
408-
409- @ Override
410- public Iterator <InternalRow > iterator () {
411- return rows .iterator ();
412- }
413- }
414-
415335 /** InputStream over a direct ByteBuffer (no copy). */
416336 private static class DirectBufInputStream extends InputStream {
417337 private final ByteBuffer buf ;
0 commit comments