Skip to content

Commit 616fa81

Browse files
adivaishCopilot
andcommitted
feat(delta): native Delta write via delta-kernel-rs
Adds an experimental native Delta write path, gated by `spark.comet.write.deltaNative.enabled` (default off) and the `delta` cargo feature. Native engine: - delta::write_batches / write_arrow_ipc: create the table from the Arrow IPC schema if it does not exist, then write all batches in one transactional commit (commit/retry logic factored into a shared helper). New JNI entry Native.writeDeltaTable(tableUri, ipcBytes). The arrow `ipc` feature is enabled only with the `delta` feature. Spark side: - CometDeltaWriteRule (post-hoc resolution) rewrites an eligible Delta SaveIntoDataSourceCommand into CometDeltaWriteCommand before Spark eagerly executes it. No-op unless the config is on. - CometDeltaWriteCommand writes natively only for a brand-new, non-partitioned table with kernel-supported primitive column types; any other case, or any native error, transparently falls back to Delta's own write, so a correct table is always produced. The query result is collected to the driver and serialized to a single Arrow IPC stream (CometDeltaWrite), so this is for small writes. Tests: native round-trip unit test (create + append + read back); CometDeltaNativeWriteSuite e2e (kernel-written table read back by Spark with matching data + commit provenance; fallback for partitioned writes and when disabled), registered in CI. Validated on Spark 4.1 + Delta 4.1.0: write 3/3, read 7/7, native delta 14/14; clippy clean (default + delta), check-suites=0. Part of apache#174. Co-authored-by: Copilot <[email protected]>
1 parent 9ab88ff commit 616fa81

12 files changed

Lines changed: 553 additions & 7 deletions

File tree

‎.github/workflows/pr_build_linux.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -319,6 +319,7 @@ jobs:
319319
org.apache.comet.CometFuzzIcebergSuite
320320
org.apache.comet.DataGeneratorSuite
321321
org.apache.comet.CometDeltaNativeReadSuite
322+
org.apache.comet.CometDeltaNativeWriteSuite
322323
- name: "shuffle"
323324
value: |
324325
org.apache.comet.exec.CometShuffleSuite

‎.github/workflows/pr_build_macos.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -135,6 +135,7 @@ jobs:
135135
org.apache.comet.CometFuzzIcebergSuite
136136
org.apache.comet.DataGeneratorSuite
137137
org.apache.comet.CometDeltaNativeReadSuite
138+
org.apache.comet.CometDeltaNativeWriteSuite
138139
- name: "shuffle"
139140
value: |
140141
org.apache.comet.exec.CometShuffleSuite

‎native/core/Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ datafusion-functions-nested = { version = "53.1.0" }
102102
[features]
103103
backtrace = ["datafusion/backtrace"]
104104
default = ["hdfs-opendal"]
105-
delta = ["dep:delta_kernel"]
105+
delta = ["dep:delta_kernel", "arrow/ipc"]
106106
hdfs = ["datafusion-comet-objectstore-hdfs"]
107107
hdfs-opendal = ["opendal", "object_store_opendal", "hdfs-sys"]
108108
jemalloc = ["tikv-jemallocator", "tikv-jemalloc-ctl"]

‎native/core/src/delta/mod.rs‎

Lines changed: 121 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ use std::sync::Arc;
3333
use delta_kernel::arrow::array::RecordBatch;
3434
use delta_kernel::arrow::datatypes::SchemaRef as ArrowSchemaRef;
3535
use delta_kernel::committer::FileSystemCommitter;
36-
use delta_kernel::engine::arrow_conversion::TryIntoArrow;
36+
use delta_kernel::engine::arrow_conversion::{TryFromArrow, TryIntoArrow};
3737
use delta_kernel::engine::arrow_data::{ArrowEngineData, EngineDataArrowExt};
3838
use delta_kernel::engine::default::executor::tokio::TokioBackgroundExecutor;
3939
use delta_kernel::engine::default::storage::store_from_url_opts;
@@ -200,14 +200,22 @@ pub async fn append(table_uri: &str, data: RecordBatch) -> DeltaResult<u64> {
200200
.await?;
201201
txn.add_files(file_metadata);
202202

203+
commit_with_retries(txn, &engine)
204+
}
205+
206+
/// Commit a prepared transaction, retrying on retryable conflicts. Returns the committed version.
207+
fn commit_with_retries(
208+
mut txn: delta_kernel::transaction::Transaction,
209+
engine: &KernelEngine,
210+
) -> DeltaResult<u64> {
203211
let mut retries = 0;
204212
loop {
205213
if retries > 5 {
206214
return Err(Error::generic(
207215
"exceeded maximum retries committing transaction",
208216
));
209217
}
210-
txn = match txn.commit(&engine)? {
218+
txn = match txn.commit(engine)? {
211219
CommitResult::CommittedTransaction(committed) => return Ok(committed.commit_version()),
212220
CommitResult::ConflictedTransaction(conflicted) => {
213221
return Err(Error::generic(format!(
@@ -223,6 +231,63 @@ pub async fn append(table_uri: &str, data: RecordBatch) -> DeltaResult<u64> {
223231
}
224232
}
225233

234+
/// Write a sequence of Arrow `RecordBatch`es to the unpartitioned Delta table at `table_uri` in a
235+
/// single transactional commit, creating the table (with `arrow_schema` as its logical schema) if
236+
/// it does not yet exist. Returns the committed version. The caller must restrict this to eligible
237+
/// tables (non-partitioned, kernel-supported types); partition columns are not handled here.
238+
pub async fn write_batches(
239+
table_uri: &str,
240+
arrow_schema: ArrowSchemaRef,
241+
batches: Vec<RecordBatch>,
242+
) -> DeltaResult<u64> {
243+
let (url, engine) = build_engine(table_uri)?;
244+
245+
// Create the table on first write (no existing snapshot).
246+
if Snapshot::builder_for(url.clone()).build(&engine).is_err() {
247+
let kernel_schema: SchemaRef = Arc::new(
248+
StructType::try_from_arrow(arrow_schema.as_ref())
249+
.map_err(|e| Error::generic(format!("arrow->delta schema conversion: {e}")))?,
250+
);
251+
let _ = kernel_create_table(url.as_str(), kernel_schema, "datafusion-comet/delta")
252+
.build(&engine, Box::new(FileSystemCommitter::new()))?
253+
.commit(&engine)?;
254+
}
255+
256+
let snapshot = Snapshot::builder_for(url).build(&engine)?;
257+
let mut txn = snapshot
258+
.transaction(Box::new(FileSystemCommitter::new()), &engine)?
259+
.with_operation("WRITE".to_string())
260+
.with_engine_info("datafusion-comet/delta")
261+
.with_data_change(true);
262+
263+
let write_context = Arc::new(txn.unpartitioned_write_context()?);
264+
for batch in batches {
265+
let engine_data = ArrowEngineData::new(batch);
266+
let file_metadata = engine
267+
.write_parquet(&engine_data, write_context.as_ref())
268+
.await?;
269+
txn.add_files(file_metadata);
270+
}
271+
272+
commit_with_retries(txn, &engine)
273+
}
274+
275+
/// Decode an Arrow IPC stream (schema + zero or more batches) from `ipc_bytes` and write it to the
276+
/// Delta table at `table_uri` as a single transactional commit (creating the table if missing).
277+
/// Returns the committed version. Synchronous: drives the kernel's async write on a fresh runtime
278+
/// so it can be called directly from a JNI thread.
279+
pub fn write_arrow_ipc(table_uri: &str, ipc_bytes: &[u8]) -> DeltaResult<u64> {
280+
let reader = arrow::ipc::reader::StreamReader::try_new(std::io::Cursor::new(ipc_bytes), None)
281+
.map_err(|e| Error::generic(format!("invalid arrow ipc stream: {e}")))?;
282+
let arrow_schema = reader.schema();
283+
let batches: Vec<RecordBatch> = reader
284+
.collect::<Result<Vec<_>, _>>()
285+
.map_err(|e| Error::generic(format!("failed to read arrow ipc batches: {e}")))?;
286+
let runtime = tokio::runtime::Runtime::new()
287+
.map_err(|e| Error::generic(format!("failed to start tokio runtime: {e}")))?;
288+
runtime.block_on(write_batches(table_uri, arrow_schema, batches))
289+
}
290+
226291
#[cfg(test)]
227292
mod tests {
228293
use super::*;
@@ -332,6 +397,60 @@ mod tests {
332397
assert!(summary.contains("id") && summary.contains("score"));
333398
}
334399

400+
#[test]
401+
fn write_arrow_ipc_creates_table_and_round_trips() {
402+
let dir = TempDir::new().unwrap();
403+
let uri = table_uri(&dir);
404+
let schema = test_schema();
405+
let arrow_schema: ArrowSchema = schema.as_ref().try_into_arrow().unwrap();
406+
let arrow_schema_ref = Arc::new(arrow_schema);
407+
408+
// Two batches => two parquet files written in a single commit; the table is created on the
409+
// fly from the IPC stream's schema.
410+
let mut buf: Vec<u8> = Vec::new();
411+
{
412+
let mut writer =
413+
arrow::ipc::writer::StreamWriter::try_new(&mut buf, arrow_schema_ref.as_ref())
414+
.unwrap();
415+
writer.write(&make_batch(&schema, 0, 4)).unwrap();
416+
writer.write(&make_batch(&schema, 4, 3)).unwrap();
417+
writer.finish().unwrap();
418+
}
419+
420+
let version = write_arrow_ipc(&uri, &buf).unwrap();
421+
assert_eq!(version, 1, "create (v0) then write (v1)");
422+
assert_eq!(list_scan_files(&uri).unwrap().len(), 2);
423+
424+
let batches = scan_to_batches(&uri, None, None).unwrap();
425+
assert_eq!(total_rows(&batches), 7);
426+
let mut ids: Vec<i64> = batches
427+
.iter()
428+
.flat_map(|b| {
429+
b.column(0)
430+
.as_any()
431+
.downcast_ref::<Int64Array>()
432+
.unwrap()
433+
.values()
434+
.to_vec()
435+
})
436+
.collect();
437+
ids.sort_unstable();
438+
assert_eq!(ids, (0..7).collect::<Vec<_>>());
439+
440+
// A second IPC write appends to the now-existing table.
441+
let mut buf2: Vec<u8> = Vec::new();
442+
{
443+
let mut writer =
444+
arrow::ipc::writer::StreamWriter::try_new(&mut buf2, arrow_schema_ref.as_ref())
445+
.unwrap();
446+
writer.write(&make_batch(&schema, 7, 2)).unwrap();
447+
writer.finish().unwrap();
448+
}
449+
let version2 = write_arrow_ipc(&uri, &buf2).unwrap();
450+
assert_eq!(version2, 2);
451+
assert_eq!(total_rows(&scan_to_batches(&uri, None, None).unwrap()), 9);
452+
}
453+
335454
#[test]
336455
fn projection_reads_only_requested_columns_in_order() {
337456
let dir = TempDir::new().unwrap();

‎native/core/src/execution/jni_api.rs‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1312,3 +1312,28 @@ pub unsafe extern "system" fn Java_org_apache_comet_Native_columnarToRowClose(
13121312
Ok(())
13131313
})
13141314
}
1315+
1316+
// ============================================================================
1317+
// Native Delta Lake write (delta feature only)
1318+
// ============================================================================
1319+
1320+
/// Write an Arrow IPC stream (`ipc_data`, schema + batches) to the Delta table at `table_uri` as a
1321+
/// single transactional commit, creating the table if it does not exist. Returns the committed
1322+
/// version. Compiled only with the `delta` feature; the Scala side gates the call on
1323+
/// `isFeatureEnabled("delta")`, so the missing symbol in a non-delta build is never reached.
1324+
#[cfg(feature = "delta")]
1325+
#[no_mangle]
1326+
pub extern "system" fn Java_org_apache_comet_Native_writeDeltaTable(
1327+
e: EnvUnowned,
1328+
_class: JClass,
1329+
table_uri: JString,
1330+
ipc_data: JByteArray,
1331+
) -> jlong {
1332+
try_unwrap_or_throw(&e, |env| {
1333+
let uri: String = table_uri.try_to_string(env)?;
1334+
let bytes = env.convert_byte_array(ipc_data)?;
1335+
let version = crate::delta::write_arrow_ipc(&uri, bytes.as_slice())
1336+
.map_err(|err| CometError::Internal(format!("native delta write failed: {err}")))?;
1337+
Ok(version as jlong)
1338+
})
1339+
}

‎spark/src/main/scala/org/apache/comet/CometConf.scala‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -138,9 +138,21 @@ object CometConf extends ShimCometConf {
138138
.doc(
139139
"Whether to enable native Delta Lake table scan using delta-kernel-rs. Experimental and " +
140140
"requires Comet to be built with the `delta` cargo feature. When enabled, eligible " +
141-
"Delta scans are read directly through native execution. Currently limited to " +
142-
"non-partitioned tables read without projection or data filters; other cases fall back " +
143-
"to Spark.")
141+
"Delta scans (including column projection, partitioned tables, and best-effort data " +
142+
"filter pushdown) are read directly through native execution; scans carrying partition " +
143+
"filters and other unsupported cases fall back to Spark.")
144+
.booleanConf
145+
.createWithDefault(false)
146+
147+
val COMET_DELTA_NATIVE_WRITE_ENABLED: ConfigEntry[Boolean] =
148+
conf("spark.comet.write.deltaNative.enabled")
149+
.category(CATEGORY_SCAN)
150+
.doc("Whether to enable native Delta Lake table writes using delta-kernel-rs. Experimental " +
151+
"and requires Comet to be built with the `delta` cargo feature. When enabled, an " +
152+
"eligible Delta save (creating a brand-new, non-partitioned table with kernel-supported " +
153+
"primitive column types) is written natively; any other case, or any native error, " +
154+
"transparently falls back to Delta's own write. The data is collected to the driver, so " +
155+
"this is intended for small writes only.")
144156
.booleanConf
145157
.createWithDefault(false)
146158

‎spark/src/main/scala/org/apache/comet/CometSparkSessionExtensions.scala‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ import org.apache.spark.sql.execution._
3232
import org.apache.spark.sql.internal.SQLConf
3333

3434
import org.apache.comet.CometConf._
35-
import org.apache.comet.rules.{CometExecRule, CometPlanAdaptiveDynamicPruningFilters, CometReuseSubquery, CometScanRule, CometSpark34AqeDppFallbackRule, EliminateRedundantTransitions}
35+
import org.apache.comet.rules.{CometDeltaWriteRule, CometExecRule, CometPlanAdaptiveDynamicPruningFilters, CometReuseSubquery, CometScanRule, CometSpark34AqeDppFallbackRule, EliminateRedundantTransitions}
3636
import org.apache.comet.shims.ShimCometSparkSessionExtensions
3737

3838
/**
@@ -87,6 +87,9 @@ class CometSparkSessionExtensions
8787
with Logging
8888
with ShimCometSparkSessionExtensions {
8989
override def apply(extensions: SparkSessionExtensions): Unit = {
90+
// Rewrite eligible Delta writes to a native write command during analysis (before Spark
91+
// eagerly executes the command). No-op unless spark.comet.write.deltaNative.enabled is set.
92+
extensions.injectPostHocResolutionRule { session => CometDeltaWriteRule(session) }
9093
extensions.injectColumnar { session => CometScanColumnar(session) }
9194
extensions.injectColumnar { session => CometExecColumnar(session) }
9295
// Pre-3.5 only: tag AQE DPP regions so the conversion rules below leave them Spark-native.

‎spark/src/main/scala/org/apache/comet/Native.scala‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,21 @@ class Native extends NativeBase {
7373
taskContext: TaskContext): Long
7474
// scalastyle:on
7575

76+
/**
77+
* Write an Arrow IPC stream (schema + batches) to a Delta table natively via delta-kernel-rs,
78+
* as a single transactional commit, creating the table if it does not exist. Only implemented
79+
* in native builds with the `delta` cargo feature; callers must gate on
80+
* `isFeatureEnabled("delta")`.
81+
*
82+
* @param tableUri
83+
* the table root URI (e.g. file:///path).
84+
* @param ipcData
85+
* the Arrow IPC stream bytes (schema followed by record batches).
86+
* @return
87+
* the committed Delta table version.
88+
*/
89+
@native def writeDeltaTable(tableUri: String, ipcData: Array[Byte]): Long
90+
7691
/**
7792
* Execute a native query plan based on given input Arrow arrays.
7893
*
Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.comet.rules
21+
22+
import org.apache.spark.sql.SparkSession
23+
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
24+
import org.apache.spark.sql.catalyst.rules.Rule
25+
import org.apache.spark.sql.comet.CometDeltaWriteCommand
26+
import org.apache.spark.sql.execution.datasources.SaveIntoDataSourceCommand
27+
28+
import org.apache.comet.CometConf
29+
30+
/**
31+
* Post-hoc resolution rule that rewrites an eligible Delta `SaveIntoDataSourceCommand` into a
32+
* [[CometDeltaWriteCommand]] so the table can be written natively via delta-kernel-rs. Runs
33+
* during analysis, before Spark eagerly executes the command. The actual eligibility decision
34+
* (and a transparent fallback to Delta) happens at execution time inside
35+
* [[CometDeltaWriteCommand]]; this rule only narrows to Delta save commands and is a no-op unless
36+
* `spark.comet.write.deltaNative.enabled` is set, so its blast radius is minimal.
37+
*/
38+
case class CometDeltaWriteRule(session: SparkSession) extends Rule[LogicalPlan] {
39+
40+
override def apply(plan: LogicalPlan): LogicalPlan = {
41+
if (!CometConf.COMET_DELTA_NATIVE_WRITE_ENABLED.get()) {
42+
return plan
43+
}
44+
plan match {
45+
case s: SaveIntoDataSourceCommand
46+
if CometDeltaWriteCommand.isDeltaDataSource(s.dataSource) =>
47+
CometDeltaWriteCommand(s)
48+
case other => other
49+
}
50+
}
51+
}

0 commit comments

Comments
 (0)