Skip to content

Commit f82a366

Browse files
schenksjclaude
andcommitted
fix(delta): feed kernel the analysis-time schema for with_schema (apache#78 schema-change-since-analysis)
Regression from the kernel-schema-shipping work: DeltaColumnMappingSuite "column mapping batch scan should detect physical name changes" read the current data instead of null-filling, because the driver fed `ScanBuilder::with_schema` the LIVE snapshot schema. Kernel resolves physical names from the schema passed to `with_schema` (StateInfo::try_new -> StructField::make_physical), so it must be fed the schema the query was PLANNED with -- then kernel's field-id matching null-fills any column whose id changed since analysis (Delta's schema-on-read escape hatch), which a pure-kernel engine handles itself with no fallback. - The JVM ships the analysis-time read schema as Delta schema JSON (`StructType.json` from DeltaScanRule's stashed reference schema, carrying `delta.columnMapping.physicalName` + id at every level). The driver parses it (`serde_json` -> kernel `StructType`, the same format kernel reads from the log) and feeds it straight to `with_schema`; it falls back to projecting the live snapshot by column name only when no analysis-time schema is available. - The analysis-time JSON and the Arrow-IPC names are mutually exclusive on the wire (ship the JSON when present, else the names) -- no redundant double-ship. - Fix `planDeltaReadSchemas` to build kernel schemas when EITHER carrier is present (it previously gated on the IPC only, so a JSON-only payload silently produced no schemas). Red->green guard: CometDeltaSchemaChangeReproSuite (Comet returned data, Spark null-filled; now both null-fill). Own-suite "physical name changes" goes green; all contrib Delta suites stay green. (DeltaColumnMappingSuite "explicit id matching" -- a contrived manual field-id repoint -- is a separate kernel id-vs-name matching nuance, tracked in apache#79.) Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
1 parent 18ea6d8 commit f82a366

7 files changed

Lines changed: 227 additions & 36 deletions

File tree

‎contrib/delta/native/Cargo.toml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,11 @@ parquet = { version = "58.1.0", default-features = false, features = ["experimen
6767
futures = "0.3"
6868
thiserror = "2"
6969
prost = "0.14.3"
70+
# Parse the analysis-time Delta schema JSON (shipped from the JVM) into a kernel `StructType` for
71+
# `ScanBuilder::with_schema`, so kernel resolves column-mapping physical names against the schema the
72+
# query was PLANNED with (correct for schema-change-since-analysis). Same format kernel itself uses
73+
# for the snapshot schema in the log.
74+
serde_json = "1"
7075
jni = "0.22.4"
7176
# Used by parse_delta_partition_scalar for timestamp parsing across the JVM's TZ shapes
7277
# (IANA names, GMT+/-HH:MM, etc).

‎contrib/delta/native/src/jni.rs‎

Lines changed: 33 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,11 @@ pub unsafe extern "system" fn Java_org_apache_comet_contrib_delta_Native_planDel
6767
// level. Drives `scan.with_schema(...)` so the driver returns kernel's projected
6868
// physical/logical schemas. Null/empty => full-table scan, no kernel schemas returned.
6969
projected_schema_ipc: JByteArray,
70+
// ANALYSIS-TIME read schema as Delta schema JSON (`StructType.json`, carrying
71+
// `delta.columnMapping.physicalName` + `id`). Preferred over `projected_schema_ipc` for
72+
// `with_schema` so kernel resolves the physical names the query was PLANNED with (correct for
73+
// schema-change-since-analysis). Empty => use the IPC names + snapshot projection.
74+
projected_schema_json: JString,
7075
) -> jbyteArray {
7176
try_unwrap_or_throw(&e, |env| {
7277
let url_str: String = table_url.try_to_string(env)?;
@@ -137,13 +142,16 @@ pub unsafe extern "system" fn Java_org_apache_comet_contrib_delta_Native_planDel
137142
// projects the SNAPSHOT schema by these (keeping column-mapping annotations). Absent/empty
138143
// => None (full-table scan, no kernel schemas returned).
139144
let projected_columns = decode_projected_columns(env, &projected_schema_ipc)?;
145+
// Analysis-time read schema (Delta JSON), preferred over the snapshot projection.
146+
let projected_schema_json = decode_jstring(env, &projected_schema_json)?;
140147

141148
let plan = plan_delta_scan_with_predicate(
142149
&url_str,
143150
&config,
144151
version,
145152
kernel_predicate,
146153
projected_columns,
154+
projected_schema_json,
147155
)
148156
.map_err(|e| CometError::Internal(format!("delta_kernel log replay failed: {e}")))?;
149157

@@ -245,6 +253,7 @@ pub unsafe extern "system" fn Java_org_apache_comet_contrib_delta_Native_planDel
245253
snapshot_version: jlong,
246254
storage_options: JObject,
247255
projected_schema_ipc: JByteArray,
256+
projected_schema_json: JString,
248257
) -> jbyteArray {
249258
try_unwrap_or_throw(&e, |env| {
250259
let url_str: String = table_url.try_to_string(env)?;
@@ -264,15 +273,25 @@ pub unsafe extern "system" fn Java_org_apache_comet_contrib_delta_Native_planDel
264273
extract_storage_config(env, &jmap, s3_bucket.as_deref())?
265274
};
266275

276+
let projected_schema_json = decode_jstring(env, &projected_schema_json)?;
277+
let projected_columns = decode_projected_columns(env, &projected_schema_ipc)?;
278+
// Build kernel schemas when EITHER carrier is present (the analysis-time JSON is preferred;
279+
// the IPC names are the fallback -- they're mutually exclusive on the wire). Both absent =>
280+
// zero data columns => no schemas needed.
267281
let (physical_schema, logical_schema) =
268-
match decode_projected_columns(env, &projected_schema_ipc)? {
269-
Some(cols) => crate::scan::plan_delta_read_schemas(
270-
&url_str, &config, version, cols,
282+
if projected_schema_json.is_some() || projected_columns.is_some() {
283+
crate::scan::plan_delta_read_schemas(
284+
&url_str,
285+
&config,
286+
version,
287+
projected_schema_json,
288+
projected_columns.unwrap_or_default(),
271289
)
272290
.map_err(|e| {
273291
CometError::Internal(format!("delta kernel read-schema build failed: {e}"))
274-
})?,
275-
None => (Vec::new(), Vec::new()),
292+
})?
293+
} else {
294+
(Vec::new(), Vec::new())
276295
};
277296

278297
let msg = DeltaScanTaskList {
@@ -427,6 +446,15 @@ fn decode_projected_columns(env: &mut Env, arr: &JByteArray) -> CometResult<Opti
427446
))
428447
}
429448

449+
/// Decode a Java `String` into `Option<String>`: `None` for null/empty.
450+
fn decode_jstring(env: &mut Env, s: &JString) -> CometResult<Option<String>> {
451+
if s.is_null() {
452+
return Ok(None);
453+
}
454+
let v = s.try_to_string(env)?;
455+
Ok(if v.is_empty() { None } else { Some(v) })
456+
}
457+
430458
/// Read a Java `String[]` into a `Vec<String>`. Returns empty vec for null arrays.
431459
fn read_string_array(env: &mut Env, arr: &jni::objects::JObjectArray) -> CometResult<Vec<String>> {
432460
if arr.is_null() {

‎contrib/delta/native/src/scan.rs‎

Lines changed: 53 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -140,16 +140,48 @@ fn scan_schemas_to_ipc(scan: &delta_kernel::scan::Scan) -> DeltaResult<(Vec<u8>,
140140
))
141141
}
142142

143+
/// Build the logical read schema for `ScanBuilder::with_schema`.
144+
///
145+
/// Prefers the ANALYSIS-TIME schema (`projected_schema_json` -- the Delta schema JSON the JVM
146+
/// stashed at query analysis, carrying `delta.columnMapping.physicalName` + `delta.columnMapping.id`
147+
/// at every nesting level, the same format kernel uses for the snapshot schema in the log). Feeding
148+
/// kernel the analysis-time schema is what makes schema-change-since-analysis correct: kernel's
149+
/// `make_physical` resolves physical names from THIS schema's own annotations and its field-id
150+
/// matching null-fills any column whose id changed since analysis (Delta's schema-on-read escape
151+
/// hatch), instead of reading the LIVE snapshot's current names. Falls back to projecting the
152+
/// snapshot's own schema by column name when no analysis-time JSON is available (e.g. a Delta
153+
/// FileIndex over a plain ParquetFileFormat, where no reference schema was stashed). `None` => no
154+
/// projection (full-table scan).
155+
fn build_read_schema(
156+
snapshot: &delta_kernel::snapshot::Snapshot,
157+
projected_schema_json: Option<&str>,
158+
projected_columns: Option<&[String]>,
159+
) -> DeltaResult<Option<delta_kernel::schema::SchemaRef>> {
160+
if let Some(json) = projected_schema_json {
161+
let st: delta_kernel::schema::StructType = serde_json::from_str(json).map_err(|e| {
162+
DeltaError::Internal(format!("parse analysis-time read schema JSON: {e}"))
163+
})?;
164+
Ok(Some(Arc::new(st)))
165+
} else if let Some(names) = projected_columns {
166+
Ok(Some(snapshot.schema().project(names).map_err(|e| {
167+
DeltaError::Internal(format!("project snapshot schema to read columns: {e}"))
168+
})?))
169+
} else {
170+
Ok(None)
171+
}
172+
}
173+
143174
/// Schema-only kernel scan for the batch-file-index read path (`PreparedDeltaFileIndex` /
144175
/// `TahoeBatchFileIndex` / ...), where the file list comes from Delta's `AddFile`s (NOT kernel log
145176
/// replay, for correctness) but the kernel-read executor still needs kernel's resolved
146177
/// physical/logical schemas. Builds the snapshot at `version` + a projected `Scan` and returns its
147-
/// `(physical_schema_ipc, logical_schema_ipc)`. Does NOT enumerate files. `projected_logical_schema`
148-
/// is the query's data-read columns in pure-logical names (see `plan_delta_scan_with_predicate`).
178+
/// `(physical_schema_ipc, logical_schema_ipc)`. Does NOT enumerate files. See `build_read_schema`
179+
/// for the analysis-time-schema vs snapshot projection.
149180
pub fn plan_delta_read_schemas(
150181
url_str: &str,
151182
config: &DeltaStorageConfig,
152183
version: Option<u64>,
184+
projected_schema_json: Option<String>,
153185
projected_columns: Vec<String>,
154186
) -> DeltaResult<(Vec<u8>, Vec<u8>)> {
155187
let url = normalize_url(url_str)?;
@@ -161,16 +193,15 @@ pub fn plan_delta_read_schemas(
161193
}
162194
builder.build(&*engine)?
163195
};
164-
// Project the snapshot's own schema (keeps the column-mapping annotations `with_schema` needs).
165-
let projected_schema = snapshot.schema().project(&projected_columns).map_err(|e| {
166-
DeltaError::Internal(format!("project snapshot schema to read columns: {e}"))
167-
})?;
196+
let read_schema = build_read_schema(
197+
&snapshot,
198+
projected_schema_json.as_deref(),
199+
Some(&projected_columns),
200+
)?
201+
.ok_or_else(|| DeltaError::Internal("read schemas requested with no projection".into()))?;
168202
// `Snapshot::build()` already returns `Arc<Snapshot>` (= `SnapshotRef`); `scan_builder` consumes
169203
// it by value.
170-
let scan = snapshot
171-
.scan_builder()
172-
.with_schema(projected_schema)
173-
.build()?;
204+
let scan = snapshot.scan_builder().with_schema(read_schema).build()?;
174205
scan_schemas_to_ipc(&scan)
175206
}
176207

@@ -233,7 +264,7 @@ pub fn plan_delta_scan(
233264
config: &DeltaStorageConfig,
234265
version: Option<u64>,
235266
) -> DeltaResult<DeltaScanPlan> {
236-
plan_delta_scan_with_predicate(url_str, config, version, None, None)
267+
plan_delta_scan_with_predicate(url_str, config, version, None, None, None)
237268
}
238269

239270
/// `projected_columns`: the top-level data columns the query reads, in logical names (the Spark
@@ -251,6 +282,7 @@ pub fn plan_delta_scan_with_predicate(
251282
version: Option<u64>,
252283
kernel_predicate: Option<delta_kernel::expressions::Predicate>,
253284
projected_columns: Option<Vec<String>>,
285+
projected_schema_json: Option<String>,
254286
) -> DeltaResult<DeltaScanPlan> {
255287
let url = normalize_url(url_str)?;
256288
let engine = get_or_create_engine(&url, config)?;
@@ -305,15 +337,16 @@ pub fn plan_delta_scan_with_predicate(
305337
if let Some(pred) = kernel_predicate {
306338
scan_builder = scan_builder.with_predicate(Arc::new(pred));
307339
}
308-
// Project the scan to the query's top-level data columns so `scan.physical_schema()` /
309-
// `scan.logical_schema()` carry the projected shape kernel resolves. Projecting the snapshot's
310-
// own schema keeps the column-mapping annotations `with_schema` needs.
311-
let projected = projected_columns.is_some();
312-
if let Some(ref names) = projected_columns {
313-
let projected_schema = snapshot_arc.schema().project(names).map_err(|e| {
314-
DeltaError::Internal(format!("project snapshot schema to read columns: {e}"))
315-
})?;
316-
scan_builder = scan_builder.with_schema(projected_schema);
340+
// Project the scan to the query's data columns so `scan.physical_schema()` /
341+
// `scan.logical_schema()` carry the projected shape kernel resolves -- preferring the
342+
// analysis-time schema so schema-change-since-analysis reads correctly (see `build_read_schema`).
343+
let projected = projected_schema_json.is_some() || projected_columns.is_some();
344+
if let Some(read_schema) = build_read_schema(
345+
&snapshot_arc,
346+
projected_schema_json.as_deref(),
347+
projected_columns.as_deref(),
348+
)? {
349+
scan_builder = scan_builder.with_schema(read_schema);
317350
}
318351
let scan = scan_builder.build()?;
319352

‎contrib/delta/src/main/scala/org/apache/comet/contrib/delta/CometDeltaNativeScan.scala‎

Lines changed: 51 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -185,6 +185,33 @@ object CometDeltaNativeScan extends CometOperatorSerde[CometDeltaScanMarker] wit
185185
else CometUtils.toArrowSchema(StructType(dataFields), timeZoneId).serializeAsMessage()
186186
}
187187

188+
/**
189+
* The ANALYSIS-TIME read schema as Delta schema JSON (`StructType.json`) -- the same data columns
190+
* as [[dataReadSchemaIpc]] but drawn from the schema the query was PLANNED with (DeltaScanRule's
191+
* stashed reference schema), so it carries `delta.columnMapping.physicalName` + `id` at every
192+
* nesting level. The driver feeds this to kernel's `ScanBuilder::with_schema`, which resolves
193+
* physical names from THESE annotations and null-fills columns whose field-id changed since
194+
* analysis (Delta's schema-on-read escape hatch). Returns `""` when the analysis-time schema is
195+
* absent or doesn't cover every data column (the driver then falls back to projecting the live
196+
* snapshot by the IPC names). Order matches [[dataReadSchemaIpc]] (requiredSchema's data order).
197+
*/
198+
private[delta] def dataReadSchemaJson(
199+
analyzedSchema: Option[StructType],
200+
requiredSchema: StructType): String = {
201+
analyzedSchema match {
202+
case Some(analyzed) =>
203+
val byName = analyzed.fields.map(f => f.name.toLowerCase(Locale.ROOT) -> f).toMap
204+
val dataFields = requiredSchema.fields.filterNot(f =>
205+
SyntheticReadFieldNames.contains(f.name.toLowerCase(Locale.ROOT)))
206+
val projected = dataFields.flatMap(f => byName.get(f.name.toLowerCase(Locale.ROOT)))
207+
// Only use the analysis-time schema when it covers every data column; otherwise fall back.
208+
if (projected.nonEmpty && projected.length == dataFields.length) {
209+
StructType(projected).json
210+
} else ""
211+
case None => ""
212+
}
213+
}
214+
188215
private[delta] def translateDeltaFieldIdToParquet(field: StructField): StructField = {
189216
val newDataType = translateDataTypeFieldIds(field.dataType)
190217
val newMetadata =
@@ -585,18 +612,27 @@ object CometDeltaNativeScan extends CometOperatorSerde[CometDeltaScanMarker] wit
585612
// wrapper detection upstream in `CometScanRule.scanBelowFallsBackForDvs`
586613
// is responsible for keeping DV-aware internal reads on vanilla.
587614
try {
615+
// The driver's `with_schema` prefers the ANALYSIS-TIME read schema (Delta JSON, carrying
616+
// column-mapping physicalName/id) so kernel resolves the names the query was planned with
617+
// -> correct under schema-change-since-analysis. The Arrow-IPC names are only the fallback
618+
// (no analysis-time schema available), so ship them ONLY then -- never both.
619+
val projJson = CometDeltaNativeScan.dataReadSchemaJson(
620+
scan.deltaMetadata.analyzedSchema,
621+
scan.requiredSchema)
622+
val projIpc =
623+
if (projJson.nonEmpty) Array.emptyByteArray
624+
else
625+
CometDeltaNativeScan.dataReadSchemaIpc(
626+
scan.requiredSchema,
627+
scan.conf.sessionLocalTimeZone)
588628
nativeLib.planDeltaScan(
589629
tableRoot,
590630
snapshotVersion,
591631
storageOptions,
592632
predicateBytes,
593633
columnNames,
594-
// Projected data-read schema -> drives kernel's `scan.with_schema(...)` so the returned
595-
// task list carries `scan.physical_schema()` / `scan.logical_schema()` for the
596-
// kernel-read path (correct physical names + field-ids at every nesting level).
597-
CometDeltaNativeScan.dataReadSchemaIpc(
598-
scan.requiredSchema,
599-
scan.conf.sessionLocalTimeZone))
634+
projIpc,
635+
projJson)
600636
} catch {
601637
case scala.util.control.NonFatal(e) =>
602638
logWarning(
@@ -831,14 +867,20 @@ object CometDeltaNativeScan extends CometOperatorSerde[CometDeltaScanMarker] wit
831867
if (!taskList.getPhysicalSchema.isEmpty) {
832868
taskList
833869
} else {
870+
// Prefer the analysis-time schema (JSON); ship the Arrow-IPC names only as the fallback
871+
// (never both). Both empty => zero data columns => no kernel schemas needed.
872+
val projJson =
873+
CometDeltaNativeScan.dataReadSchemaJson(scan.deltaMetadata.analyzedSchema, scan.requiredSchema)
834874
val projIpc =
835-
CometDeltaNativeScan.dataReadSchemaIpc(scan.requiredSchema, scan.conf.sessionLocalTimeZone)
836-
if (projIpc.isEmpty) {
875+
if (projJson.nonEmpty) Array.emptyByteArray
876+
else CometDeltaNativeScan.dataReadSchemaIpc(scan.requiredSchema, scan.conf.sessionLocalTimeZone)
877+
if (projJson.isEmpty && projIpc.isEmpty) {
837878
taskList
838879
} else {
839880
try {
840881
DeltaScanTaskList.parseFrom(
841-
nativeLib.planDeltaReadSchemas(tableRoot, snapshotVersion, storageOptions, projIpc))
882+
nativeLib
883+
.planDeltaReadSchemas(tableRoot, snapshotVersion, storageOptions, projIpc, projJson))
842884
} catch {
843885
case scala.util.control.NonFatal(e) =>
844886
// The kernel-read path has no Comet-side physicalisation fallback; if kernel can't

‎contrib/delta/src/main/scala/org/apache/comet/contrib/delta/Native.scala‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,18 +64,23 @@ class Native extends NativeBase {
6464
storageOptions: java.util.Map[String, String],
6565
predicateBytes: Array[Byte],
6666
columnNames: Array[String],
67-
projectedSchemaIpc: Array[Byte]): Array[Byte]
67+
projectedSchemaIpc: Array[Byte],
68+
projectedSchemaJson: String): Array[Byte]
6869

6970
/**
7071
* Schema-only companion to [[planDeltaScan]] for the batch-file-index read path (file list comes
7172
* from Delta `AddFile`s, but the kernel-read executor still needs kernel's resolved
7273
* physical/logical schemas). Returns a `DeltaScanTaskList` with only `physical_schema` /
7374
* `logical_schema` set (Arrow IPC). `projectedSchemaIpc` is the data-read projection in
7475
* pure-logical names (`Schema.serializeAsMessage()`); empty array => no schemas returned.
76+
* `projectedSchemaJson` is the ANALYSIS-TIME read schema as Delta schema JSON
77+
* (`StructType.json`, carrying column-mapping physicalName/id); preferred over the IPC names so
78+
* kernel resolves the names the query was planned with (schema-change-since-analysis correctness).
7579
*/
7680
@native def planDeltaReadSchemas(
7781
tableUrl: String,
7882
snapshotVersion: Long,
7983
storageOptions: java.util.Map[String, String],
80-
projectedSchemaIpc: Array[Byte]): Array[Byte]
84+
projectedSchemaIpc: Array[Byte],
85+
projectedSchemaJson: String): Array[Byte]
8186
}

0 commit comments

Comments
 (0)