Skip to content

Commit fea28d7

Browse files
schenksjclaude
andcommitted
perf(contrib-delta) P7j: pre-parse session TZ and key injectors by op kind
Two perf-sweep items from apache#135: #7 parse_delta_partition_scalar TZ parse-once. The per-row chrono_tz::Tz::from_str (or fixed-offset parse) was happening inside parse_delta_partition_scalar for every TIMESTAMP partition value, but the session TZ string doesn't change within a scan. Introduce SessionTimezone enum (Tz | Offset | Invalid), parse once in build_delta_partitioned_files, pass the parsed value through. parse_delta_partition_scalar's signature gains &SessionTimezone and keeps session_tz: &str only for the error message. #2 PlanDataInjector lookup by op kind. injectPlanData was running `for (injector <- injectors if injector.canInject(op))` against every operator in the tree; for a 50-op plan with 3 injectors that's 150 canInject calls just to find no match on most ops. Add `opStructCase` to the PlanDataInjector trait, build a Map[OpStructCase, PlanDataInjector] once at object init, and look up by op.getOpStructCase before any canInject call. Iceberg/NativeScan/Delta injectors set their own opStructCase. Co-Authored-By: Claude Opus 4.7 (1M context) <[email protected]>
1 parent 56c2b01 commit fea28d7

3 files changed

Lines changed: 103 additions & 55 deletions

File tree

‎contrib/delta/native/src/planner.rs‎

Lines changed: 65 additions & 40 deletions
Original file line numberDiff line numberDiff line change
@@ -43,13 +43,68 @@ use url::Url;
4343

4444
use crate::proto::DeltaScanTask;
4545

46+
/// Pre-parsed session timezone, computed once per scan and reused across every partition
47+
/// value parse. Avoids the per-row `chrono_tz::Tz::from_str` lookup
48+
/// `parse_delta_partition_scalar` would otherwise do for every TIMESTAMP partition value.
49+
pub enum SessionTimezone {
50+
Tz(chrono_tz::Tz),
51+
Offset(chrono::FixedOffset),
52+
/// `session_tz` didn't parse as either a named TZ or a fixed offset. We defer the
53+
/// "invalid session TZ" error to the per-row parse path so callers that don't have any
54+
/// TIMESTAMP partitions never see it.
55+
Invalid,
56+
}
57+
58+
impl SessionTimezone {
59+
pub fn parse(session_tz: &str) -> Self {
60+
if let Ok(tz) = session_tz.parse::<chrono_tz::Tz>() {
61+
return Self::Tz(tz);
62+
}
63+
if let Some(off) = parse_fixed_offset(session_tz) {
64+
return Self::Offset(off);
65+
}
66+
Self::Invalid
67+
}
68+
}
69+
70+
fn parse_fixed_offset(s: &str) -> Option<chrono::FixedOffset> {
71+
let trimmed = s.trim();
72+
let body = trimmed
73+
.strip_prefix("GMT")
74+
.or_else(|| trimmed.strip_prefix("UTC"))
75+
.unwrap_or(trimmed);
76+
if body.is_empty() || body.eq_ignore_ascii_case("Z") {
77+
return Some(chrono::FixedOffset::east_opt(0).unwrap());
78+
}
79+
let (sign, rest) = match body.chars().next()? {
80+
'+' => (1, &body[1..]),
81+
'-' => (-1, &body[1..]),
82+
_ => return None,
83+
};
84+
let secs = if rest.contains(':') {
85+
let mut parts = rest.splitn(2, ':');
86+
let h: i32 = parts.next()?.parse().ok()?;
87+
let m: i32 = parts.next()?.parse().ok()?;
88+
h * 3600 + m * 60
89+
} else if rest.len() == 4 {
90+
let h: i32 = rest[..2].parse().ok()?;
91+
let m: i32 = rest[2..].parse().ok()?;
92+
h * 3600 + m * 60
93+
} else {
94+
let h: i32 = rest.parse().ok()?;
95+
h * 3600
96+
};
97+
chrono::FixedOffset::east_opt(sign * secs)
98+
}
99+
46100
/// Convert `DeltaScanTask`s into DataFusion `PartitionedFile`s. Delta's add.path is
47101
/// already an absolute URL once kernel has resolved it on the driver.
48102
pub fn build_delta_partitioned_files(
49103
tasks: &[DeltaScanTask],
50104
partition_schema: &Schema,
51105
session_tz: &str,
52106
) -> Result<Vec<PartitionedFile>, String> {
107+
let parsed_tz = SessionTimezone::parse(session_tz);
53108
let mut files = Vec::with_capacity(tasks.len());
54109
for task in tasks {
55110
let url = Url::parse(task.file_path.as_ref())
@@ -76,7 +131,7 @@ pub fn build_delta_partitioned_files(
76131
.iter()
77132
.find(|p| p.name == *field.name());
78133
let scalar = match proto_value.and_then(|p| p.value.clone()) {
79-
Some(s) => parse_delta_partition_scalar(&s, field.data_type(), session_tz)
134+
Some(s) => parse_delta_partition_scalar(&s, field.data_type(), &parsed_tz, session_tz)
80135
.map_err(|e| {
81136
format!(
82137
"Failed to parse Delta partition value for column '{}': {e}",
@@ -110,12 +165,12 @@ pub fn build_delta_partitioned_files(
110165
pub fn parse_delta_partition_scalar(
111166
s: &str,
112167
dt: &DataType,
168+
parsed_tz: &SessionTimezone,
113169
session_tz: &str,
114170
) -> Result<ScalarValue, String> {
115171
match dt {
116172
DataType::Timestamp(unit, tz_opt) => {
117173
use chrono::{DateTime, NaiveDateTime, TimeZone};
118-
use chrono_tz::Tz;
119174
if tz_opt.is_none() {
120175
let naive = NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S%.f")
121176
.or_else(|_| NaiveDateTime::parse_from_str(s, "%Y-%m-%d %H:%M:%S"))
@@ -155,52 +210,22 @@ pub fn parse_delta_partition_scalar(
155210
.map(|d| d.and_hms_opt(0, 0, 0).unwrap())
156211
})
157212
.map_err(|e| format!("cannot parse timestamp '{s}': {e}"))?;
158-
use chrono::{FixedOffset, LocalResult};
159-
fn parse_fixed_offset(s: &str) -> Option<FixedOffset> {
160-
let trimmed = s.trim();
161-
let body = trimmed
162-
.strip_prefix("GMT")
163-
.or_else(|| trimmed.strip_prefix("UTC"))
164-
.unwrap_or(trimmed);
165-
if body.is_empty() || body.eq_ignore_ascii_case("Z") {
166-
return Some(FixedOffset::east_opt(0).unwrap());
167-
}
168-
let (sign, rest) = match body.chars().next()? {
169-
'+' => (1, &body[1..]),
170-
'-' => (-1, &body[1..]),
171-
_ => return None,
172-
};
173-
let secs = if rest.contains(':') {
174-
let mut parts = rest.splitn(2, ':');
175-
let h: i32 = parts.next()?.parse().ok()?;
176-
let m: i32 = parts.next()?.parse().ok()?;
177-
h * 3600 + m * 60
178-
} else if rest.len() == 4 {
179-
let h: i32 = rest[..2].parse().ok()?;
180-
let m: i32 = rest[2..].parse().ok()?;
181-
h * 3600 + m * 60
182-
} else {
183-
let h: i32 = rest.parse().ok()?;
184-
h * 3600
185-
};
186-
FixedOffset::east_opt(sign * secs)
187-
}
188-
189-
if let Ok(tz) = session_tz.parse::<Tz>() {
190-
match tz.from_local_datetime(&naive) {
213+
use chrono::LocalResult;
214+
match parsed_tz {
215+
SessionTimezone::Tz(tz) => match tz.from_local_datetime(&naive) {
191216
LocalResult::Single(dt) => dt.timestamp_micros(),
192217
LocalResult::Ambiguous(earlier, _later) => earlier.timestamp_micros(),
193218
LocalResult::None => {
194219
chrono::Utc.from_utc_datetime(&naive).timestamp_micros()
195220
}
196-
}
197-
} else if let Some(off) = parse_fixed_offset(session_tz) {
198-
match off.from_local_datetime(&naive) {
221+
},
222+
SessionTimezone::Offset(off) => match off.from_local_datetime(&naive) {
199223
LocalResult::Single(dt) => dt.timestamp_micros(),
200224
_ => chrono::Utc.from_utc_datetime(&naive).timestamp_micros(),
225+
},
226+
SessionTimezone::Invalid => {
227+
return Err(format!("invalid session TZ '{session_tz}'"));
201228
}
202-
} else {
203-
return Err(format!("invalid session TZ '{session_tz}'"));
204229
}
205230
};
206231
match unit {

‎contrib/delta/src/main/scala/org/apache/spark/sql/comet/DeltaPlanDataInjector.scala‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@ import org.apache.comet.serde.OperatorOuterClass.Operator
4444
*/
4545
object DeltaPlanDataInjector extends PlanDataInjector {
4646

47+
override val opStructCase: Operator.OpStructCase = Operator.OpStructCase.DELTA_SCAN
48+
4749
override def canInject(op: Operator): Boolean = {
4850
if (!op.hasDeltaScan) return false
4951
// The common-only proto produced at planning time has zero tasks. After injection

‎spark/src/main/scala/org/apache/spark/sql/comet/operators.scala‎

Lines changed: 36 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,13 @@ import org.apache.comet.serde.operator.CometSink
6666
*/
6767
private[comet] trait PlanDataInjector {
6868

69+
/**
70+
* Which `OpStructCase` this injector cares about. Used by `injectPlanData` for an O(1)
71+
* pre-filter so we don't run every injector's `canInject` against every operator in the
72+
* tree.
73+
*/
74+
def opStructCase: Operator.OpStructCase
75+
6976
/** Check if this injector can handle the given operator. */
7077
def canInject(op: Operator): Boolean
7178

@@ -101,6 +108,13 @@ private[comet] object PlanDataInjector {
101108
builtin ++ deltaOpt
102109
}
103110

111+
// O(1) lookup by op kind: most operators in any tree don't match any injector, so the
112+
// per-op `for (injector <- injectors if injector.canInject(op))` walk was paying N*M
113+
// canInject calls (N operators, M injectors) just to find no match. Keying by
114+
// `OpStructCase` lets us skip the iteration entirely for non-scan operators.
115+
private val injectorsByKind: Map[Operator.OpStructCase, PlanDataInjector] =
116+
injectors.map(i => i.opStructCase -> i).toMap
117+
104118
/**
105119
* Injects planning data into an Operator tree by finding nodes that need injection and applying
106120
* the appropriate injector.
@@ -114,21 +128,24 @@ private[comet] object PlanDataInjector {
114128
partitionByKey: Map[String, Array[Byte]]): Operator = {
115129
val builder = op.toBuilder
116130

117-
// Try each injector to see if it can handle this operator
118-
for (injector <- injectors if injector.canInject(op)) {
119-
injector.getKey(op) match {
120-
case Some(key) =>
121-
(commonByKey.get(key), partitionByKey.get(key)) match {
122-
case (Some(commonBytes), Some(partitionBytes)) =>
123-
val injectedOp = injector.inject(op, commonBytes, partitionBytes)
124-
// Copy the injected operator's fields to our builder
125-
builder.clear()
126-
builder.mergeFrom(injectedOp)
127-
case _ =>
128-
throw new CometRuntimeException(s"Missing planning data for key: $key")
129-
}
130-
case None =>
131-
}
131+
// O(1) by op kind, then a canInject confirm (which may inspect detail fields like
132+
// `hasCommon` / `!hasFilePartition`). Most operators in any tree are non-scan and skip.
133+
injectorsByKind.get(op.getOpStructCase) match {
134+
case Some(injector) if injector.canInject(op) =>
135+
injector.getKey(op) match {
136+
case Some(key) =>
137+
(commonByKey.get(key), partitionByKey.get(key)) match {
138+
case (Some(commonBytes), Some(partitionBytes)) =>
139+
val injectedOp = injector.inject(op, commonBytes, partitionBytes)
140+
// Copy the injected operator's fields to our builder
141+
builder.clear()
142+
builder.mergeFrom(injectedOp)
143+
case _ =>
144+
throw new CometRuntimeException(s"Missing planning data for key: $key")
145+
}
146+
case None =>
147+
}
148+
case _ =>
132149
}
133150

134151
// Recursively process children
@@ -172,6 +189,8 @@ private[comet] object IcebergPlanDataInjector extends PlanDataInjector {
172189
}
173190
})
174191

192+
override val opStructCase: Operator.OpStructCase = Operator.OpStructCase.ICEBERG_SCAN
193+
175194
override def canInject(op: Operator): Boolean =
176195
op.hasIcebergScan &&
177196
op.getIcebergScan.getFileScanTasksCount == 0 &&
@@ -211,6 +230,8 @@ private[comet] object IcebergPlanDataInjector extends PlanDataInjector {
211230
*/
212231
private[comet] object NativeScanPlanDataInjector extends PlanDataInjector {
213232

233+
override val opStructCase: Operator.OpStructCase = Operator.OpStructCase.NATIVE_SCAN
234+
214235
override def canInject(op: Operator): Boolean =
215236
op.hasNativeScan &&
216237
op.getNativeScan.hasCommon &&

0 commit comments

Comments
 (0)