|
| 1 | +// SPDX-License-Identifier: Apache-2.0 |
| 2 | +// SPDX-FileCopyrightText: Copyright the Vortex contributors |
| 3 | + |
| 4 | +//! Executes row kernels that return one independent owned value per row. |
| 5 | +//! |
| 6 | +//! [`execute_owned`] decodes inputs once, prepares constant state, writes into spare vector |
| 7 | +//! capacity, and reduces compact failure evidence without putting error construction in the hot |
| 8 | +//! loop. [`execute_owned_infallible`] removes that failure path for infallible kernels. |
| 9 | +
|
| 10 | +use std::ops::BitOrAssign; |
| 11 | + |
| 12 | +use vortex_compute::lane_kernels::IndexedSourceExt; |
| 13 | +use vortex_error::VortexResult; |
| 14 | +use vortex_error::vortex_ensure; |
| 15 | + |
| 16 | +use super::RowExecution; |
| 17 | +use crate::ExecutionCtx; |
| 18 | +use crate::scalar_fn::ExecutionArgs; |
| 19 | +use crate::scalar_fn::unstable::row::FailureEvidence; |
| 20 | +use crate::scalar_fn::unstable::row::IndexedElementTuple; |
| 21 | +use crate::scalar_fn::unstable::row::OutputElement; |
| 22 | +use crate::scalar_fn::unstable::row::ViewLen; |
| 23 | +use crate::scalar_fn::unstable::row::visitor::assert_owned_output_needs_no_drop; |
| 24 | + |
| 25 | +/// Zero-sized failure accumulator for infallible owned visits. |
| 26 | +#[derive(Clone, Copy, Default)] |
| 27 | +struct NoFailure; |
| 28 | + |
| 29 | +impl BitOrAssign for NoFailure { |
| 30 | + fn bitor_assign(&mut self, _rhs: Self) {} |
| 31 | +} |
| 32 | + |
| 33 | +/// Decode every input column, then store one output per row from an infallible kernel. |
| 34 | +pub(crate) fn execute_owned_infallible<Args, Out, Prepared>( |
| 35 | + args: &dyn ExecutionArgs, |
| 36 | + ctx: &mut ExecutionCtx, |
| 37 | + prepare: impl FnOnce(Args::ConstElems<'_>) -> Prepared, |
| 38 | + apply: impl Fn(&Prepared, Args::Elems<'_>) -> Out, |
| 39 | +) -> VortexResult<RowExecution> |
| 40 | +where |
| 41 | + Args: IndexedElementTuple, |
| 42 | + Out: OutputElement, |
| 43 | +{ |
| 44 | + execute_owned::<Args, Out, Prepared, NoFailure>( |
| 45 | + args, |
| 46 | + ctx, |
| 47 | + prepare, |
| 48 | + move |prepared, args| (apply(prepared, args), NoFailure), |
| 49 | + |_| Ok(()), |
| 50 | + ) |
| 51 | +} |
| 52 | + |
| 53 | +/// Decode every input column, then store outputs and combine per-row failure evidence. |
| 54 | +pub(crate) fn execute_owned<Args, Out, Prepared, Fail>( |
| 55 | + args: &dyn ExecutionArgs, |
| 56 | + ctx: &mut ExecutionCtx, |
| 57 | + prepare: impl FnOnce(Args::ConstElems<'_>) -> Prepared, |
| 58 | + apply: impl Fn(&Prepared, Args::Elems<'_>) -> (Out, Fail), |
| 59 | + finish_failure: impl FnOnce(Fail) -> VortexResult<()>, |
| 60 | +) -> VortexResult<RowExecution> |
| 61 | +where |
| 62 | + Args: IndexedElementTuple, |
| 63 | + Out: OutputElement, |
| 64 | + Fail: FailureEvidence, |
| 65 | +{ |
| 66 | + // The output vector stays at length zero until every slot is initialized so that an unwind |
| 67 | + // abandons partially initialized spare capacity. This no-drop assertion proves that no |
| 68 | + // initialized value requires a destructor to run. |
| 69 | + const { assert_owned_output_needs_no_drop::<Out>() }; |
| 70 | + |
| 71 | + let columns = Args::decode(args, ctx)?; |
| 72 | + let prepared = prepare(Args::const_values(&columns)); |
| 73 | + |
| 74 | + let row_count = args.row_count(); |
| 75 | + let mut values = Vec::<Out>::with_capacity(row_count); |
| 76 | + let output = &mut values.spare_capacity_mut()[..row_count]; |
| 77 | + |
| 78 | + let failure = if let Some(views) = Args::views_if_no_consts(&columns) { |
| 79 | + // Keep this validation beside the views so LLVM sees their common length here. |
| 80 | + vortex_ensure!( |
| 81 | + Args::ARITY == 0 || views.len() == row_count, |
| 82 | + "a decoded row input does not address exactly {row_count} rows", |
| 83 | + ); |
| 84 | + |
| 85 | + // SAFETY: the tuple length check proved every non-nullary view addresses exactly |
| 86 | + // `row_count` rows immediately above. Nullary tuples do not access an input view. |
| 87 | + let source = unsafe { Args::indexed_source(views, row_count) }; |
| 88 | + |
| 89 | + source.map_checked_into(output, |elements| apply(&prepared, elements)) |
| 90 | + } else { |
| 91 | + // Keep this proof branch-local. Shared validation prevents LLVM from specializing this |
| 92 | + // loop for each batch-constant arrangement, leaving it scalar under multiple CGUs without |
| 93 | + // LTO. |
| 94 | + vortex_ensure!( |
| 95 | + Args::decoded_lens_match(&columns, row_count), |
| 96 | + "a decoded row input does not address exactly {row_count} rows", |
| 97 | + ); |
| 98 | + |
| 99 | + let mut accumulated = Fail::default(); |
| 100 | + |
| 101 | + // Iterate over `output` directly. A `0..row_count` range reuses the address-taken value |
| 102 | + // from the validation error formatter and retains an output bounds check. |
| 103 | + for (index, slot) in output.iter_mut().enumerate() { |
| 104 | + // LLVM unswitches the batch-constant checks in `Args::get` before vectorizing the loop. |
| 105 | + let (value, row_failure) = apply(&prepared, Args::get(&columns, index)); |
| 106 | + |
| 107 | + slot.write(value); |
| 108 | + accumulated |= row_failure; |
| 109 | + } |
| 110 | + |
| 111 | + accumulated |
| 112 | + }; |
| 113 | + |
| 114 | + // SAFETY: normal completion of either execution path initializes `0..row_count` exactly |
| 115 | + // once, and `values` was allocated with at least `row_count` capacity. |
| 116 | + unsafe { values.set_len(row_count) }; |
| 117 | + |
| 118 | + // Defer failures so batch execution can retry with only valid rows. |
| 119 | + match finish_failure(failure) { |
| 120 | + Ok(()) => Ok(RowExecution::Output(Out::build(values))), |
| 121 | + Err(error) => Ok(RowExecution::DeferredError(error)), |
| 122 | + } |
| 123 | +} |
0 commit comments