Skip to content

Commit 0c141b1

Browse files
connortsui20claude
authored andcommitted
feat(row): add a UTF-8 input column and output sink for row functions
Add `Utf8Column` and `Utf8Sink` to the unstable row function types. `Utf8Column` decodes a `Utf8` column once per batch and hands each row callback a `Utf8View`. The view dereferences to `str`, so a callback calls `str` methods on it directly. It also exposes the string-view length, prefix and inline flag, which a comparison kernel can read before it reads the bytes. The column also accepts a batch constant. `Utf8Sink` builds a `VarBinViewArray` output. It initializes every row to an empty string, so its skipped-row initializer does no work and skip-invalid execution needs no input filter. The decode step validates every view and replaces the view of each null row with an empty view. A dense callback can therefore read the payload of a null row, and `Utf8View::as_str` can skip the UTF-8 check. These types come from the SpiralDB engine, where the `spql.str.*` functions use them. Signed-off-by: Connor Tsui <[email protected]>
1 parent a67cd1b commit 0c141b1

6 files changed

Lines changed: 599 additions & 1 deletion

File tree

‎vortex-array/src/scalar_fn/unstable/row/mod.rs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,10 @@ pub use types::OutputElement;
4040
pub use types::OutputSink;
4141
pub use types::SinkResult;
4242
pub use types::UninitElementSink;
43+
pub use types::Utf8Column;
44+
pub use types::Utf8Sink;
45+
pub use types::Utf8View;
46+
pub use types::Utf8Writer;
4347
pub use types::ViewLen;
4448

4549
mod visitor;

‎vortex-array/src/scalar_fn/unstable/row/types/element/mod.rs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,9 @@
66
//! [`InputElement::Elem`] can borrow from its decoded column. Owned row computations return an
77
//! [`OutputElement`]. Runtime-shaped outputs use an
88
//! [`OutputSink`](crate::scalar_fn::unstable::row::OutputSink).
9+
//!
10+
//! Booleans and primitives implement both traits directly. UTF-8 columns decode through
11+
//! [`Utf8Column`] and build through [`Utf8Sink`](crate::scalar_fn::unstable::row::Utf8Sink).
912
1013
mod bool;
1114

@@ -22,3 +25,7 @@ pub use tuple::ElementTuple;
2225
pub use tuple::IndexedElementTuple;
2326
pub use tuple::batch_const;
2427
pub(in crate::scalar_fn::unstable::row) use tuple::decoded_source;
28+
29+
mod utf8;
30+
pub use utf8::Utf8Column;
31+
pub use utf8::Utf8View;
Lines changed: 363 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,363 @@
1+
// SPDX-License-Identifier: Apache-2.0
2+
// SPDX-FileCopyrightText: Copyright the Vortex contributors
3+
4+
//! UTF-8 input columns for row functions.
5+
//!
6+
//! [`Utf8Column`] decodes a `Utf8` column once per batch and hands each row callback a
7+
//! [`Utf8View`]. A view dereferences to `str`, so a callback can call `str` methods directly. It
8+
//! also exposes the string-view metadata that a comparison kernel can use before it reads the
9+
//! bytes.
10+
11+
use std::ops::Deref;
12+
use std::sync::Arc;
13+
14+
use vortex_buffer::Buffer;
15+
use vortex_buffer::ByteBuffer;
16+
use vortex_error::VortexResult;
17+
use vortex_error::vortex_bail;
18+
use vortex_error::vortex_ensure;
19+
20+
use crate::ArrayRef;
21+
use crate::ExecutionCtx;
22+
use crate::arrays::Constant;
23+
use crate::arrays::VarBinViewArray;
24+
use crate::arrays::varbinview::BinaryView;
25+
use crate::arrays::varbinview::VarBinViewArrayExt as _;
26+
use crate::buffer::BufferHandle;
27+
use crate::dtype::DType;
28+
use crate::scalar::ScalarValue;
29+
use crate::scalar_fn::unstable::row::InputElement;
30+
use crate::scalar_fn::unstable::row::ViewLen;
31+
32+
/// A UTF-8 input element that yields [`Utf8View`] values.
33+
pub struct Utf8Column;
34+
35+
/// One decoded UTF-8 column.
36+
pub struct Utf8Values {
37+
views: Buffer<BinaryView>,
38+
buffers: Arc<[ByteBuffer]>,
39+
}
40+
41+
impl Utf8Values {
42+
/// Build a single-value representation of one batch-constant string.
43+
fn single(value: ByteBuffer) -> Self {
44+
let view = BinaryView::make_view(value.as_slice(), 0, 0);
45+
let buffers: Vec<ByteBuffer> = if view.is_inlined() {
46+
Vec::new()
47+
} else {
48+
vec![value]
49+
};
50+
51+
Self {
52+
views: Buffer::from(vec![view]),
53+
buffers: Arc::from(buffers),
54+
}
55+
}
56+
57+
fn views(&self) -> &[BinaryView] {
58+
self.views.as_slice()
59+
}
60+
}
61+
62+
/// A borrowed UTF-8 column prepared for a row loop.
63+
#[derive(Clone, Copy)]
64+
pub struct Utf8ValuesView<'a> {
65+
views: &'a [BinaryView],
66+
buffers: &'a [ByteBuffer],
67+
}
68+
69+
impl ViewLen for Utf8ValuesView<'_> {
70+
fn len(&self) -> usize {
71+
self.views.len()
72+
}
73+
}
74+
75+
/// One UTF-8 value together with its Vortex string-view representation.
76+
#[derive(Clone, Copy)]
77+
pub struct Utf8View<'a> {
78+
view: &'a BinaryView,
79+
buffers: &'a [ByteBuffer],
80+
}
81+
82+
impl<'a> Utf8View<'a> {
83+
/// Return the raw Vortex string view.
84+
///
85+
/// The view reports the byte length without a read of the value.
86+
pub fn raw_view(&self) -> &BinaryView {
87+
self.view
88+
}
89+
90+
/// Return whether the complete string is stored inside the view.
91+
pub fn is_inlined(&self) -> bool {
92+
self.view.is_inlined()
93+
}
94+
95+
/// Return the string's first four bytes, or the complete string when it is shorter.
96+
pub fn prefix(&self) -> &[u8] {
97+
if self.view.is_inlined() {
98+
let value = self.view.as_inlined().value();
99+
&value[..value.len().min(4)]
100+
} else {
101+
&self.view.as_view().prefix
102+
}
103+
}
104+
105+
/// Return the complete UTF-8 string with the lifetime of the decoded column.
106+
///
107+
/// [`Deref`] borrows from the view instead, so a value that must outlive this view uses this
108+
/// method.
109+
pub fn as_str(&self) -> &'a str {
110+
let bytes = if self.view.is_inlined() {
111+
self.view.as_inlined().value()
112+
} else {
113+
let view = self.view.as_view();
114+
&self.buffers[view.buffer_index as usize][view.as_range()]
115+
};
116+
117+
// SAFETY: the `VarBinViewArray` Utf8 invariant requires every valid view to contain UTF-8.
118+
// `decode_utf8` replaces each null row's view before this value can be constructed.
119+
unsafe { std::str::from_utf8_unchecked(bytes) }
120+
}
121+
}
122+
123+
impl Deref for Utf8View<'_> {
124+
type Target = str;
125+
126+
fn deref(&self) -> &str {
127+
self.as_str()
128+
}
129+
}
130+
131+
impl AsRef<str> for Utf8View<'_> {
132+
fn as_ref(&self) -> &str {
133+
self.as_str()
134+
}
135+
}
136+
137+
/// Decodes `array` into views and data buffers whose every view addresses valid UTF-8.
138+
///
139+
/// The second [`VarBinViewArray::try_new`] is the sanitization step, not a round trip: it
140+
/// validates each valid view and replaces every null row's view with an empty one. The
141+
/// [`InputElement`] implementation relies on that, so a dense callback can read a null row's
142+
/// payload and [`Utf8View::as_str`] can skip the UTF-8 check. Removing it makes those unsafe.
143+
fn decode_utf8(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Utf8Values> {
144+
let array = array.execute::<VarBinViewArray>(ctx)?;
145+
let views = Buffer::<BinaryView>::from_byte_buffer(array.views_handle().try_to_host_sync()?);
146+
let buffers: Arc<[ByteBuffer]> = Arc::from(
147+
array
148+
.data_buffers()
149+
.iter()
150+
.map(BufferHandle::try_to_host_sync)
151+
.collect::<VortexResult<Vec<_>>>()?,
152+
);
153+
154+
let validity = array.varbinview_validity();
155+
let array = VarBinViewArray::try_new(
156+
views,
157+
Arc::clone(&buffers),
158+
array.dtype().clone(),
159+
validity,
160+
ctx,
161+
)?;
162+
let views = Buffer::<BinaryView>::from_byte_buffer(array.views_handle().try_to_host_sync()?);
163+
164+
Ok(Utf8Values { views, buffers })
165+
}
166+
167+
/// Extracts the one string held by a UTF-8 batch constant.
168+
fn decode_constant_utf8(array: &ArrayRef) -> VortexResult<Utf8Values> {
169+
let Some(constant) = array.as_opt::<Constant>() else {
170+
vortex_bail!(
171+
"a Utf8 batch constant must use the Constant encoding, got {}",
172+
array.encoding_id()
173+
);
174+
};
175+
let scalar = constant.scalar();
176+
let Some(ScalarValue::Utf8(value)) = scalar.value() else {
177+
vortex_bail!("a Utf8 batch constant must contain a non-null value, got {scalar}");
178+
};
179+
180+
Ok(Utf8Values::single(value.inner().clone()))
181+
}
182+
183+
fn utf8_view(values: &Utf8Values) -> Utf8ValuesView<'_> {
184+
Utf8ValuesView {
185+
views: values.views(),
186+
buffers: &values.buffers,
187+
}
188+
}
189+
190+
fn value_at<'a>(view: &Utf8ValuesView<'a>, index: usize) -> Utf8View<'a> {
191+
Utf8View {
192+
view: &view.views[index],
193+
buffers: view.buffers,
194+
}
195+
}
196+
197+
/// Read a UTF-8 view without checking its row index.
198+
///
199+
/// # Safety
200+
///
201+
/// `index` must be less than the length reported by `view`.
202+
unsafe fn value_at_unchecked<'a>(view: &Utf8ValuesView<'a>, index: usize) -> Utf8View<'a> {
203+
// SAFETY: forwarded from this function's contract.
204+
let value = unsafe { view.views.get_unchecked(index) };
205+
206+
Utf8View {
207+
view: value,
208+
buffers: view.buffers,
209+
}
210+
}
211+
212+
// SAFETY: `decode_utf8` validates every valid view and replaces null rows with empty views. The
213+
// borrowed view reports that exact stable length. Therefore each view addresses valid UTF-8, and
214+
// null rows are safe for dense callbacks whose outputs the executor masks with the input validity.
215+
unsafe impl InputElement for Utf8Column {
216+
type Column = Utf8Values;
217+
type Constant = Utf8Values;
218+
type View<'a> = Utf8ValuesView<'a>;
219+
type Elem<'a> = Utf8View<'a>;
220+
221+
const DENSE_SAFE: bool = true;
222+
const DECODE_INFALLIBLE: bool = true;
223+
224+
fn validate(dtype: &DType) -> VortexResult<()> {
225+
vortex_ensure!(
226+
matches!(dtype, DType::Utf8(_)),
227+
"expected a Utf8 column, got {dtype}"
228+
);
229+
230+
Ok(())
231+
}
232+
233+
fn decode(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Self::Column> {
234+
decode_utf8(array, ctx)
235+
}
236+
237+
fn decode_constant(array: ArrayRef, _ctx: &mut ExecutionCtx) -> VortexResult<Self::Constant> {
238+
decode_constant_utf8(&array)
239+
}
240+
241+
fn can_decode_null_tolerant(_array: &ArrayRef) -> VortexResult<bool> {
242+
Ok(true)
243+
}
244+
245+
fn get(column: &Self::Column, index: usize) -> Self::Elem<'_> {
246+
value_at(&utf8_view(column), index)
247+
}
248+
249+
fn get_constant(constant: &Self::Constant) -> Self::Elem<'_> {
250+
value_at(&utf8_view(constant), 0)
251+
}
252+
253+
fn view(column: &Self::Column) -> Self::View<'_> {
254+
utf8_view(column)
255+
}
256+
257+
fn get_from_view<'a>(view: &Self::View<'a>, index: usize) -> Self::Elem<'a>
258+
where
259+
Self: 'a,
260+
{
261+
value_at(view, index)
262+
}
263+
264+
unsafe fn get_from_view_unchecked<'a>(view: &Self::View<'a>, index: usize) -> Self::Elem<'a>
265+
where
266+
Self: 'a,
267+
{
268+
// SAFETY: the executor validated `index` against this view's exact view slice length.
269+
unsafe { value_at_unchecked(view, index) }
270+
}
271+
}
272+
273+
#[cfg(test)]
274+
mod tests {
275+
use rstest::rstest;
276+
use vortex_buffer::Buffer;
277+
use vortex_error::VortexResult;
278+
use vortex_session::VortexSession;
279+
280+
use super::Utf8Column;
281+
use crate::IntoArray as _;
282+
use crate::VortexSessionExecute as _;
283+
use crate::arrays::BoolArray;
284+
use crate::arrays::ConstantArray;
285+
use crate::arrays::VarBinViewArray;
286+
use crate::arrays::varbinview::BinaryView;
287+
use crate::buffer::BufferHandle;
288+
use crate::dtype::DType;
289+
use crate::dtype::Nullability;
290+
use crate::scalar::Scalar;
291+
use crate::scalar_fn::unstable::row::InputElement;
292+
use crate::validity::Validity;
293+
294+
#[test]
295+
fn input_reads_inline_referenced_and_view_metadata() -> VortexResult<()> {
296+
let array = VarBinViewArray::from_iter_str(["short", "a referenced string"]);
297+
let mut ctx = VortexSession::empty().create_execution_ctx();
298+
let column = Utf8Column::decode(array.into_array(), &mut ctx)?;
299+
300+
let inline = Utf8Column::get(&column, 0);
301+
assert_eq!(&*inline, "short");
302+
assert!(inline.is_inlined());
303+
assert_eq!(inline.prefix(), b"shor");
304+
305+
let referenced = Utf8Column::get(&column, 1);
306+
assert_eq!(&*referenced, "a referenced string");
307+
assert!(!referenced.is_inlined());
308+
assert_eq!(referenced.prefix(), b"a re");
309+
assert_eq!(referenced.raw_view().len(), 19);
310+
311+
Ok(())
312+
}
313+
314+
#[test]
315+
fn input_sanitizes_unvalidated_null_views() -> VortexResult<()> {
316+
let invalid_view = BinaryView::from(u128::MAX);
317+
let views = BufferHandle::new_host(Buffer::from(vec![invalid_view]).into_byte_buffer());
318+
let validity = BoolArray::from_iter([false]).into_array();
319+
let array = VarBinViewArray::new_handle(
320+
views,
321+
Default::default(),
322+
DType::Utf8(Nullability::Nullable),
323+
Validity::Array(validity),
324+
);
325+
let mut ctx = VortexSession::empty().create_execution_ctx();
326+
let column = Utf8Column::decode(array.into_array(), &mut ctx)?;
327+
328+
assert_eq!(&*Utf8Column::get(&column, 0), "");
329+
330+
Ok(())
331+
}
332+
333+
#[test]
334+
fn input_rejects_unvalidated_utf8() {
335+
let invalid_view = BinaryView::make_view(&[0xff], 0, 0);
336+
let views = BufferHandle::new_host(Buffer::from(vec![invalid_view]).into_byte_buffer());
337+
let array = VarBinViewArray::new_handle(
338+
views,
339+
Default::default(),
340+
DType::Utf8(Nullability::NonNullable),
341+
Validity::NonNullable,
342+
);
343+
let mut ctx = VortexSession::empty().create_execution_ctx();
344+
345+
assert!(Utf8Column::decode(array.into_array(), &mut ctx).is_err());
346+
}
347+
348+
#[rstest]
349+
#[case("inlined")]
350+
#[case("a referenced batch constant")]
351+
fn constant_decodes_inlined_and_referenced_values(#[case] value: &str) -> VortexResult<()> {
352+
let array = ConstantArray::new(Scalar::from(value), 8).into_array();
353+
let mut ctx = VortexSession::empty().create_execution_ctx();
354+
355+
let constant = Utf8Column::decode_constant(array, &mut ctx)?;
356+
let view = Utf8Column::get_constant(&constant);
357+
358+
assert_eq!(&*view, value);
359+
assert_eq!(view.raw_view().len() as usize, value.len());
360+
361+
Ok(())
362+
}
363+
}

0 commit comments

Comments
 (0)