|
| 1 | +// SPDX-License-Identifier: Apache-2.0 |
| 2 | +// SPDX-FileCopyrightText: Copyright the Vortex contributors |
| 3 | + |
| 4 | +//! UTF-8 input columns for row functions. |
| 5 | +//! |
| 6 | +//! [`Utf8Column`] decodes a `Utf8` column once per batch and hands each row callback a |
| 7 | +//! [`Utf8View`]. A view dereferences to `str`, so a callback can call `str` methods directly. It |
| 8 | +//! also exposes the string-view metadata that a comparison kernel can use before it reads the |
| 9 | +//! bytes. |
| 10 | +
|
| 11 | +use std::ops::Deref; |
| 12 | +use std::sync::Arc; |
| 13 | + |
| 14 | +use vortex_buffer::Buffer; |
| 15 | +use vortex_buffer::ByteBuffer; |
| 16 | +use vortex_error::VortexResult; |
| 17 | +use vortex_error::vortex_bail; |
| 18 | +use vortex_error::vortex_ensure; |
| 19 | + |
| 20 | +use crate::ArrayRef; |
| 21 | +use crate::ExecutionCtx; |
| 22 | +use crate::arrays::Constant; |
| 23 | +use crate::arrays::VarBinViewArray; |
| 24 | +use crate::arrays::varbinview::BinaryView; |
| 25 | +use crate::arrays::varbinview::VarBinViewArrayExt as _; |
| 26 | +use crate::buffer::BufferHandle; |
| 27 | +use crate::dtype::DType; |
| 28 | +use crate::scalar::ScalarValue; |
| 29 | +use crate::scalar_fn::unstable::row::InputElement; |
| 30 | +use crate::scalar_fn::unstable::row::ViewLen; |
| 31 | + |
| 32 | +/// A UTF-8 input element that yields [`Utf8View`] values. |
| 33 | +pub struct Utf8Column; |
| 34 | + |
| 35 | +/// One decoded UTF-8 column. |
| 36 | +pub struct Utf8Values { |
| 37 | + views: Buffer<BinaryView>, |
| 38 | + buffers: Arc<[ByteBuffer]>, |
| 39 | +} |
| 40 | + |
| 41 | +impl Utf8Values { |
| 42 | + /// Build a single-value representation of one batch-constant string. |
| 43 | + fn single(value: ByteBuffer) -> Self { |
| 44 | + let view = BinaryView::make_view(value.as_slice(), 0, 0); |
| 45 | + let buffers: Vec<ByteBuffer> = if view.is_inlined() { |
| 46 | + Vec::new() |
| 47 | + } else { |
| 48 | + vec![value] |
| 49 | + }; |
| 50 | + |
| 51 | + Self { |
| 52 | + views: Buffer::from(vec![view]), |
| 53 | + buffers: Arc::from(buffers), |
| 54 | + } |
| 55 | + } |
| 56 | + |
| 57 | + fn views(&self) -> &[BinaryView] { |
| 58 | + self.views.as_slice() |
| 59 | + } |
| 60 | +} |
| 61 | + |
| 62 | +/// A borrowed UTF-8 column prepared for a row loop. |
| 63 | +#[derive(Clone, Copy)] |
| 64 | +pub struct Utf8ValuesView<'a> { |
| 65 | + views: &'a [BinaryView], |
| 66 | + buffers: &'a [ByteBuffer], |
| 67 | +} |
| 68 | + |
| 69 | +impl ViewLen for Utf8ValuesView<'_> { |
| 70 | + fn len(&self) -> usize { |
| 71 | + self.views.len() |
| 72 | + } |
| 73 | +} |
| 74 | + |
| 75 | +/// One UTF-8 value together with its Vortex string-view representation. |
| 76 | +#[derive(Clone, Copy)] |
| 77 | +pub struct Utf8View<'a> { |
| 78 | + view: &'a BinaryView, |
| 79 | + buffers: &'a [ByteBuffer], |
| 80 | +} |
| 81 | + |
| 82 | +impl<'a> Utf8View<'a> { |
| 83 | + /// Return the raw Vortex string view. |
| 84 | + /// |
| 85 | + /// The view reports the byte length without a read of the value. |
| 86 | + pub fn raw_view(&self) -> &BinaryView { |
| 87 | + self.view |
| 88 | + } |
| 89 | + |
| 90 | + /// Return whether the complete string is stored inside the view. |
| 91 | + pub fn is_inlined(&self) -> bool { |
| 92 | + self.view.is_inlined() |
| 93 | + } |
| 94 | + |
| 95 | + /// Return the string's first four bytes, or the complete string when it is shorter. |
| 96 | + pub fn prefix(&self) -> &[u8] { |
| 97 | + if self.view.is_inlined() { |
| 98 | + let value = self.view.as_inlined().value(); |
| 99 | + &value[..value.len().min(4)] |
| 100 | + } else { |
| 101 | + &self.view.as_view().prefix |
| 102 | + } |
| 103 | + } |
| 104 | + |
| 105 | + /// Return the complete UTF-8 string with the lifetime of the decoded column. |
| 106 | + /// |
| 107 | + /// [`Deref`] borrows from the view instead, so a value that must outlive this view uses this |
| 108 | + /// method. |
| 109 | + pub fn as_str(&self) -> &'a str { |
| 110 | + let bytes = if self.view.is_inlined() { |
| 111 | + self.view.as_inlined().value() |
| 112 | + } else { |
| 113 | + let view = self.view.as_view(); |
| 114 | + &self.buffers[view.buffer_index as usize][view.as_range()] |
| 115 | + }; |
| 116 | + |
| 117 | + // SAFETY: the `VarBinViewArray` Utf8 invariant requires every valid view to contain UTF-8. |
| 118 | + // `decode_utf8` replaces each null row's view before this value can be constructed. |
| 119 | + unsafe { std::str::from_utf8_unchecked(bytes) } |
| 120 | + } |
| 121 | +} |
| 122 | + |
| 123 | +impl Deref for Utf8View<'_> { |
| 124 | + type Target = str; |
| 125 | + |
| 126 | + fn deref(&self) -> &str { |
| 127 | + self.as_str() |
| 128 | + } |
| 129 | +} |
| 130 | + |
| 131 | +impl AsRef<str> for Utf8View<'_> { |
| 132 | + fn as_ref(&self) -> &str { |
| 133 | + self.as_str() |
| 134 | + } |
| 135 | +} |
| 136 | + |
| 137 | +/// Decodes `array` into views and data buffers whose every view addresses valid UTF-8. |
| 138 | +/// |
| 139 | +/// The second [`VarBinViewArray::try_new`] is the sanitization step, not a round trip: it |
| 140 | +/// validates each valid view and replaces every null row's view with an empty one. The |
| 141 | +/// [`InputElement`] implementation relies on that, so a dense callback can read a null row's |
| 142 | +/// payload and [`Utf8View::as_str`] can skip the UTF-8 check. Removing it makes those unsafe. |
| 143 | +fn decode_utf8(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Utf8Values> { |
| 144 | + let array = array.execute::<VarBinViewArray>(ctx)?; |
| 145 | + let views = Buffer::<BinaryView>::from_byte_buffer(array.views_handle().try_to_host_sync()?); |
| 146 | + let buffers: Arc<[ByteBuffer]> = Arc::from( |
| 147 | + array |
| 148 | + .data_buffers() |
| 149 | + .iter() |
| 150 | + .map(BufferHandle::try_to_host_sync) |
| 151 | + .collect::<VortexResult<Vec<_>>>()?, |
| 152 | + ); |
| 153 | + |
| 154 | + let validity = array.varbinview_validity(); |
| 155 | + let array = VarBinViewArray::try_new( |
| 156 | + views, |
| 157 | + Arc::clone(&buffers), |
| 158 | + array.dtype().clone(), |
| 159 | + validity, |
| 160 | + ctx, |
| 161 | + )?; |
| 162 | + let views = Buffer::<BinaryView>::from_byte_buffer(array.views_handle().try_to_host_sync()?); |
| 163 | + |
| 164 | + Ok(Utf8Values { views, buffers }) |
| 165 | +} |
| 166 | + |
| 167 | +/// Extracts the one string held by a UTF-8 batch constant. |
| 168 | +fn decode_constant_utf8(array: &ArrayRef) -> VortexResult<Utf8Values> { |
| 169 | + let Some(constant) = array.as_opt::<Constant>() else { |
| 170 | + vortex_bail!( |
| 171 | + "a Utf8 batch constant must use the Constant encoding, got {}", |
| 172 | + array.encoding_id() |
| 173 | + ); |
| 174 | + }; |
| 175 | + let scalar = constant.scalar(); |
| 176 | + let Some(ScalarValue::Utf8(value)) = scalar.value() else { |
| 177 | + vortex_bail!("a Utf8 batch constant must contain a non-null value, got {scalar}"); |
| 178 | + }; |
| 179 | + |
| 180 | + Ok(Utf8Values::single(value.inner().clone())) |
| 181 | +} |
| 182 | + |
| 183 | +fn utf8_view(values: &Utf8Values) -> Utf8ValuesView<'_> { |
| 184 | + Utf8ValuesView { |
| 185 | + views: values.views(), |
| 186 | + buffers: &values.buffers, |
| 187 | + } |
| 188 | +} |
| 189 | + |
| 190 | +fn value_at<'a>(view: &Utf8ValuesView<'a>, index: usize) -> Utf8View<'a> { |
| 191 | + Utf8View { |
| 192 | + view: &view.views[index], |
| 193 | + buffers: view.buffers, |
| 194 | + } |
| 195 | +} |
| 196 | + |
| 197 | +/// Read a UTF-8 view without checking its row index. |
| 198 | +/// |
| 199 | +/// # Safety |
| 200 | +/// |
| 201 | +/// `index` must be less than the length reported by `view`. |
| 202 | +unsafe fn value_at_unchecked<'a>(view: &Utf8ValuesView<'a>, index: usize) -> Utf8View<'a> { |
| 203 | + // SAFETY: forwarded from this function's contract. |
| 204 | + let value = unsafe { view.views.get_unchecked(index) }; |
| 205 | + |
| 206 | + Utf8View { |
| 207 | + view: value, |
| 208 | + buffers: view.buffers, |
| 209 | + } |
| 210 | +} |
| 211 | + |
| 212 | +// SAFETY: `decode_utf8` validates every valid view and replaces null rows with empty views. The |
| 213 | +// borrowed view reports that exact stable length. Therefore each view addresses valid UTF-8, and |
| 214 | +// null rows are safe for dense callbacks whose outputs the executor masks with the input validity. |
| 215 | +unsafe impl InputElement for Utf8Column { |
| 216 | + type Column = Utf8Values; |
| 217 | + type Constant = Utf8Values; |
| 218 | + type View<'a> = Utf8ValuesView<'a>; |
| 219 | + type Elem<'a> = Utf8View<'a>; |
| 220 | + |
| 221 | + const DENSE_SAFE: bool = true; |
| 222 | + const DECODE_INFALLIBLE: bool = true; |
| 223 | + |
| 224 | + fn validate(dtype: &DType) -> VortexResult<()> { |
| 225 | + vortex_ensure!( |
| 226 | + matches!(dtype, DType::Utf8(_)), |
| 227 | + "expected a Utf8 column, got {dtype}" |
| 228 | + ); |
| 229 | + |
| 230 | + Ok(()) |
| 231 | + } |
| 232 | + |
| 233 | + fn decode(array: ArrayRef, ctx: &mut ExecutionCtx) -> VortexResult<Self::Column> { |
| 234 | + decode_utf8(array, ctx) |
| 235 | + } |
| 236 | + |
| 237 | + fn decode_constant(array: ArrayRef, _ctx: &mut ExecutionCtx) -> VortexResult<Self::Constant> { |
| 238 | + decode_constant_utf8(&array) |
| 239 | + } |
| 240 | + |
| 241 | + fn can_decode_null_tolerant(_array: &ArrayRef) -> VortexResult<bool> { |
| 242 | + Ok(true) |
| 243 | + } |
| 244 | + |
| 245 | + fn get(column: &Self::Column, index: usize) -> Self::Elem<'_> { |
| 246 | + value_at(&utf8_view(column), index) |
| 247 | + } |
| 248 | + |
| 249 | + fn get_constant(constant: &Self::Constant) -> Self::Elem<'_> { |
| 250 | + value_at(&utf8_view(constant), 0) |
| 251 | + } |
| 252 | + |
| 253 | + fn view(column: &Self::Column) -> Self::View<'_> { |
| 254 | + utf8_view(column) |
| 255 | + } |
| 256 | + |
| 257 | + fn get_from_view<'a>(view: &Self::View<'a>, index: usize) -> Self::Elem<'a> |
| 258 | + where |
| 259 | + Self: 'a, |
| 260 | + { |
| 261 | + value_at(view, index) |
| 262 | + } |
| 263 | + |
| 264 | + unsafe fn get_from_view_unchecked<'a>(view: &Self::View<'a>, index: usize) -> Self::Elem<'a> |
| 265 | + where |
| 266 | + Self: 'a, |
| 267 | + { |
| 268 | + // SAFETY: the executor validated `index` against this view's exact view slice length. |
| 269 | + unsafe { value_at_unchecked(view, index) } |
| 270 | + } |
| 271 | +} |
| 272 | + |
| 273 | +#[cfg(test)] |
| 274 | +mod tests { |
| 275 | + use rstest::rstest; |
| 276 | + use vortex_buffer::Buffer; |
| 277 | + use vortex_error::VortexResult; |
| 278 | + use vortex_session::VortexSession; |
| 279 | + |
| 280 | + use super::Utf8Column; |
| 281 | + use crate::IntoArray as _; |
| 282 | + use crate::VortexSessionExecute as _; |
| 283 | + use crate::arrays::BoolArray; |
| 284 | + use crate::arrays::ConstantArray; |
| 285 | + use crate::arrays::VarBinViewArray; |
| 286 | + use crate::arrays::varbinview::BinaryView; |
| 287 | + use crate::buffer::BufferHandle; |
| 288 | + use crate::dtype::DType; |
| 289 | + use crate::dtype::Nullability; |
| 290 | + use crate::scalar::Scalar; |
| 291 | + use crate::scalar_fn::unstable::row::InputElement; |
| 292 | + use crate::validity::Validity; |
| 293 | + |
| 294 | + #[test] |
| 295 | + fn input_reads_inline_referenced_and_view_metadata() -> VortexResult<()> { |
| 296 | + let array = VarBinViewArray::from_iter_str(["short", "a referenced string"]); |
| 297 | + let mut ctx = VortexSession::empty().create_execution_ctx(); |
| 298 | + let column = Utf8Column::decode(array.into_array(), &mut ctx)?; |
| 299 | + |
| 300 | + let inline = Utf8Column::get(&column, 0); |
| 301 | + assert_eq!(&*inline, "short"); |
| 302 | + assert!(inline.is_inlined()); |
| 303 | + assert_eq!(inline.prefix(), b"shor"); |
| 304 | + |
| 305 | + let referenced = Utf8Column::get(&column, 1); |
| 306 | + assert_eq!(&*referenced, "a referenced string"); |
| 307 | + assert!(!referenced.is_inlined()); |
| 308 | + assert_eq!(referenced.prefix(), b"a re"); |
| 309 | + assert_eq!(referenced.raw_view().len(), 19); |
| 310 | + |
| 311 | + Ok(()) |
| 312 | + } |
| 313 | + |
| 314 | + #[test] |
| 315 | + fn input_sanitizes_unvalidated_null_views() -> VortexResult<()> { |
| 316 | + let invalid_view = BinaryView::from(u128::MAX); |
| 317 | + let views = BufferHandle::new_host(Buffer::from(vec![invalid_view]).into_byte_buffer()); |
| 318 | + let validity = BoolArray::from_iter([false]).into_array(); |
| 319 | + let array = VarBinViewArray::new_handle( |
| 320 | + views, |
| 321 | + Default::default(), |
| 322 | + DType::Utf8(Nullability::Nullable), |
| 323 | + Validity::Array(validity), |
| 324 | + ); |
| 325 | + let mut ctx = VortexSession::empty().create_execution_ctx(); |
| 326 | + let column = Utf8Column::decode(array.into_array(), &mut ctx)?; |
| 327 | + |
| 328 | + assert_eq!(&*Utf8Column::get(&column, 0), ""); |
| 329 | + |
| 330 | + Ok(()) |
| 331 | + } |
| 332 | + |
| 333 | + #[test] |
| 334 | + fn input_rejects_unvalidated_utf8() { |
| 335 | + let invalid_view = BinaryView::make_view(&[0xff], 0, 0); |
| 336 | + let views = BufferHandle::new_host(Buffer::from(vec![invalid_view]).into_byte_buffer()); |
| 337 | + let array = VarBinViewArray::new_handle( |
| 338 | + views, |
| 339 | + Default::default(), |
| 340 | + DType::Utf8(Nullability::NonNullable), |
| 341 | + Validity::NonNullable, |
| 342 | + ); |
| 343 | + let mut ctx = VortexSession::empty().create_execution_ctx(); |
| 344 | + |
| 345 | + assert!(Utf8Column::decode(array.into_array(), &mut ctx).is_err()); |
| 346 | + } |
| 347 | + |
| 348 | + #[rstest] |
| 349 | + #[case("inlined")] |
| 350 | + #[case("a referenced batch constant")] |
| 351 | + fn constant_decodes_inlined_and_referenced_values(#[case] value: &str) -> VortexResult<()> { |
| 352 | + let array = ConstantArray::new(Scalar::from(value), 8).into_array(); |
| 353 | + let mut ctx = VortexSession::empty().create_execution_ctx(); |
| 354 | + |
| 355 | + let constant = Utf8Column::decode_constant(array, &mut ctx)?; |
| 356 | + let view = Utf8Column::get_constant(&constant); |
| 357 | + |
| 358 | + assert_eq!(&*view, value); |
| 359 | + assert_eq!(view.raw_view().len() as usize, value.len()); |
| 360 | + |
| 361 | + Ok(()) |
| 362 | + } |
| 363 | +} |
0 commit comments