Skip to content

Commit 0c86a6c

Browse files
Merge branch 'develop' into ji/laughing-knuth-rq5q8u
2 parents b3d0f76 + 3039ba1 commit 0c86a6c

89 files changed

Lines changed: 5537 additions & 1243 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/ci.yml‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -98,13 +98,14 @@ jobs:
9898
# benchmarks-database fact row. The golden vectors are a frozen artifact (see the note inside
9999
# the JSON); this job fails if a change to scripts/_measurement_id.py drifts the hash, which
100100
# would break ON CONFLICT idempotency against the rows already in production.
101-
- name: Pytest - measurement_id, benchmark reporting and ingestion
101+
- name: Pytest - measurement_id, benchmark reporting, ingestion and CodSpeed shards
102102
if: "!cancelled()"
103103
run: |
104104
uv run --no-project --with pytest --with pandas --with numpy --with orjson \
105105
--with tabulate --with xxhash \
106106
pytest scripts/tests/test_measurement_id.py \
107-
scripts/tests/test_benchmark_reporting.py scripts/tests/test_post_ingest.py
107+
scripts/tests/test_benchmark_reporting.py scripts/tests/test_post_ingest.py \
108+
scripts/tests/test_codspeed_shards.py
108109
109110
python-test:
110111
name: "Python (test)"

‎.github/workflows/codspeed.yml‎

Lines changed: 152 additions & 117 deletions
Large diffs are not rendered by default.

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎docs/specs/versioning/using-editions.md‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,21 @@ currently enables `core2026.08.3`. Calling `enable_edition` replaces the enabled
3535
same family. Set the selection before starting the write, because that is when the writer captures
3636
the permitted wire IDs.
3737

38+
The Java and Python bindings select an edition by its name. In Java, enable the edition on the
39+
session before you open a writer:
40+
41+
```java
42+
Session session = Session.create();
43+
session.enableEdition("core2025.10.0");
44+
```
45+
46+
In Python, set the edition on the write options, or pass it to `write`:
47+
48+
```python
49+
options = vortex.io.VortexWriteOptions.default().with_edition("core2025.10.0")
50+
vortex.io.write(array, "data.vortex", edition="core2025.10.0")
51+
```
52+
3853
An edition declaration describes permitted formats, but registering it does not install the code to
3954
read or write those formats. When constructing a session without the defaults, register the required
4055
implementations and declarations, then enable the target editions. See

‎encodings/alp/src/alp/compute/between.rs‎

Lines changed: 165 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ use std::fmt::Debug;
66
use num_traits::Bounded;
77
use vortex_array::ArrayRef;
88
use vortex_array::ArrayView;
9+
use vortex_array::ExecutionCtx;
910
use vortex_array::IntoArray;
1011
use vortex_array::arrays::ConstantArray;
1112
use vortex_array::builtins::ArrayBuiltins;
@@ -14,6 +15,7 @@ use vortex_array::dtype::NativePType;
1415
use vortex_array::dtype::Nullability;
1516
use vortex_array::scalar::PValue;
1617
use vortex_array::scalar::Scalar;
18+
use vortex_array::scalar_fn::fns::between::BetweenKernel;
1719
use vortex_array::scalar_fn::fns::between::BetweenOptions;
1820
use vortex_array::scalar_fn::fns::between::BetweenReduce;
1921
use vortex_array::scalar_fn::fns::between::StrictComparison;
@@ -24,8 +26,13 @@ use crate::ALPFloat;
2426
use crate::Exponents;
2527
use crate::alp::array::ALPArrayExt;
2628
use crate::alp::array::ALPArraySlotsExt;
29+
use crate::alp::compute::predicate::apply_patch_predicate;
30+
use crate::alp::compute::predicate::constant_predicate;
31+
use crate::alp::compute::predicate::encode_strictly_above;
32+
use crate::alp::compute::predicate::encode_strictly_below;
2733
use crate::match_each_alp_float_ptype;
2834

35+
/// Rewrites a between over a patch-free array into a between over its encoded integers.
2936
impl BetweenReduce for ALP {
3037
fn between(
3138
array: ArrayView<'_, Self>,
@@ -37,6 +44,7 @@ impl BetweenReduce for ALP {
3744
return Ok(None);
3845
};
3946

47+
// Patched rows need their exact values, which the execute kernel below supplies.
4048
if array.patches().is_some() {
4149
return Ok(None);
4250
}
@@ -56,6 +64,42 @@ impl BetweenReduce for ALP {
5664
}
5765
}
5866

67+
/// Answers a between over a patched array from its encoded integers, then re-evaluates the
68+
/// patched rows on their exact values.
69+
impl BetweenKernel for ALP {
70+
fn between(
71+
array: ArrayView<'_, Self>,
72+
lower: &ArrayRef,
73+
upper: &ArrayRef,
74+
options: &BetweenOptions,
75+
ctx: &mut ExecutionCtx,
76+
) -> VortexResult<Option<ArrayRef>> {
77+
let Some(patches) = array.patches() else {
78+
return Ok(None);
79+
};
80+
let (Some(lower), Some(upper)) = (lower.as_constant(), upper.as_constant()) else {
81+
return Ok(None);
82+
};
83+
84+
let nullability =
85+
array.dtype().nullability() | lower.dtype().nullability() | upper.dtype().nullability();
86+
match_each_alp_float_ptype!(array.dtype().as_ptype(), |F| {
87+
let lower = F::try_from(lower)?;
88+
let upper = F::try_from(upper)?;
89+
let encoded = between_impl::<F>(array, lower, upper, nullability, options)?;
90+
let lower_fn = options.lower_strict.to_predicate::<F>();
91+
let upper_fn = options.upper_strict.to_predicate::<F>();
92+
apply_patch_predicate(
93+
encoded,
94+
&patches,
95+
|value: F| lower_fn(lower, value) & upper_fn(value, upper),
96+
ctx,
97+
)
98+
})
99+
.map(Some)
100+
}
101+
}
102+
59103
fn between_impl<T: NativePType + ALPFloat>(
60104
array: ArrayView<'_, ALP>,
61105
lower: T,
@@ -69,23 +113,22 @@ where
69113
{
70114
let exponents = array.exponents();
71115

72-
// There are always compared
73-
// the below bound is `value {< | <=} x`, either value encodes into the ALPInt domain
74-
// in which case we can leave the comparison unchanged `enc(value) {< | <=} x` or it doesn't
75-
// and we encode into value below enc_below(value) < value < x, in which case the comparison
76-
// becomes enc(value) < x. See `alp_scalar_compare` for more details.
77-
// note that if the value doesn't encode than value != x, so must use strict comparison.
116+
// The lower bound is `value {< | <=} x`. Either `value` encodes into the ALPInt domain, in
117+
// which case the comparison is unchanged as `enc(value) {< | <=} x`, or it does not and
118+
// `enc_below(value) < value < x`, in which case the comparison becomes `enc(value) < x`. See
119+
// `alp_scalar_compare` for more details. Note that if `value` does not encode then
120+
// `value != x`, so the comparison must be strict.
78121
let Some((lower_enc, lower_strict)) =
79122
encode_lower_bound::<T>(lower, exponents, options.lower_strict)
80123
else {
81-
return Ok(ConstantArray::new(Scalar::bool(false, nullability), array.len()).into_array());
124+
return constant_predicate(array, false, nullability);
82125
};
83126

84-
// the upper value `x { < | <= } value` similarly encodes or `x < value < enc_above(value())`
127+
// The upper bound `x {< | <=} value` similarly encodes, or `x < value < enc_above(value)`.
85128
let Some((upper_enc, upper_strict)) =
86129
encode_upper_bound::<T>(upper, exponents, options.upper_strict)
87130
else {
88-
return Ok(ConstantArray::new(Scalar::bool(false, nullability), array.len()).into_array());
131+
return constant_predicate(array, false, nullability);
89132
};
90133

91134
let options = BetweenOptions {
@@ -113,7 +156,12 @@ fn encode_lower_bound<T: ALPFloat + NativePType>(
113156
Some(
114157
T::encode_single(lower, exponents)
115158
.map(|x| (x, strict))
116-
.unwrap_or_else(|| (T::encode_below(lower, exponents), StrictComparison::Strict)),
159+
.unwrap_or_else(|| {
160+
(
161+
encode_strictly_below(lower, exponents),
162+
StrictComparison::Strict,
163+
)
164+
}),
117165
)
118166
}
119167

@@ -130,23 +178,33 @@ fn encode_upper_bound<T: ALPFloat + NativePType>(
130178
Some(
131179
T::encode_single(upper, exponents)
132180
.map(|x| (x, strict))
133-
.unwrap_or_else(|| (T::encode_above(upper, exponents), StrictComparison::Strict)),
181+
.unwrap_or_else(|| {
182+
(
183+
encode_strictly_above(upper, exponents),
184+
StrictComparison::Strict,
185+
)
186+
}),
134187
)
135188
}
136189

137190
#[cfg(test)]
138191
mod tests {
192+
use std::f32::consts::PI;
139193
use std::sync::LazyLock;
140194

195+
use rstest::rstest;
141196
use vortex_array::IntoArray;
142197
use vortex_array::VortexSessionExecute;
198+
use vortex_array::arrays::BoolArray;
143199
use vortex_array::arrays::ConstantArray;
144200
use vortex_array::arrays::PrimitiveArray;
145201
use vortex_array::assert_arrays_eq;
202+
use vortex_array::builtins::ArrayBuiltins;
146203
use vortex_array::dtype::Nullability;
147204
use vortex_array::scalar::Scalar;
148205
use vortex_array::scalar_fn::fns::between::BetweenOptions;
149206
use vortex_array::scalar_fn::fns::between::StrictComparison;
207+
use vortex_error::VortexResult;
150208
use vortex_session::VortexSession;
151209

152210
use crate::ALPArray;
@@ -263,4 +321,100 @@ mod tests {
263321
assert_between(&encoded, 0.0, f32::INFINITY, &options, true);
264322
assert_between(&encoded, 0.0, f32::NEG_INFINITY, &options, false);
265323
}
324+
325+
#[rstest]
326+
#[case::negative_zero_lower(-0.0f32, 1.0f32, true)]
327+
#[case::negative_zero_upper(-1.0f32, -0.0f32, false)]
328+
#[case::subnormal_lower(f32::from_bits(1), 1.0f32, false)]
329+
#[case::subnormal_upper(-1.0f32, -f32::from_bits(1), false)]
330+
fn bounds_near_zero_use_total_order(
331+
#[case] lower: f32,
332+
#[case] upper: f32,
333+
#[case] expected: bool,
334+
) {
335+
let array = PrimitiveArray::from_iter([0.0f32; 10]);
336+
let encoded =
337+
alp_encode(array.as_view(), None, &mut SESSION.create_execution_ctx()).unwrap();
338+
assert!(encoded.patches().is_none());
339+
340+
let options = BetweenOptions {
341+
lower_strict: StrictComparison::NonStrict,
342+
upper_strict: StrictComparison::NonStrict,
343+
};
344+
assert_between(&encoded, lower, upper, &options, expected);
345+
}
346+
347+
#[test]
348+
fn nullable_constant_answer_keeps_validity() -> VortexResult<()> {
349+
let mut ctx = SESSION.create_execution_ctx();
350+
let array = PrimitiveArray::from_option_iter([Some(1.234f32), None, Some(1.234)]);
351+
let encoded = alp_encode(array.as_view(), None, &mut ctx)?;
352+
assert!(encoded.patches().is_none());
353+
354+
let options = BetweenOptions {
355+
lower_strict: StrictComparison::NonStrict,
356+
upper_strict: StrictComparison::NonStrict,
357+
};
358+
let res = between_impl(
359+
encoded.as_view(),
360+
f32::NAN,
361+
2.0,
362+
Nullability::Nullable,
363+
&options,
364+
)?;
365+
let expected = BoolArray::from_iter([Some(false), None, Some(false)]);
366+
assert_arrays_eq!(res, expected, &mut ctx);
367+
Ok(())
368+
}
369+
370+
#[rstest]
371+
#[case(StrictComparison::NonStrict, StrictComparison::NonStrict)]
372+
#[case(StrictComparison::Strict, StrictComparison::NonStrict)]
373+
#[case(StrictComparison::NonStrict, StrictComparison::Strict)]
374+
#[case(StrictComparison::Strict, StrictComparison::Strict)]
375+
fn patched_matches_primitive(
376+
#[case] lower_strict: StrictComparison,
377+
#[case] upper_strict: StrictComparison,
378+
) -> VortexResult<()> {
379+
let mut ctx = SESSION.create_execution_ctx();
380+
let values: Vec<Option<f32>> = (0..3000)
381+
.map(|i| {
382+
if i % 11 == 0 {
383+
None
384+
} else if i % 97 == 0 {
385+
Some(PI * i as f32)
386+
} else {
387+
Some(i as f32 / 100.0)
388+
}
389+
})
390+
.collect();
391+
let array = PrimitiveArray::from_option_iter(values);
392+
let encoded = alp_encode(array.as_view(), None, &mut ctx)?;
393+
assert!(encoded.patches().is_some());
394+
let options = BetweenOptions {
395+
lower_strict,
396+
upper_strict,
397+
};
398+
399+
for (lower, upper) in [
400+
(10.0f32, 20.0f32),
401+
(PI * 97.0, PI * 1_067.0),
402+
(f32::NEG_INFINITY, 5.0),
403+
(f32::NAN, 5.0),
404+
] {
405+
let lower = ConstantArray::new(lower, array.len()).into_array();
406+
let upper = ConstantArray::new(upper, array.len()).into_array();
407+
let actual = encoded.clone().into_array().between(
408+
lower.clone(),
409+
upper.clone(),
410+
options.clone(),
411+
)?;
412+
let expected = array
413+
.clone()
414+
.into_array()
415+
.between(lower, upper, options.clone())?;
416+
assert_arrays_eq!(actual, expected, &mut ctx);
417+
}
418+
Ok(())
419+
}
266420
}

0 commit comments

Comments
 (0)