Skip to content

Commit b1ab074

Browse files
authored
Merge branch 'develop' into spark-4.1.2-upgrade
Signed-off-by: Robert Kruszewski <[email protected]>
2 parents 5a48f31 + 3513367 commit b1ab074

9 files changed

Lines changed: 485 additions & 151 deletions

File tree

‎java/gradle/libs.versions.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ netty = "4.2.15.Final"
1212
nopen = "1.0.1"
1313
roaringbitmap = "1.6.14"
1414
slf4j = "2.0.18"
15-
spark3 = "3.5.8"
15+
spark3 = "3.5.9"
1616
spark4 = "4.1.2"
1717
s3mock = "5.0.0"
1818
testcontainers-jupiter = "1.21.4"

‎java/vortex-spark/src/main/java/dev/vortex/spark/write/SparkToArrowSchema.java‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@
2323
import org.apache.spark.sql.types.FloatType;
2424
import org.apache.spark.sql.types.IntegerType;
2525
import org.apache.spark.sql.types.LongType;
26-
import org.apache.spark.sql.types.MapType;
2726
import org.apache.spark.sql.types.ShortType;
2827
import org.apache.spark.sql.types.StringType;
2928
import org.apache.spark.sql.types.StructField;
@@ -125,9 +124,6 @@ private static ArrowType convertType(DataType sparkType) {
125124
return new ArrowType.List();
126125
} else if (sparkType instanceof StructType) {
127126
return new ArrowType.Struct();
128-
} else if (sparkType instanceof MapType) {
129-
// Map is represented as List<Struct<key, value>> in Arrow
130-
return new ArrowType.List();
131127
} else {
132128
throw new UnsupportedOperationException("Unsupported Spark type for Arrow conversion: "
133129
+ sparkType.getClass().getName());

‎java/vortex-spark/src/test/java/dev/vortex/spark/write/SparkToArrowSchemaTest.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -173,4 +173,14 @@ void unsupportedTypeIsRejected() {
173173
assertThrows(UnsupportedOperationException.class, () -> SparkToArrowSchema.convert(schema));
174174
assertTrue(e.getMessage().contains("CalendarIntervalType"));
175175
}
176+
177+
@Test
178+
@DisplayName("MapType is rejected rather than converted to a childless List")
179+
void mapTypeIsRejected() {
180+
StructType schema =
181+
new StructType().add("m", DataTypes.createMapType(DataTypes.StringType, DataTypes.IntegerType));
182+
UnsupportedOperationException e =
183+
assertThrows(UnsupportedOperationException.class, () -> SparkToArrowSchema.convert(schema));
184+
assertTrue(e.getMessage().contains("MapType"));
185+
}
176186
}

‎vortex-cuda/benches/dynamic_dispatch_cuda.rs‎

Lines changed: 47 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77

88
mod bench_config;
99

10+
use std::f64::consts::PI;
1011
use std::marker::PhantomData;
1112
use std::mem::size_of;
1213
use std::os::raw::c_void;
@@ -39,6 +40,7 @@ use vortex::dtype::NativePType;
3940
use vortex::dtype::PType;
4041
use vortex::encodings::alp::ALP;
4142
use vortex::encodings::alp::ALPArrayExt;
43+
use vortex::encodings::alp::ALPArrayOwnedExt;
4244
use vortex::encodings::alp::ALPArraySlotsExt;
4345
use vortex::encodings::alp::ALPFloat;
4446
use vortex::encodings::alp::Exponents;
@@ -745,40 +747,50 @@ fn bench_alp_for_bitpacked_f64(c: &mut Criterion) {
745747
for (len, len_str) in BENCH_SIZES {
746748
group.throughput(Throughput::Bytes((len * size_of::<f64>()) as u64));
747749

748-
// Generate f64 values that ALP-encode without patches.
749-
let floats: Vec<f64> = (0..*len)
750-
.map(|i| <f64 as ALPFloat>::decode_single(10 + (i as i64 % 64), exponents))
751-
.collect();
752-
let float_prim = PrimitiveArray::new(Buffer::from(floats), NonNullable);
753-
754-
// Encode: ALP → FoR → BitPacked
755-
let alp =
756-
alp_encode(float_prim.as_view(), Some(exponents), &mut ctx).vortex_expect("alp_encode");
757-
assert!(alp.patches().is_none());
758-
let for_arr = FoRData::encode(
759-
alp.encoded()
760-
.clone()
761-
.execute::<PrimitiveArray>(&mut ctx)
762-
.vortex_expect("to primitive"),
763-
&mut ctx,
764-
)
765-
.vortex_expect("for encode");
766-
let bp = BitPackedData::encode(for_arr.encoded(), bit_width, &mut ctx)
767-
.vortex_expect("bitpack encode");
768-
769-
let tree = ALP::new(
770-
FoR::try_new(bp.into_array(), for_arr.reference_scalar().clone())
771-
.vortex_expect("for_new")
772-
.into_array(),
773-
exponents,
774-
None,
775-
);
776-
let array = tree.into_array();
750+
for (patch_interval, benchmark_name) in [
751+
(None, "cuda/alp_for_bp_6bw_f64/dispatch_f64"),
752+
(
753+
Some(100),
754+
"cuda/alp_for_bp_6bw_f64_1pct_patches/dispatch_f64",
755+
),
756+
] {
757+
let floats: Vec<f64> = (0..*len)
758+
.map(|i| {
759+
if patch_interval.is_some_and(|interval| i % interval == 0) {
760+
PI
761+
} else {
762+
<f64 as ALPFloat>::decode_single(10 + (i as i64 % 64), exponents)
763+
}
764+
})
765+
.collect();
766+
let float_prim = PrimitiveArray::new(Buffer::from(floats), NonNullable);
767+
768+
// Encode: ALP → FoR → BitPacked, preserving ALP's exception patches.
769+
let alp = alp_encode(float_prim.as_view(), Some(exponents), &mut ctx)
770+
.vortex_expect("alp_encode");
771+
assert_eq!(alp.patches().is_some(), patch_interval.is_some());
772+
let (encoded, alp_exponents, patches) = alp.into_parts();
773+
let for_arr = FoRData::encode(
774+
encoded
775+
.execute::<PrimitiveArray>(&mut ctx)
776+
.vortex_expect("to primitive"),
777+
&mut ctx,
778+
)
779+
.vortex_expect("for encode");
780+
let bp = BitPackedData::encode(for_arr.encoded(), bit_width, &mut ctx)
781+
.vortex_expect("bitpack encode");
782+
assert!(bp.patches().is_none(), "expected only ALP patches");
783+
784+
let tree = ALP::new(
785+
FoR::try_new(bp.into_array(), for_arr.reference_scalar().clone())
786+
.vortex_expect("for_new")
787+
.into_array(),
788+
alp_exponents,
789+
patches,
790+
);
791+
let array = tree.into_array();
777792

778-
group.bench_with_input(
779-
BenchmarkId::new("cuda/alp_for_bp_6bw_f64/dispatch_f64", len_str),
780-
len,
781-
|b, &n| {
793+
group.bench_with_input(BenchmarkId::new(benchmark_name, len_str), len, |b, &n| {
782794
let mut cuda_ctx =
783795
CudaSession::create_execution_ctx(&cuda_session()).vortex_expect("ctx");
784796

@@ -791,8 +803,8 @@ fn bench_alp_for_bitpacked_f64(c: &mut Criterion) {
791803
}
792804
total_time
793805
});
794-
},
795-
);
806+
});
807+
}
796808
}
797809

798810
group.finish();

‎vortex-cuda/kernels/src/dynamic_dispatch.cu‎

Lines changed: 64 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -117,12 +117,8 @@ __shared__ uint64_t runend_cursors[BLOCK_SIZE];
117117
// ═══════════════════════════════════════════════════════════════════════════
118118

119119
/// Apply one scalar operation to N values in registers.
120-
///
121-
/// `abs_pos` is the absolute output position of the first value to process.
122-
/// It is used by scalar operations that apply patches, e.g. ALP.
123120
template <typename T, uint32_t N>
124-
__device__ inline void
125-
scalar_op(T *values, const struct ScalarOp &op, char *__restrict smem, uint64_t abs_pos = 0) {
121+
__device__ inline void scalar_op(T *values, const struct ScalarOp &op, char *__restrict smem) {
126122
switch (op.op_code) {
127123
case ScalarOp::FOR: {
128124
const T ref = static_cast<T>(op.params.frame_of_ref.reference);
@@ -165,30 +161,9 @@ scalar_op(T *values, const struct ScalarOp &op, char *__restrict smem, uint64_t
165161
values[i] = static_cast<T>(__double_as_longlong(r));
166162
}
167163
}
168-
// Apply ALP patches: override positions whose float value couldn't
169-
// be reconstructed through the ALP encode/decode cycle.
170-
// Per-value cursor — with a slice offset, a tile's N values can
171-
// straddle two FL chunks, so each value needs its own lookup.
172-
if (op.params.alp.patches_ptr != 0) {
173-
const auto &patches = *reinterpret_cast<const GPUPatches *>(op.params.alp.patches_ptr);
174-
const uint32_t chunk_start = patches.offset / FL_CHUNK;
175-
#pragma unroll
176-
for (uint32_t i = 0; i < N; ++i) {
177-
uint64_t my_pos = (N > 1) ? abs_pos + i * blockDim.x + threadIdx.x : abs_pos;
178-
uint64_t orig = my_pos + patches.offset;
179-
uint32_t chunk = static_cast<uint32_t>(orig / FL_CHUNK) - chunk_start;
180-
uint32_t within = static_cast<uint32_t>(orig % FL_CHUNK);
181-
PatchesCursor<T> cursor(patches, chunk, 0, 1);
182-
auto patch = cursor.next();
183-
while (patch.index != FL_CHUNK) {
184-
if (patch.index == within) {
185-
values[i] = patch.value;
186-
break;
187-
}
188-
patch = cursor.next();
189-
}
190-
}
191-
}
164+
// ALP patches are scattered cooperatively after the stage has decoded.
165+
// Looking up patches here would restart a chunk cursor for every value,
166+
// turning sparse exception handling into O(values * patches).
192167
break;
193168
}
194169
case ScalarOp::DICT: {
@@ -225,6 +200,58 @@ scatter_patches_chunk(const GPUPatches &patches, T *__restrict out, uint32_t chu
225200
}
226201
}
227202

203+
/// Scatter patches overlapping a logical output range.
204+
///
205+
/// `logical_start` and `range_len` are relative to the sliced array represented by
206+
/// `patches`. Patch indices are stored in the coordinate space of the original array,
207+
/// so add `patches.offset` while locating chunks and subtract it when addressing `out`.
208+
/// Every thread cooperates on each overlapping FastLanes chunk.
209+
template <typename T>
210+
__device__ inline void scatter_patches_range(const GPUPatches &patches,
211+
T *__restrict out,
212+
uint64_t logical_start,
213+
uint32_t range_len) {
214+
if (range_len == 0) {
215+
return;
216+
}
217+
218+
const uint64_t original_start = logical_start + patches.offset;
219+
const uint64_t original_end = original_start + range_len;
220+
const uint32_t patch_chunk_start = patches.offset / FL_CHUNK;
221+
const uint32_t first_chunk = static_cast<uint32_t>(original_start / FL_CHUNK);
222+
const uint32_t last_chunk = static_cast<uint32_t>((original_end - 1) / FL_CHUNK);
223+
224+
for (uint32_t original_chunk = first_chunk; original_chunk <= last_chunk; ++original_chunk) {
225+
const uint32_t chunk = original_chunk - patch_chunk_start;
226+
PatchesCursor<T> cursor(patches, chunk, threadIdx.x, blockDim.x);
227+
auto patch = cursor.next();
228+
while (patch.index != FL_CHUNK) {
229+
const uint64_t original_pos = static_cast<uint64_t>(original_chunk) * FL_CHUNK + patch.index;
230+
if (original_pos >= original_start && original_pos < original_end) {
231+
out[original_pos - patches.offset] = patch.value;
232+
}
233+
patch = cursor.next();
234+
}
235+
}
236+
}
237+
238+
/// Apply patch payloads attached to scalar operations after their stage has decoded.
239+
template <typename T>
240+
__device__ inline void
241+
scatter_scalar_patches(const Stage &stage, T *__restrict out, uint64_t logical_start, uint32_t range_len) {
242+
for (uint8_t op_idx = 0; op_idx < stage.num_scalar_ops; ++op_idx) {
243+
const auto &op = stage.scalar_ops[op_idx];
244+
if (op.op_code == ScalarOp::ALP && op.params.alp.patches_ptr != 0) {
245+
const auto &patches = *reinterpret_cast<const GPUPatches *>(op.params.alp.patches_ptr);
246+
// All ordinary writes must complete before exception values overwrite them.
247+
__syncthreads();
248+
scatter_patches_range<T>(patches, out, logical_start, range_len);
249+
// A later stage may consume this patched shared-memory output.
250+
__syncthreads();
251+
}
252+
}
253+
}
254+
228255
// ═══════════════════════════════════════════════════════════════════════════
229256
// Source ops
230257
// ═══════════════════════════════════════════════════════════════════════════
@@ -491,7 +518,7 @@ __device__ void execute_output_stage(T *__restrict output,
491518
smem);
492519

493520
for (uint8_t op = 0; op < stage.num_scalar_ops; ++op) {
494-
scalar_op<T, VALUES_PER_TILE>(values, stage.scalar_ops[op], smem, tile_start);
521+
scalar_op<T, VALUES_PER_TILE>(values, stage.scalar_ops[op], smem);
495522
}
496523

497524
#pragma unroll
@@ -513,7 +540,7 @@ __device__ void execute_output_stage(T *__restrict output,
513540
source_op<T, 1>(&val, src, raw_input, ptype, smem_src, i, gpos, smem);
514541

515542
for (uint8_t op = 0; op < stage.num_scalar_ops; ++op) {
516-
scalar_op<T, 1>(&val, stage.scalar_ops[op], smem, gpos);
543+
scalar_op<T, 1>(&val, stage.scalar_ops[op], smem);
517544
}
518545
__stcs(&output[gpos], val);
519546
}
@@ -525,6 +552,8 @@ __device__ void execute_output_stage(T *__restrict output,
525552
}
526553
elem_idx += chunk_len;
527554
}
555+
556+
scatter_scalar_patches<T>(stage, output, block_start, block_len);
528557
}
529558

530559
// ═══════════════════════════════════════════════════════════════════════════
@@ -559,7 +588,7 @@ __device__ void execute_input_stage(const Stage &stage, char *__restrict smem) {
559588
for (uint32_t elem_idx = threadIdx.x; elem_idx < stage.len; elem_idx += blockDim.x) {
560589
T val = smem_out[elem_idx];
561590
for (uint8_t op = 0; op < stage.num_scalar_ops; ++op) {
562-
scalar_op<T, 1>(&val, stage.scalar_ops[op], smem, elem_idx);
591+
scalar_op<T, 1>(&val, stage.scalar_ops[op], smem);
563592
}
564593
smem_out[elem_idx] = val;
565594
}
@@ -583,14 +612,16 @@ __device__ void execute_input_stage(const Stage &stage, char *__restrict smem) {
583612
T val;
584613
source_op<T, 1>(&val, src, raw_input, stage.source_ptype, nullptr, 0, elem_idx, smem);
585614
for (uint8_t op = 0; op < stage.num_scalar_ops; ++op) {
586-
scalar_op<T, 1>(&val, stage.scalar_ops[op], smem, elem_idx);
615+
scalar_op<T, 1>(&val, stage.scalar_ops[op], smem);
587616
}
588617
smem_out[elem_idx] = val;
589618
}
590619
// Write barrier: smem region is fully populated for subsequent
591620
// stages to read.
592621
__syncthreads();
593622
}
623+
624+
scatter_scalar_patches<T>(stage, smem_out, 0, stage.len);
594625
}
595626

596627
// ═══════════════════════════════════════════════════════════════════════════

‎vortex-datafusion/src/convert/schema.rs‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,10 @@ pub fn calculate_physical_schema(
5656
})
5757
.collect::<DFResult<Vec<_>>>()?;
5858

59-
Ok(Schema::new(fields))
59+
Ok(Schema::new_with_metadata(
60+
fields,
61+
reference_logical_schema.metadata().clone(),
62+
))
6063
}
6164

6265
/// Calculate the physical Arrow type for a field, preferring the logical type when the
@@ -246,6 +249,32 @@ mod tests {
246249
);
247250
}
248251

252+
#[test]
253+
fn test_schema_metadata_preserved() -> DFResult<()> {
254+
let logical_schema = Schema::new_with_metadata(
255+
vec![Field::new("col", DataType::Int32, false)],
256+
[("table".to_string(), "metadata".to_string())]
257+
.into_iter()
258+
.collect(),
259+
);
260+
let dtype = DType::Struct(
261+
StructFields::from_iter([(
262+
"col",
263+
DType::Primitive(PType::I32, Nullability::NonNullable),
264+
)]),
265+
Nullability::NonNullable,
266+
);
267+
268+
let physical_schema =
269+
calculate_physical_schema(&dtype, &logical_schema, &ArrowSession::default())?;
270+
271+
assert_eq!(
272+
physical_schema.metadata().get("table"),
273+
Some(&"metadata".to_string())
274+
);
275+
Ok(())
276+
}
277+
249278
#[test]
250279
fn test_utf8_variants_preserved() {
251280
// Non-view string types become view types after roundtrip through DType,

0 commit comments

Comments
 (0)