Skip to content

Commit aaed723

Browse files
robert3005claudejoseph-isaacs
authored
Add mechamism to lock edition definition and how to evolve objects in editions (#9660)
Record frozen editions in the repository and add ser/de plugin points to evolve encodings --------- Signed-off-by: "Joe Isaacs" <[email protected]> Signed-off-by: Robert Kruszewski <[email protected]> Signed-off-by: Joe Isaacs <[email protected]> Co-authored-by: Claude <[email protected]> Co-authored-by: Joe Isaacs <[email protected]>
1 parent b4863a1 commit aaed723

102 files changed

Lines changed: 3544 additions & 1359 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/ci.yml‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -729,6 +729,9 @@ jobs:
729729
with:
730730
sccache: s3
731731
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
732+
with:
733+
# check-editions compares against the merge base, so it needs real history.
734+
fetch-depth: 0
732735
- uses: ./.github/actions/setup-prebuild
733736
with:
734737
enable-sccache: "true"
@@ -738,6 +741,16 @@ jobs:
738741
run: |
739742
cargo run --profile ci -p xtask -- generate-fbs
740743
cargo run --profile ci -p xtask -- generate-proto
744+
- name: "regenerate the edition records"
745+
run: |
746+
cargo run --profile ci -p xtask -- generate-editions
747+
- name: "check frozen edition records never change"
748+
# Independent of the regeneration above: a stale record must not mask a frozen one
749+
# being edited, nor the other way round.
750+
if: "!cancelled()"
751+
run: |
752+
BASE="${{ github.event.pull_request.base.sha || 'HEAD^' }}"
753+
cargo run --profile ci -p xtask -- check-editions --base "$BASE"
741754
- name: "regenerate FFI header file"
742755
run: |
743756
cargo +$NIGHTLY_TOOLCHAIN build --profile ci -p vortex-ffi

‎Cargo.lock‎

Lines changed: 52 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,6 +171,7 @@ geo-types = "0.7.19"
171171
geoarrow = "0.8.0"
172172
geoarrow-cast = "0.8.0"
173173
get_dir = "0.5.0"
174+
git2 = { version = "0.21", default-features = false }
174175
glob = "0.3.2"
175176
goldenfile = "1"
176177
half = { version = "2.7.1", features = ["std", "num-traits"] }
@@ -276,6 +277,7 @@ thiserror = "2.0.3"
276277
tokio = { version = "1.52" }
277278
tokio-stream = "0.1.17"
278279
tokio-util = "0.7.17"
280+
toml = "0.9"
279281
tpchgen = "3.0.0"
280282
tpchgen-arrow = "3.0.0"
281283
tracing = { version = "0.1.41", default-features = false }

‎benchmarks/compress-bench/Cargo.toml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,11 @@ vortex-cuda = { workspace = true, optional = true }
4141
[features]
4242
cuda = ["dep:tempfile", "dep:vortex-cuda"]
4343
lance = ["dep:lance-bench"]
44-
unstable_encodings = ["vortex/unstable_encodings", "vortex-cuda?/unstable_encodings"]
44+
unstable_encodings = [
45+
"vortex/unstable_encodings",
46+
"vortex-bench/unstable_encodings",
47+
"vortex-cuda?/unstable_encodings",
48+
]
4549

4650
[[bin]]
4751
name = "compress-bench"

‎benchmarks/compress-bench/src/gpu/vortex.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ use vortex::layout::scan::split_by::SplitBy;
3232
use vortex_arrow::ArrowSessionExt;
3333
use vortex_bench::Format;
3434
use vortex_bench::SESSION;
35+
use vortex_bench::benchmark_write_options;
3536
use vortex_bench::compress::Compressor;
3637
use vortex_bench::conversions::parquet_to_vortex_chunks_with_batch_size;
3738
use vortex_cuda::CanonicalCudaExt;
@@ -95,8 +96,7 @@ impl Compressor for GpuVortexCompressor {
9596
.only_cuda_compatible()
9697
.build(),
9798
)));
98-
SESSION
99-
.write_options()
99+
benchmark_write_options(SESSION.write_options())
100100
.with_strategy(strategy)
101101
.write(&mut output, uncompressed.into_array().to_array_stream())
102102
.await?;

‎benchmarks/compress-bench/src/vortex.rs‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ use vortex::file::WriteOptionsSessionExt;
2121
use vortex_arrow::ArrowSessionExt;
2222
use vortex_bench::Format;
2323
use vortex_bench::SESSION;
24+
use vortex_bench::benchmark_write_options;
2425
use vortex_bench::compress::Compressor;
2526
use vortex_bench::compress::read_projection;
2627
use vortex_bench::conversions::parquet_to_vortex_chunks;
@@ -41,8 +42,7 @@ impl Compressor for VortexCompressor {
4142
let mut buf = Vec::new();
4243
let start = Instant::now();
4344
let mut cursor = Cursor::new(&mut buf);
44-
SESSION
45-
.write_options()
45+
benchmark_write_options(SESSION.write_options())
4646
.write(&mut cursor, uncompressed.into_array().to_array_stream())
4747
.await?;
4848
let elapsed = start.elapsed();
@@ -55,8 +55,7 @@ impl Compressor for VortexCompressor {
5555
let uncompressed = parquet_to_vortex_chunks(parquet_path.to_path_buf()).await?;
5656
let mut buf = Vec::new();
5757
let mut cursor = Cursor::new(&mut buf);
58-
SESSION
59-
.write_options()
58+
benchmark_write_options(SESSION.write_options())
6059
.write(&mut cursor, uncompressed.into_array().to_array_stream())
6160
.await?;
6261

‎benchmarks/datafusion-bench/Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,7 @@ custom-labels = { workspace = true }
5858

5959
[features]
6060
cuda = ["dep:vortex-cuda"]
61-
unstable_encodings = ["vortex/unstable_encodings"]
61+
unstable_encodings = ["vortex/unstable_encodings", "vortex-bench/unstable_encodings"]
6262

6363
[lints]
6464
workspace = true

‎benchmarks/duckdb-bench/Cargo.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ vortex-duckdb = { workspace = true }
3737

3838
[features]
3939
cuda = ["dep:vortex-cuda"]
40-
unstable_encodings = ["vortex/unstable_encodings"]
40+
unstable_encodings = ["vortex/unstable_encodings", "vortex-bench/unstable_encodings"]
4141

4242
[lints]
4343
workspace = true

‎benchmarks/string-bench/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ vortex-onpair = { workspace = true }
3535
[features]
3636
unstable_encodings = [
3737
"vortex/unstable_encodings",
38+
"vortex-bench/unstable_encodings",
3839
"vortex-btrblocks/unstable_encodings",
3940
]
4041

‎benchmarks/string-bench/src/serialized.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ use vortex::file::WriteStrategyBuilder;
3838
use vortex::layout::LayoutStrategy;
3939
use vortex::session::VortexSession;
4040
use vortex_bench::Format;
41+
use vortex_bench::benchmark_write_options;
4142
use vortex_bench::measurements::CustomUnitMeasurement;
4243
use vortex_btrblocks::SchemeExt;
4344
use vortex_btrblocks::SchemeId;
@@ -191,8 +192,7 @@ async fn write_serialized_file(
191192
let mut buf = Vec::new();
192193
{
193194
let mut cursor = Cursor::new(&mut buf);
194-
session
195-
.write_options()
195+
benchmark_write_options(session.write_options())
196196
.with_strategy(Arc::clone(strategy))
197197
.write(&mut cursor, input.to_array_stream())
198198
.await?;

0 commit comments

Comments
 (0)