Skip to content

Commit 3f1fea8

Browse files
committed
Merge remote-tracking branch 'origin/develop' into claude/vortex-aarch-pattern-abstraction-s3javb
Signed-off-by: Joe Isaacs <[email protected]> # Conflicts: # vortex-buffer/Cargo.toml
2 parents 841d0e4 + 8f75dbe commit 3f1fea8

125 files changed

Lines changed: 9350 additions & 587 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/bench.yml‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -171,3 +171,31 @@ jobs:
171171
secrets: inherit
172172
with:
173173
mode: "develop"
174+
175+
sql-vortex:
176+
uses: ./.github/workflows/sql-benchmarks.yml
177+
secrets: inherit
178+
with:
179+
mode: "develop"
180+
benchmark_matrix: |
181+
[
182+
{
183+
"id": "vortex-queries",
184+
"subcommand": "vortex",
185+
"name": "Vortex queries",
186+
"data_formats": ["parquet", "vortex"],
187+
"pr_targets": [
188+
{"engine": "datafusion", "format": "parquet"},
189+
{"engine": "datafusion", "format": "vortex"},
190+
{"engine": "duckdb", "format": "parquet"},
191+
{"engine": "duckdb", "format": "vortex"}
192+
],
193+
"develop_targets": [
194+
{"engine": "datafusion", "format": "parquet"},
195+
{"engine": "datafusion", "format": "vortex"},
196+
{"engine": "duckdb", "format": "parquet"},
197+
{"engine": "duckdb", "format": "vortex"}
198+
],
199+
"iterations": "100"
200+
}
201+
]

‎.github/workflows/ci.yml‎

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -146,7 +146,11 @@ jobs:
146146
timeout-minutes: 10
147147
steps:
148148
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
149+
# sync: false — the test runs via `uv run --no-project` and needs no workspace
150+
# packages; the default `uv sync` builds the vortex-data Rust extension (~6 min).
149151
- uses: spiraldb/actions/.github/actions/setup-uv@a746510eafaa926484c354541cfc49b2ec06cc63 # 0.18.6
152+
with:
153+
sync: false
150154
- name: Pytest - measurement_id golden vectors
151155
run: |
152156
uv run --no-project --with pytest --with xxhash \
@@ -211,7 +215,7 @@ jobs:
211215
# including all private docs.
212216
cargo doc --profile ci --no-deps --document-private-items --workspace --exclude vortex-python --exclude vortex-python-cuda
213217
# nextest doesn't support doc tests, so we run it here
214-
cargo test --profile ci --doc --workspace --all-features --exclude vortex-cxx --exclude vortex-jni --exclude vortex-ffi --exclude xtask --exclude vortex-python-cuda --no-fail-fast
218+
cargo test --profile ci --doc --workspace --all-features --exclude vortex-jni --exclude vortex-ffi --exclude xtask --exclude vortex-python-cuda --no-fail-fast
215219
216220
build-rust:
217221
name: "Rust build (${{matrix.config.name}})"
@@ -415,6 +419,10 @@ jobs:
415419
--exclude compress-bench --exclude xtask --exclude vortex-datafusion `
416420
--exclude gpu-scan-cli --exclude vortex-sqllogictest
417421
422+
- name: sccache stats
423+
if: always() && github.repository == 'vortex-data/vortex'
424+
run: sccache --show-stats
425+
418426
- name: Alert incident.io
419427
if: failure() && github.event_name == 'push' && github.ref == 'refs/heads/develop'
420428
uses: ./.github/actions/alert-incident-io
@@ -553,8 +561,11 @@ jobs:
553561
- name: Build C++ library (asan)
554562
run: |
555563
mkdir -p build
556-
cmake -S lang/cpp -B build -DSANITIZER=asan -DTARGET_TRIPLE="x86_64-unknown-linux-gnu"
564+
cmake -S lang/cpp -Bbuild -DSANITIZER=asan -DBUILD_TESTS=1 -DTARGET_TRIPLE="x86_64-unknown-linux-gnu"
557565
cmake --build build --parallel $(nproc)
566+
- name: Run C++ tests
567+
run: |
568+
build/tests/vortex_cxx_test
558569
559570
sqllogic-test:
560571
name: "SQL logic tests"
@@ -584,14 +595,23 @@ jobs:
584595

585596
wasm-integration:
586597
name: "WASM integration smoke test"
587-
runs-on: ubuntu-latest
588598
timeout-minutes: 30
599+
runs-on: >-
600+
${{ github.repository == 'vortex-data/vortex'
601+
&& format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=wasm-integration', github.run_id)
602+
|| 'ubuntu-latest' }}
589603
steps:
604+
- uses: runs-on/action@v2
605+
if: github.repository == 'vortex-data/vortex'
606+
with:
607+
sccache: s3
590608
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
591-
- uses: ./.github/actions/setup-rust
609+
- uses: ./.github/actions/setup-prebuild
592610
with:
593-
repo-token: ${{ secrets.GITHUB_TOKEN }}
611+
enable-sccache: "true"
594612
targets: "wasm32-wasip1"
613+
- name: Install wasm32-wasip1 target
614+
run: rustup target add wasm32-wasip1
595615
- name: Setup Wasmer
596616
shell: bash
597617
run: |

‎.github/workflows/rust-instrumented.yml‎

Lines changed: 37 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,7 +101,7 @@ jobs:
101101
--ignore 'home/*' --ignore 'xtask/*' --ignore 'target/*' --ignore 'vortex-error/*' \
102102
--ignore 'vortex-python/*' --ignore 'vortex-jni/*' --ignore 'vortex-flatbuffers/*' \
103103
--ignore 'vortex-proto/*' --ignore 'vortex-tui/*' --ignore 'vortex-datafusion/examples/*' \
104-
--ignore 'vortex-ffi/examples/*' --ignore '*/arbitrary/*' --ignore '*/arbitrary.rs' --ignore 'vortex-cxx/*' \
104+
--ignore 'vortex-ffi/examples/*' --ignore '*/arbitrary/*' --ignore '*/arbitrary.rs' \
105105
--ignore benchmarks/* --ignore 'vortex-test/*' \
106106
-o ${{ env.GRCOV_OUTPUT_FILE }}
107107
test -s ${{ env.GRCOV_OUTPUT_FILE }}
@@ -114,6 +114,42 @@ jobs:
114114
flags: ${{ matrix.suite }}
115115
use_oidc: true
116116

117+
cxx-coverage:
118+
name: "C++ API (coverage)"
119+
timeout-minutes: 15
120+
permissions:
121+
id-token: write
122+
runs-on: >-
123+
${{ github.repository == 'vortex-data/vortex'
124+
&& format('runs-on={0}/runner=amd64-medium/image=ubuntu24-full-x64-pre-v2/extras=s3-cache/tag=cxx-coverage', github.run_id)
125+
|| 'ubuntu-latest' }}
126+
steps:
127+
- uses: runs-on/action@v2
128+
if: github.repository == 'vortex-data/vortex'
129+
with:
130+
sccache: s3
131+
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7
132+
- uses: ./.github/actions/setup-prebuild
133+
with:
134+
enable-sccache: "true"
135+
- name: Install lcov
136+
run: |
137+
sudo apt-get update
138+
sudo apt-get install -y lcov libjson-xs-perl
139+
- name: Build FFI library
140+
run: cargo build -p vortex-ffi
141+
- name: Build and test C++ API with coverage
142+
working-directory: lang/cpp
143+
run: ./gcov-report.sh
144+
- name: Codecov
145+
uses: codecov/codecov-action@fb8b3582c8e4def4969c97caa2f19720cb33a72f # v7
146+
with:
147+
name: cxx-api
148+
files: lang/cpp/coverage.info
149+
disable_search: true
150+
flags: cxx-api
151+
use_oidc: true
152+
117153
rust-test-sanitizer:
118154
strategy:
119155
fail-fast: false
Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
# Run Vortex SQL benchmark on every commit in a PR.
2+
# This is not gated behind action/benchmark-sql
3+
4+
name: PR Vortex Query Benchmark
5+
6+
concurrency:
7+
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
8+
cancel-in-progress: false
9+
10+
on:
11+
pull_request:
12+
types: [opened, synchronize, reopened]
13+
branches: ["develop"]
14+
15+
permissions:
16+
contents: read
17+
pull-requests: write
18+
id-token: write
19+
20+
jobs:
21+
sql:
22+
uses: ./.github/workflows/sql-benchmarks.yml
23+
secrets: inherit
24+
with:
25+
mode: "pr"
26+
benchmark_matrix: |
27+
[
28+
{
29+
"id": "vortex-queries",
30+
"subcommand": "vortex",
31+
"name": "Vortex queries",
32+
"data_formats": ["parquet", "vortex"],
33+
"pr_targets": [
34+
{"engine": "datafusion", "format": "parquet"},
35+
{"engine": "datafusion", "format": "vortex"},
36+
{"engine": "duckdb", "format": "parquet"},
37+
{"engine": "duckdb", "format": "vortex"}
38+
],
39+
"develop_targets": [
40+
{"engine": "datafusion", "format": "parquet"},
41+
{"engine": "datafusion", "format": "vortex"},
42+
{"engine": "duckdb", "format": "parquet"},
43+
{"engine": "duckdb", "format": "vortex"}
44+
],
45+
"iterations": "100"
46+
}
47+
]

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -224,6 +224,7 @@ compile_commands.json
224224

225225
# AI Agents
226226
.agents/settings.local.json
227+
.agents/scheduled_tasks.lock
227228
.claude/settings.local.json
228229
.opencode
229230

‎Cargo.lock‎

Lines changed: 5 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎bench-orchestrator/bench_orchestrator/config.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,7 @@ class Benchmark(Enum):
5454
PUBLIC_BI = "public-bi"
5555
STATPOPGEN = "statpopgen"
5656
SPATIALBENCH = "spatialbench"
57+
VORTEX_QUERIES = "vortex"
5758

5859

5960
# Engine to supported formats mapping.

‎docs/api/cpp/index.rst‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -164,8 +164,8 @@ Source code for this example is `writer.cpp
164164

165165
.. code-block:: cpp
166166
167-
const Session session;
168-
const DataType dtype = dtype::struct_({
167+
Session session;
168+
DataType dtype = dtype::struct_({
169169
{"age", dtype::uint8()},
170170
{"height", dtype::uint16(Nullable)},
171171
});
@@ -187,15 +187,14 @@ Source code for this example is `writer.cpp
187187
Expression age_gt_10 = expr::gt(expr::col("age"), expr::lit<uint8_t>(10));
188188
Array validity_array = array.apply(age_gt_10);
189189
190-
const Validity validity = Validity::from_array(validity_array);
190+
Validity validity = Validity::from_array(validity_array);
191191
Array array2 = make_struct({
192192
{"age", age},
193193
{"height", Array::primitive<uint16_t>(height_buffer, validity)},
194194
});
195195
196196
Writer writer = Writer::open(session, argv[1], dtype);
197-
writer.push(array);
198-
writer.push(array2);
197+
writer.push({array, array2});
199198
writer.finish();
200199
201200
DataType

‎encodings/bytebool/src/compute.rs‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -158,7 +158,12 @@ impl BooleanKernel for ByteBool {
158158

159159
fn truthy_bit_buffer(array: ArrayView<'_, ByteBool>) -> BitBuffer {
160160
let bytes = array.truthy_bytes();
161-
BitBuffer::collect_bool(bytes.len(), |idx| bytes[idx] != 0)
161+
// SAFETY: `collect_bool_multiversioned` invokes the predicate with indices `0..bytes.len()` only;
162+
// the trivially cheap unchecked gather vectorizes inside the wide pack loop.
163+
BitBuffer::collect_bool_multiversioned(
164+
bytes.len(),
165+
|idx| unsafe { *bytes.get_unchecked(idx) } != 0,
166+
)
162167
}
163168

164169
#[cfg(test)]

‎java/vortex-jni/src/main/java/dev/vortex/api/DataSource.java‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55

66
import com.google.common.base.Preconditions;
77
import dev.vortex.VortexCleaner;
8+
import dev.vortex.io.NativeReadable;
89
import dev.vortex.jni.NativeDataSource;
910
import java.util.Arrays;
1011
import java.util.Collections;
@@ -71,6 +72,51 @@ public static DataSource open(Session session, List<String> uris, Map<String, St
7172
return new DataSource(session, pointer);
7273
}
7374

75+
/** Open a single file through a caller-provided byte source. See {@link #open(Session, List, int)}. */
76+
public static DataSource open(Session session, NativeReadable readable) {
77+
return open(session, List.of(readable), 0);
78+
}
79+
80+
/** Open files through caller-provided byte sources with the default read concurrency. */
81+
public static DataSource open(Session session, List<NativeReadable> readables) {
82+
return open(session, readables, 0);
83+
}
84+
85+
/**
86+
* Open one or more files through caller-provided byte sources instead of native storage clients. Every read the
87+
* scan performs becomes an upcall into the corresponding {@link NativeReadable}, so this is the integration point
88+
* for external I/O abstractions (for example Iceberg's {@code FileIO}). Each file is identified by
89+
* {@link NativeReadable#name()}, which must be unique within {@code readables}.
90+
*
91+
* <p>The readables must remain open until this data source and all scans created from it are closed; native code
92+
* never closes them.
93+
*
94+
* @param session open session
95+
* @param readables byte sources
96+
* @param readConcurrency maximum in-flight {@code readFully} calls across all files of this data source; {@code 0}
97+
* selects the default. Each in-flight read occupies one native thread and typically one stream on its readable.
98+
*/
99+
public static DataSource open(Session session, List<NativeReadable> readables, int readConcurrency) {
100+
Objects.requireNonNull(session, "session");
101+
Objects.requireNonNull(readables, "readables");
102+
Preconditions.checkArgument(!readables.isEmpty(), "at least one readable is required");
103+
Preconditions.checkArgument(readConcurrency >= 0, "readConcurrency must not be negative");
104+
Object[] readableArray = readables.toArray();
105+
String[] names = new String[readableArray.length];
106+
long[] lengths = new long[readableArray.length];
107+
for (int i = 0; i < readableArray.length; i++) {
108+
Preconditions.checkArgument(readableArray[i] != null, "readables must not contain null values");
109+
NativeReadable readable = readables.get(i);
110+
names[i] = readable.name();
111+
Preconditions.checkArgument(names[i] != null, "readable at index %s returned a null name", i);
112+
lengths[i] = readable.length();
113+
Preconditions.checkArgument(lengths[i] >= 0, "readable for %s reported negative length", names[i]);
114+
}
115+
long pointer =
116+
NativeDataSource.openFiles(session.nativePointer(), readableArray, names, lengths, readConcurrency);
117+
return new DataSource(session, pointer);
118+
}
119+
74120
/** Arrow schema of the data source (and of scans produced from it). */
75121
public Schema arrowSchema(BufferAllocator allocator) {
76122
try (ArrowSchema schema = ArrowSchema.allocateNew(allocator)) {

0 commit comments

Comments
 (0)