Skip to content

Commit 620e120

Browse files
committed
Merge remote-tracking branch 'upstream/develop' into fix/10756-rle-runend-deserialize-panic
#11285 moved array serde from VTable into ArrayPlugin impls, so carry the fix over: RLE's plugin deserialize uses PType::try_from(dtype) instead of dtype.as_ptype(), RunEnd's plugin deserialize returns errors for num_runs / offset values that do not fit in usize, and the regression tests now call ArrayPlugin::deserialize with ArrayDeserialization::new. Signed-off-by: Vincent <[email protected]>
2 parents 7bce475 + 49acec0 commit 620e120

1,603 files changed

Lines changed: 110912 additions & 56893 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.agents/skills/bench-performance/SKILL.md‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,8 +39,8 @@ Do not wait for a deep code read before showing benchmark comparisons or first s
3939

4040
2. Identify the exact benchmark target:
4141

42-
- suite: `tpch`, `tpcds`, `clickbench`, `fineweb`, `gh-archive`, `job`, `polarsignals`,
43-
`public-bi`, or `statpopgen`;
42+
- suite: `tpch`, `tpch-nested`, `tpcds`, `clickbench`, `fineweb`, `gh-archive`, `job`,
43+
`polarsignals`, `public-bi`, or `statpopgen`;
4444
- query number(s), for example `-q <query>`;
4545
- engine/format target(s), for example `datafusion:vortex` versus `datafusion:parquet`;
4646
- runtime environment toggles, if the branch exposes any.

‎.config/nextest.toml‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,14 @@
11
[profile.default]
22
slow-timeout = { period = "30s", terminate-after = 5 }
33

4+
# Spatial tests concurrently call INSTALL spatial which fails
5+
[test-groups]
6+
duckdb-extension-install = { max-threads = 1 }
7+
48
[[profile.default.overrides]]
59
filter = 'test(compress_large_int | fsst_compress_offsets_overflow_i32)'
610
priority = 100
11+
12+
[[profile.default.overrides]]
13+
filter = 'package(vortex-duckdb) & (test(spatial_) | test(test_geometry))'
14+
test-group = 'duckdb-extension-install'
Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
name: "Assert CPU"
2+
description: "Fail unless the runner has the expected CPU, so `-C target-cpu=native` builds match the benchmark machine"
3+
4+
inputs:
5+
implementer:
6+
description: "Expected `CPU implementer` from /proc/cpuinfo"
7+
required: false
8+
default: "0x41" # Arm
9+
part:
10+
description: "Expected `CPU part` from /proc/cpuinfo"
11+
required: false
12+
default: "0xd4f" # Neoverse V2 (Graviton4)
13+
14+
runs:
15+
using: "composite"
16+
steps:
17+
- name: Assert CPU
18+
shell: bash
19+
env:
20+
EXPECTED_IMPLEMENTER: ${{ inputs.implementer }}
21+
EXPECTED_PART: ${{ inputs.part }}
22+
run: |
23+
implementer=$(awk -F': *' '/^CPU implementer/ {print $2; exit}' /proc/cpuinfo)
24+
part=$(awk -F': *' '/^CPU part/ {print $2; exit}' /proc/cpuinfo)
25+
echo "CPU implementer=${implementer:-unknown} part=${part:-unknown}"
26+
lscpu | grep -E "^(Model name|Vendor ID)" || true
27+
if [[ "$implementer" != "$EXPECTED_IMPLEMENTER" || "$part" != "$EXPECTED_PART" ]]; then
28+
echo "::error::Expected CPU implementer=$EXPECTED_IMPLEMENTER part=$EXPECTED_PART, got implementer=${implementer:-unknown} part=${part:-unknown}"
29+
exit 1
30+
fi
Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
# SPDX-License-Identifier: Apache-2.0
2+
# SPDX-FileCopyrightText: Copyright the Vortex contributors
3+
4+
name: "Fetch benchmark baseline"
5+
description: >-
6+
Write the develop baseline for a PR benchmark run, read from the benchmarks Postgres: assume the
7+
read-only benchmarks role over OIDC and sign in as bench_ci_read with an RDS IAM token. If the
8+
baseline cannot be read, it writes no baseline file and the report says the baseline could not be
9+
read. The calling job needs `id-token: write`. Composite actions cannot read `vars`, so callers
10+
pass the repository variables in.
11+
12+
inputs:
13+
pr-records:
14+
description: "The PR run's --ingest-jsonl output"
15+
default: results.ingest.jsonl
16+
output:
17+
description: "Where to write the baseline records"
18+
default: base.ingest.jsonl
19+
role-arn:
20+
description: "The read-only benchmarks role (vars.GH_BENCH_READ_ROLE_ARN)"
21+
required: true
22+
region:
23+
description: "AWS region of the benchmarks RDS instance (vars.RDS_BENCH_REGION)"
24+
required: true
25+
endpoint:
26+
description: "Hostname of the benchmarks RDS instance (vars.RDS_BENCH_INSTANCE_ENDPOINT)"
27+
required: true
28+
database:
29+
description: "Name of the benchmarks database (vars.RDS_BENCH_DB_NAME)"
30+
required: true
31+
32+
runs:
33+
using: "composite"
34+
steps:
35+
# The credentials stay in step outputs so the job's existing AWS credentials, which later
36+
# steps such as the S3 cleanup still use, are not replaced.
37+
- name: Configure AWS credentials for the benchmarks database (OIDC)
38+
id: aws
39+
if: inputs.role-arn != ''
40+
continue-on-error: true
41+
uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6
42+
with:
43+
role-to-assume: ${{ inputs.role-arn }}
44+
aws-region: ${{ inputs.region }}
45+
retry-max-attempts: 1
46+
output-credentials: true
47+
output-env-credentials: false
48+
- name: Fetch baseline
49+
shell: bash
50+
env:
51+
PR_RECORDS: ${{ inputs.pr-records }}
52+
OUTPUT: ${{ inputs.output }}
53+
AWS_OUTCOME: ${{ steps.aws.outcome }}
54+
AWS_ACCESS_KEY_ID: ${{ steps.aws.outputs.aws-access-key-id }}
55+
AWS_SECRET_ACCESS_KEY: ${{ steps.aws.outputs.aws-secret-access-key }}
56+
AWS_SESSION_TOKEN: ${{ steps.aws.outputs.aws-session-token }}
57+
AWS_REGION: ${{ inputs.region }}
58+
RDS_BENCH_INSTANCE_ENDPOINT: ${{ inputs.endpoint }}
59+
RDS_BENCH_DB_NAME: ${{ inputs.database }}
60+
run: |
61+
set -Eeu -o pipefail
62+
if [[ "$AWS_OUTCOME" != "success" ]]; then
63+
# No baseline file tells the report that the baseline could not be read.
64+
echo "::warning::Could not assume the read-only benchmarks role; reporting without a baseline."
65+
exit 0
66+
fi
67+
curl -fsSL https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem \
68+
-o "${RUNNER_TEMP}/rds-global-bundle.pem"
69+
DSN="postgresql://bench_ci_read@${RDS_BENCH_INSTANCE_ENDPOINT}:5432/${RDS_BENCH_DB_NAME}?sslmode=verify-full&sslrootcert=${RUNNER_TEMP}/rds-global-bundle.pem"
70+
if ! uv run --no-project scripts/fetch-benchmark-baseline.py "$PR_RECORDS" "$OUTPUT" \
71+
--postgres "$DSN" --region "$AWS_REGION"; then
72+
echo "::warning::Could not read the baseline from the benchmarks database; reporting without a baseline."
73+
rm -f "$OUTPUT"
74+
fi

‎.github/actions/setup-sccache/action.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ runs:
4040
# Run on cache hits too: the installer sets PATH and registers its statistics hook.
4141
- name: Install sccache
4242
if: steps.tool-cache.outputs.installed != 'true'
43-
uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10
43+
uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # v0.0.11
4444
with:
4545
version: ${{ inputs.version }}
4646

‎.github/runs-on.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,15 @@
11
_extends: .github-private
22

33
images:
4-
ubuntu24-full-x64-pre-v2:
4+
ubuntu26-full-x64-pre-v2:
55
platform: "linux"
66
arch: "x64"
7-
name: "vortex-ci-v2-x64-*"
7+
name: "vortex-ci-v2-ubuntu26-x64-*"
88
owner: "245040174862"
9-
ubuntu24-full-arm64-pre-v2:
9+
ubuntu26-full-arm64-pre-v2:
1010
platform: "linux"
1111
arch: "arm64"
12-
name: "vortex-ci-v2-arm64-*"
12+
name: "vortex-ci-v2-ubuntu26-arm64-*"
1313
owner: "245040174862"
1414
windows25-full-x64-pre:
1515
platform: "windows"

0 commit comments

Comments
 (0)