Skip to content

Commit dd898c3

Browse files
Merge upstream develop into nested schema pruning
Signed-off-by: Christopher Wirt <[email protected]>
2 parents 8562bc4 + 19dcd2f commit dd898c3

6 files changed

Lines changed: 1462 additions & 0 deletions

File tree

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
# SPDX-License-Identifier: Apache-2.0
2+
# SPDX-FileCopyrightText: Copyright the Vortex contributors
3+
4+
name: S3 Measurements
5+
6+
on:
7+
workflow_dispatch:
8+
inputs:
9+
windows:
10+
description: "Measurement windows, three hours apart, then stop"
11+
type: choice
12+
options: ["4", "1", "2", "3"]
13+
default: "4"
14+
push:
15+
branches: ["ji/s3-measurements"]
16+
paths:
17+
- .github/workflows/s3-measurements.yml
18+
- vortex-bench/src/bin/s3-measure.rs
19+
- benchmarks/s3/**
20+
21+
permissions:
22+
contents: read
23+
id-token: write
24+
25+
concurrency:
26+
group: s3-measurements-${{ github.ref }}
27+
cancel-in-progress: false
28+
29+
jobs:
30+
collect:
31+
timeout-minutes: 780
32+
runs-on: runs-on=${{ github.run_id }}/runner=bench-dedicated/family=c8gd.16xlarge/spot=false/image=ubuntu24-full-arm64-pre-v2/tag=s3-measure/extras=s3-cache
33+
env:
34+
AWS_REGION: us-east-1
35+
REMOTE_ROOT: s3://vortex-ci-benchmark-datasets/s3-measurements/${{ github.run_id }}-${{ github.run_attempt }}
36+
MEASUREMENT_WINDOWS: ${{ inputs.windows || '1' }}
37+
S3_PROBE_ARGS: >-
38+
--sizes 4096,65536,262144,1048576,8388608 --concurrency 1,4,16,64
39+
--requests 64 --rounds 3 --warmup-seconds 20 --timeout-seconds 60
40+
steps:
41+
- uses: runs-on/action@v2
42+
with:
43+
sccache: s3
44+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
45+
- uses: ./.github/actions/setup-prebuild
46+
with:
47+
enable-sccache: "true"
48+
- uses: ./.github/actions/system-info
49+
- name: Build collector
50+
run: cargo build --locked -p vortex-bench --bin s3-measure --release
51+
- uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6
52+
with:
53+
role-to-assume: arn:aws:iam::245040174862:role/GitHubBenchmarkRole
54+
aws-region: us-east-1
55+
role-duration-seconds: 7200
56+
- name: Verify region and create immutable probe objects
57+
run: |
58+
region=$(curl --silent --show-error --head --max-time 20 https://vortex-ci-benchmark-datasets.s3.amazonaws.com/ |
59+
awk 'tolower($1) == "x-amz-bucket-region:" {gsub("\r", "", $2); print $2}')
60+
test "$region" = "$AWS_REGION"
61+
test "$RUNS_ON_AWS_REGION" = "$AWS_REGION"
62+
python3 benchmarks/s3/create_probe.py
63+
for object in 1 2 3 4; do
64+
aws s3 cp s3-probe.bin "$REMOTE_ROOT/probe-$object.bin"
65+
done
66+
- name: Initialize the fixed measurement schedule
67+
id: schedule
68+
run: >-
69+
python3 benchmarks/s3/measure.py --source "$REMOTE_ROOT/"
70+
--count "$MEASUREMENT_WINDOWS" --seed 20261028 --initialize -- $S3_PROBE_ARGS
71+
- name: Collect window 1
72+
continue-on-error: true
73+
run: >-
74+
python3 benchmarks/s3/measure.py --source "$REMOTE_ROOT/"
75+
--count "$MEASUREMENT_WINDOWS" --seed 20261028 --run-window 1 -- $S3_PROBE_ARGS
76+
- name: Wait for window 2
77+
if: ${{ !cancelled() && steps.schedule.outcome == 'success' && fromJSON(env.MEASUREMENT_WINDOWS) >= 2 }}
78+
run: >-
79+
python3 benchmarks/s3/measure.py --source "$REMOTE_ROOT/"
80+
--count "$MEASUREMENT_WINDOWS" --seed 20261028 --wait-window 2 -- $S3_PROBE_ARGS
81+
- name: Configure credentials for window 2
82+
if: ${{ !cancelled() && steps.schedule.outcome == 'success' && fromJSON(env.MEASUREMENT_WINDOWS) >= 2 }}
83+
uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6
84+
with:
85+
role-to-assume: arn:aws:iam::245040174862:role/GitHubBenchmarkRole
86+
aws-region: us-east-1
87+
role-duration-seconds: 7200
88+
- name: Collect window 2
89+
if: ${{ !cancelled() && steps.schedule.outcome == 'success' && fromJSON(env.MEASUREMENT_WINDOWS) >= 2 }}
90+
continue-on-error: true
91+
run: >-
92+
python3 benchmarks/s3/measure.py --source "$REMOTE_ROOT/"
93+
--count "$MEASUREMENT_WINDOWS" --seed 20261028 --run-window 2 -- $S3_PROBE_ARGS
94+
- name: Wait for window 3
95+
if: ${{ !cancelled() && steps.schedule.outcome == 'success' && fromJSON(env.MEASUREMENT_WINDOWS) >= 3 }}
96+
run: >-
97+
python3 benchmarks/s3/measure.py --source "$REMOTE_ROOT/"
98+
--count "$MEASUREMENT_WINDOWS" --seed 20261028 --wait-window 3 -- $S3_PROBE_ARGS
99+
- name: Configure credentials for window 3
100+
if: ${{ !cancelled() && steps.schedule.outcome == 'success' && fromJSON(env.MEASUREMENT_WINDOWS) >= 3 }}
101+
uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6
102+
with:
103+
role-to-assume: arn:aws:iam::245040174862:role/GitHubBenchmarkRole
104+
aws-region: us-east-1
105+
role-duration-seconds: 7200
106+
- name: Collect window 3
107+
if: ${{ !cancelled() && steps.schedule.outcome == 'success' && fromJSON(env.MEASUREMENT_WINDOWS) >= 3 }}
108+
continue-on-error: true
109+
run: >-
110+
python3 benchmarks/s3/measure.py --source "$REMOTE_ROOT/"
111+
--count "$MEASUREMENT_WINDOWS" --seed 20261028 --run-window 3 -- $S3_PROBE_ARGS
112+
- name: Wait for window 4
113+
if: ${{ !cancelled() && steps.schedule.outcome == 'success' && fromJSON(env.MEASUREMENT_WINDOWS) >= 4 }}
114+
run: >-
115+
python3 benchmarks/s3/measure.py --source "$REMOTE_ROOT/"
116+
--count "$MEASUREMENT_WINDOWS" --seed 20261028 --wait-window 4 -- $S3_PROBE_ARGS
117+
- name: Configure credentials for window 4
118+
if: ${{ !cancelled() && steps.schedule.outcome == 'success' && fromJSON(env.MEASUREMENT_WINDOWS) >= 4 }}
119+
uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6
120+
with:
121+
role-to-assume: arn:aws:iam::245040174862:role/GitHubBenchmarkRole
122+
aws-region: us-east-1
123+
role-duration-seconds: 7200
124+
- name: Collect window 4
125+
if: ${{ !cancelled() && steps.schedule.outcome == 'success' && fromJSON(env.MEASUREMENT_WINDOWS) >= 4 }}
126+
continue-on-error: true
127+
run: >-
128+
python3 benchmarks/s3/measure.py --source "$REMOTE_ROOT/"
129+
--count "$MEASUREMENT_WINDOWS" --seed 20261028 --run-window 4 -- $S3_PROBE_ARGS
130+
- name: Require all requested windows and summarize
131+
if: ${{ always() && steps.schedule.outcome == 'success' }}
132+
run: >-
133+
python3 benchmarks/s3/measure.py --source "$REMOTE_ROOT/"
134+
--count "$MEASUREMENT_WINDOWS" --seed 20261028 --summarize -- $S3_PROBE_ARGS
135+
- name: Preserve raw requests, manifests, schedule and median/p90 tables
136+
if: always()
137+
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7
138+
with:
139+
name: s3-measurements-${{ github.run_attempt }}
140+
path: s3-measurements/
141+
retention-days: 30
142+
- name: Refresh credentials for cleanup
143+
if: always()
144+
uses: aws-actions/configure-aws-credentials@e1253824e5c10ff9df46874f81ed3ec929e19cfd # v6
145+
with:
146+
role-to-assume: arn:aws:iam::245040174862:role/GitHubBenchmarkRole
147+
aws-region: us-east-1
148+
- name: Remove this run's probe objects
149+
if: always()
150+
run: aws s3 rm --recursive "$REMOTE_ROOT/"

‎benchmarks/s3/README.md‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
<!-- SPDX-License-Identifier: Apache-2.0 -->
2+
<!-- SPDX-FileCopyrightText: Copyright the Vortex contributors -->
3+
4+
# S3 measurements
5+
6+
`s3-measure` collects real S3 request observations independently of query engines and simulators.
7+
The CI workflow sweeps 4 KiB, 64 KiB, 256 KiB, 1 MiB and 8 MiB ranges at concurrency 1, 4, 16
8+
and 64. Three rounds use 64 requests per size/concurrency cell, with shuffled cell order and
9+
seeded object/range selection. Each concurrency uses identical ranges so load is the changed input.
10+
11+
```bash
12+
cargo build -p vortex-bench --bin s3-measure --release
13+
python3 benchmarks/s3/measure.py --source s3://bucket/prefix/ --count 4 --interval-seconds 10800 \
14+
--seed 20261028 -- --sizes 4096,65536,262144,1048576,8388608 --concurrency 1,4,16,64 \
15+
--requests 64 --rounds 3 --warmup-seconds 20 --timeout-seconds 60
16+
```
17+
18+
Objects must already exist; the collector never writes to S3. At least one must be 8 MiB or larger
19+
for this sweep. The collector requires explicit sizes, concurrency, request count, rounds, seed,
20+
warmup and deadline; callers choose these for their hardware. For a 192-core runner, for example,
21+
extend the concurrency list to `1,4,16,64,192` and use at least `--requests 192` so the probe can
22+
reach that load. The wrapper forwards arguments after `--` and supplies `--seed` separately.
23+
The wrapper anchors four windows to 0, 3, 6 and 9 hours, then stops. It
24+
retains a SHA-256 for each trace and the binary, checks identical runner/object/probe identities,
25+
and preserves partial evidence on failure. Each request has a deadline; a window has a 30-minute
26+
budget. Starts over five minutes late are failures, without silently shifting the remaining schedule.
27+
28+
The manual `S3 Measurements` workflow keeps one on-demand `c8gd.16xlarge` in `us-east-1` for the
29+
whole study. It creates four seeded 16 MiB fixtures in a unique prefix, verifies bucket/runner
30+
region, collects one to four windows, uploads evidence and removes only its fixtures. Branch pushes
31+
run one window to exercise the collector. Authentication uses the same pinned
32+
`aws-actions/configure-aws-credentials` action as the S3 benchmarks. CI waits for each fixed start,
33+
then obtains fresh credentials before collecting that window. Python inherits the configured
34+
credentials; it does not fetch tokens, assume roles or write credential files. Standalone multi-window
35+
runs use the caller's standard AWS credential provider, which must remain valid for the study.
36+
37+
Each JSONL starts with a **schema version 1 environment**: actual CI run/attempt and commit,
38+
measurement window ID, source, hardware/region/availability zone, CPU/memory, seed, probe configuration,
39+
and the complete object size/ETag/version manifest. It also identifies local smoke tests and custom
40+
endpoints so a learner can reject them as real-S3 training inputs. Credentials are excluded.
41+
42+
Request records contain operation, object/range, observed identity, monotonic and wall-clock start,
43+
configured/actual in-flight concurrency, consumed bytes, response-header time, first-body-byte time,
44+
completion time, status and errors. A 20-second source warmup is explicitly labelled and excluded
45+
from the measured sweep. GET headers and body completion are separate observations; a learner can
46+
estimate size-dependent transfer cost, latency tails and concurrency effects without query runtimes.
47+
Partial/failed/timed-out reads remain visible and must not become latency samples.
48+
49+
Cell summaries retain duration, bytes, errors, peak/time-averaged concurrency and achieved aggregate
50+
throughput. `measurements.csv` adds per-round median/p90 header, first-byte, body and total timings.
51+
Every successful window has a terminal completion record and exact declared cell/request coverage.
52+
Object replacement is an error, guarded by ETag preconditions and response metadata checks.
53+
54+
Counts and timings cover **SDK operations**. The SDK may retry internally; those delays appear in
55+
the operation duration, while individual HTTP attempts and recovered 503s are not separately counted.
56+
The catalog LIST record covers the listing stream, including pagination. HEAD/LIST catalog setup is
57+
labelled separately. The measured workload issues no PUTs. These files provide request-level model
58+
inputs; they do not establish accuracy for SQL, random-access or decompression benchmarks.
59+
60+
Local harness exercise, explicitly ineligible as S3 training data:
61+
62+
```bash
63+
target/release/s3-measure --source "file://$PWD/fixtures/" --allow-local \
64+
--output /tmp/local-s3-measure.jsonl --measurement-id local \
65+
--sizes 4096,65536 --concurrency 1,4 --requests 8 --rounds 1 --seed 20261028 \
66+
--warmup-seconds 0 --timeout-seconds 60
67+
```
68+
69+
This is the first part of a stack: measurement, then learned RAM modelling, then opt-in SQL,
70+
random-access and compression/decompression benchmark integration. Real S3 remains the reference.

‎benchmarks/s3/create_probe.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
# SPDX-License-Identifier: Apache-2.0
2+
# SPDX-FileCopyrightText: Copyright the Vortex contributors
3+
4+
"""Create the reproducible 16 MiB probe shared by S3 measurement and calibration runs."""
5+
6+
import random
7+
from pathlib import Path
8+
9+
10+
def main():
11+
Path("s3-probe.bin").write_bytes(random.Random(20261028).randbytes(16 * 1024 * 1024))
12+
13+
14+
if __name__ == "__main__":
15+
main()

0 commit comments

Comments
 (0)