Skip to content

Commit de25d50

Browse files
committed
ci: enforce native cache input and producer contracts
1 parent 8e49f73 commit de25d50

11 files changed

Lines changed: 681 additions & 89 deletions

‎.github/actions/build-native-ci/action.yaml‎

Lines changed: 17 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -20,56 +20,60 @@ description: 'Reuse an exact-input native library, otherwise build it with the C
2020
runs:
2121
using: composite
2222
steps:
23-
- name: Pin native build flags
24-
shell: bash
25-
run: echo 'RUSTFLAGS=-Ctarget-cpu=x86-64-v3 -Clink-arg=-fuse-ld=bfd' >> "$GITHUB_ENV"
26-
2723
# Call after checkout and setup-builder. Compute once, before Cargo writes
2824
# generated Rust files, and use the same keys for both restore and save.
2925
- name: Fingerprint native build inputs
3026
id: key
3127
shell: bash
32-
run: python3 dev/ci/native-cache-key.py --profile ci --github-output "$GITHUB_OUTPUT"
28+
env:
29+
RUSTFLAGS: "-Ctarget-cpu=x86-64-v3 -Clink-arg=-fuse-ld=bfd"
30+
run: |
31+
python3 dev/ci/native-cache-key.py --profile ci --github-output "$GITHUB_OUTPUT"
32+
# Keep one flag definition without changing later steps in the caller.
33+
echo "rustflags=$RUSTFLAGS" >> "$GITHUB_OUTPUT"
3334
3435
- name: Restore native library cache
35-
id: binary-cache
36+
id: library-cache
3637
uses: actions/cache/restore@v6
3738
with:
3839
path: native/target/ci/libcomet.so
39-
key: ${{ steps.key.outputs.binary-key }}
40+
key: ${{ steps.key.outputs.library-key }}
4041
# Main restores the incremental cache below, including libcomet.so.
4142
# A lookup avoids downloading the same library separately.
4243
lookup-only: ${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }}
4344

4445
- name: Restore incremental Cargo cache
4546
id: cargo-cache
46-
if: steps.binary-cache.outputs.cache-hit != 'true' || (github.event_name == 'push' && github.ref == 'refs/heads/main')
47+
if: steps.library-cache.outputs.cache-hit != 'true' || (github.event_name == 'push' && github.ref == 'refs/heads/main')
4748
uses: actions/cache/restore@v6
4849
with:
4950
path: native/target
50-
key: ${{ steps.key.outputs.source-key }}
51+
key: ${{ steps.key.outputs.cargo-key }}
5152
restore-keys: ${{ steps.key.outputs.restore-prefix }}
5253

5354
- name: Build native library (CI profile)
54-
if: steps.binary-cache.outputs.cache-hit != 'true' || (github.event_name == 'push' && github.ref == 'refs/heads/main' && steps.cargo-cache.outputs.cache-hit != 'true')
55+
if: steps.library-cache.outputs.cache-hit != 'true' || (github.event_name == 'push' && github.ref == 'refs/heads/main' && steps.cargo-cache.outputs.cache-hit != 'true')
5556
shell: bash
57+
env:
58+
RUSTFLAGS: ${{ steps.key.outputs.rustflags }}
5659
run: |
5760
cd native
5861
# A library miss always builds, even on an exact incremental hit.
5962
# Main also builds to populate a missing incremental entry; two exact
6063
# hits already supply the library and leave neither cache to publish.
6164
cargo build --locked --profile ci
65+
python3 ../dev/ci/native-cache-key.py --check-depinfo target/ci/libcomet.d
6266
6367
- name: Save native library cache
64-
if: github.event_name == 'push' && github.ref == 'refs/heads/main' && steps.binary-cache.outputs.cache-hit != 'true'
68+
if: github.event_name == 'push' && github.ref == 'refs/heads/main' && steps.library-cache.outputs.cache-hit != 'true'
6569
uses: actions/cache/save@v6
6670
with:
6771
path: native/target/ci/libcomet.so
68-
key: ${{ steps.key.outputs.binary-key }}
72+
key: ${{ steps.key.outputs.library-key }}
6973

7074
- name: Save incremental Cargo cache
7175
if: github.event_name == 'push' && github.ref == 'refs/heads/main' && steps.cargo-cache.outputs.cache-hit != 'true'
7276
uses: actions/cache/save@v6
7377
with:
7478
path: native/target
75-
key: ${{ steps.key.outputs.source-key }}
79+
key: ${{ steps.key.outputs.cargo-key }}

‎.github/workflows/README.md‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -437,6 +437,10 @@ lacks an exact match to replenish it.
437437
An incremental cache hit alone never replaces compilation. Builds use
438438
`cargo build --locked --profile ci`; manifest changes requiring a lockfile update
439439
must include that update to `native/Cargo.lock`. Artifact paths remain unchanged.
440+
After compilation, the action checks Cargo's `libcomet.d` dependency list before
441+
publishing either cache. Local build inputs must be covered by the tracked-file
442+
fingerprint, with explicit exceptions for generated protobuf modules and JDK
443+
files. Adding a new file input requires extending that contract.
440444

441445
`dev/ci/native-cache-key.py` snapshots native sources, protobufs, dependencies,
442446
Cargo configuration and shared build/setup actions before source generation.
@@ -446,7 +450,9 @@ Caller workflows are excluded because their selected tools and environment are
446450
observed directly. Spark edits, documentation, generated files and disabled
447451
contrib sources preserve the key; contrib manifests remain inputs for `--locked`.
448452
Benchmarks enter only the debug key. The input lists and glob matcher are shared
449-
with main's routing in `compute-changes.py`; code generation uses `x86-64-v3`.
453+
with routing in `compute-changes.py` and fingerprinted independently of its
454+
unrelated routing policy. Native manifests select validation on pull requests and
455+
the merge queue as well as main's cache warmer. Code generation uses `x86-64-v3`.
450456

451457
The helper supports the official Rust container and `setup-builder`. Introducing
452458
external tools or files requires updating that contract: an override's path does
@@ -455,6 +461,13 @@ retain package and JDK identity because native dependencies compile against JNI
455461
headers and link `libjvm`, and Cargo does not fully track external tool/header
456462
changes. Unrelated package updates can therefore cause conservative misses.
457463

464+
Every consumer must match main's native producer environment to reuse its library
465+
or incremental cache. A different JDK or container needs its own publisher;
466+
otherwise it will repeatedly build cold. Preflight checks the shared Linux
467+
runner, Rust container, toolchain/JDK selection and declared producer environment,
468+
including changes before the native action. New environment overrides must be
469+
reviewed against the fingerprint before they are admitted by this check.
470+
458471
The CI and debug incremental caches hold only `native/target`, including compiled
459472
dependencies. Cargo fetches registry and Git dependency sources as needed; those
460473
downloads are not duplicated in the repository's limited cache storage. Fallback

‎.github/workflows/ci.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -198,6 +198,7 @@ jobs:
198198
run: |
199199
python3 dev/ci/test-native-cache-key.py
200200
python3 dev/ci/test-native-cache-workflow.py
201+
python3 dev/ci/test-native-ci-config.py
201202
202203
- name: Check CI config invariants
203204
run: python3 dev/ci/check-ci-config.py

‎.github/workflows/pr_build_linux.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -405,7 +405,7 @@ jobs:
405405
uses: actions/cache/restore@v6
406406
with:
407407
path: native/target
408-
key: ${{ steps.cargo-key.outputs.source-key }}
408+
key: ${{ steps.cargo-key.outputs.cargo-key }}
409409
restore-keys: ${{ steps.cargo-key.outputs.restore-prefix }}
410410

411411
- name: Rust test steps
@@ -418,7 +418,7 @@ jobs:
418418
if: github.event_name == 'push' && github.ref == 'refs/heads/main' && steps.cargo-cache.outputs.cache-hit != 'true'
419419
with:
420420
path: native/target
421-
key: ${{ steps.cargo-key.outputs.source-key }}
421+
key: ${{ steps.cargo-key.outputs.cargo-key }}
422422

423423
linux-test:
424424
# `lint` is already upstream via build-native; it is listed here so this

‎dev/ci/check-ci-config.py‎

Lines changed: 162 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
# specific language governing permissions and limitations
1616
# under the License.
1717

18-
# Guards seven CI invariants that are silent when broken:
18+
# Guards CI invariants that are silent when broken:
1919
#
2020
# 1. Change-filter routing. dev/ci/compute-changes.py decides which heavy
2121
# jobs run. A file that a job depends on but that no filter lists makes
@@ -73,6 +73,10 @@
7373
# which is how every native build in the queue came to miss its cargo
7474
# cache and pay a cold ~26 minute compile.
7575
#
76+
# 8. Native library consumers share main's builder and declare only known
77+
# environment overrides. Caller workflows are not cache-key inputs, so
78+
# an untracked override must fail preflight rather than reuse a library.
79+
#
7680
# Run from the repository root: python3 dev/ci/check-ci-config.py
7781

7882
import importlib.util
@@ -123,6 +127,7 @@
123127

124128
# The two contrib/UDF gates also run ./mvnw, but consume no shared artifact.
125129
MVN_JOBS = BUILD_JOBS | {"delta_gate", "pyarrow_udf"}
130+
LINUX_JOBS = {"build_linux", "build_linux_full", "build_linux_all_profiles"}
126131

127132
ROUTING_CASES = [
128133
# The Maven wrapper and its config feed every job that runs ./mvnw: the
@@ -144,9 +149,17 @@
144149
# Helper edits also match macOS's broad dev/ci filter, as before.
145150
([".github/actions/build-native-ci/action.yaml"], BUILD_JOBS - {"build_macos"}),
146151
(["dev/ci/native-cache-key.py"], BUILD_JOBS),
152+
# Cargo validates disabled contrib manifests with native/Cargo.lock too.
153+
(["contrib/lance/native/Cargo.toml"], LINUX_JOBS),
154+
(["contrib/delta/native/Cargo.toml"], LINUX_JOBS | {"delta_gate"}),
155+
([".cargo/config.toml"], LINUX_JOBS),
156+
(["rust-toolchain"], LINUX_JOBS),
157+
(["dev/ci/test-native-cache-key.py"], LINUX_JOBS | {"build_macos"}),
158+
(["dev/ci/compute-changes.py"], LINUX_JOBS | {"build_macos"}),
147159
# Spot checks that the additions above did not widen unrelated routes.
148160
(["docs/source/user-guide/overview.md"], {"docs"}),
149161
(["native/core/benches/parquet_read.rs"], {"benchmark"}),
162+
(["native/shuffle/benches/shuffle.rs"], LINUX_JOBS | {"build_macos", "delta_gate"}),
150163
# The mermaid guard is run by preflight, which is unconditional, and again
151164
# by the docs deploy, which is not, so the deploy has to be routed. The
152165
# build jobs come along because `dev/ci/**` already feeds them.
@@ -1435,6 +1448,153 @@ def check_cache_save_scope():
14351448
return not failures
14361449

14371450

1451+
NATIVE_ACTION = "./.github/actions/build-native-ci"
1452+
NATIVE_ENV = {
1453+
"RUST_VERSION": "stable",
1454+
"RUST_BACKTRACE": "1",
1455+
"RUSTFLAGS": "-Clink-arg=-fuse-ld=bfd",
1456+
}
1457+
1458+
1459+
def _native_scalar(value):
1460+
value = value.split(" #", 1)[0].strip()
1461+
return value[1:-1] if len(value) > 1 and value[0] == value[-1] and value[0] in "\"'" else value
1462+
1463+
1464+
def _native_map(source, key, indent):
1465+
"""Read the flat, block-style maps used by the official native callers.
1466+
1467+
Like the other guards, require the supported shape instead of adding a
1468+
YAML dependency to preflight. A new shape needs an explicit contract edit.
1469+
"""
1470+
found = re.search(rf"(?m)^{' ' * indent}{key}:([^\n]*)$", source)
1471+
if not found:
1472+
return {}
1473+
if _native_scalar(found[1]):
1474+
raise ValueError(f"{key} must be a block mapping")
1475+
result = {}
1476+
for line in source[found.end():].splitlines():
1477+
if not line.strip() or line.lstrip().startswith("#"):
1478+
continue
1479+
if len(line) - len(line.lstrip()) <= indent:
1480+
break
1481+
entry = re.fullmatch(rf"{' ' * (indent + 2)}([\w-]+):\s*(.+)", line)
1482+
if not entry or entry[1] in result:
1483+
raise ValueError(f"unsupported or duplicate {key} entry: {line.strip()}")
1484+
result[entry[1]] = _native_scalar(entry[2])
1485+
return result
1486+
1487+
1488+
def _native_jobs(source):
1489+
parts = re.split(r"(?m)^ ([\w-]+):\s*$", source.partition("\njobs:\n")[2])
1490+
return dict(zip(parts[1::2], parts[2::2]))
1491+
1492+
1493+
def native_ci_failures(sources):
1494+
"""Check only setup through the native action; later JVM env is unrelated."""
1495+
failures, producers = [], set()
1496+
for filename, source in sources.items():
1497+
found_calls = 0
1498+
for job, body in _native_jobs(source).items():
1499+
action_calls = re.finditer(r"(?m)^\s+(?:- )?uses:\s*(.+)$", body)
1500+
if not any(_native_scalar(use[1]) == NATIVE_ACTION for use in action_calls):
1501+
continue
1502+
found_calls += 1
1503+
where = f"{filename}/{job}"
1504+
producers.add((filename, job))
1505+
try:
1506+
header, separator, steps_text = body.partition(" steps:\n")
1507+
if not separator:
1508+
raise ValueError("native producer must use block-style steps")
1509+
steps = [" - " + step for step in re.split(r"(?m)^ - ", steps_text)[1:]]
1510+
uses = [re.search(r"(?m)^\s+(?:- )?uses:\s*(\S+)", step) for step in steps]
1511+
native = [index for index, use in enumerate(uses) if use and use[1] == NATIVE_ACTION]
1512+
if len(native) != 1:
1513+
raise ValueError("expected one build-native-ci action")
1514+
before = steps[:native[0] + 1]
1515+
prefix = header + "".join(before)
1516+
runner = re.search(r"(?m)^ runs-on:\s*(.+)$", header)
1517+
if not runner or _native_scalar(runner[1]) != "ubuntu-24.04":
1518+
raise ValueError("native runner must match main's ubuntu-24.04")
1519+
if _native_map(header, "container", 4) != {"image": "amd64/rust"}:
1520+
raise ValueError("native container must match main's amd64/rust")
1521+
if _native_map(source.partition("\njobs:\n")[0], "env", 0) != NATIVE_ENV:
1522+
raise ValueError(f"workflow env must match the known native environment {NATIVE_ENV}")
1523+
if re.search(r"(?m)^\s+(?:- )?['\"]env['\"]\s*:", prefix):
1524+
raise ValueError("native env mappings must use the existing unquoted env key")
1525+
for match in re.finditer(r"(?m)^( +)env:", prefix):
1526+
declared = _native_map(prefix[match.start():], "env", len(match[1]))
1527+
if any(NATIVE_ENV.get(key) != value for key, value in declared.items()):
1528+
raise ValueError("unknown native env override; update the fingerprint contract first")
1529+
if re.search(r"(?m)^\s+- env:", prefix):
1530+
raise ValueError("put native step env after its name or uses key")
1531+
if re.search(r"GITHUB_(?:ENV|PATH)", prefix):
1532+
raise ValueError("caller must not write GITHUB_ENV/GITHUB_PATH before native reuse")
1533+
setup, checkouts = [], []
1534+
for index, (step, use) in enumerate(zip(before, uses)):
1535+
if not use:
1536+
continue
1537+
if use[1].startswith("actions/checkout@"):
1538+
checkouts.append((index, step))
1539+
elif use[1] == "./.github/actions/setup-builder":
1540+
setup.append((index, step))
1541+
elif use[1] != NATIVE_ACTION:
1542+
raise ValueError(f"unsupported action before native reuse: {use[1]}")
1543+
if len(checkouts) != 1 or len(setup) != 1:
1544+
raise ValueError("native reuse requires one checkout and one setup-builder")
1545+
required_steps = checkouts[0][1] + setup[0][1] + before[-1]
1546+
if checkouts[0][0] >= setup[0][0] or re.search(r"(?m)^\s+(?:- )?if:", required_steps):
1547+
raise ValueError("checkout, setup-builder and native reuse must run unconditionally in that order")
1548+
if _native_map(checkouts[0][1], "with", 8).get("path", ".") != ".":
1549+
raise ValueError("native checkout must use the default workspace path")
1550+
settings = _native_map(setup[0][1], "with", 8)
1551+
java = settings.pop("jdk-version", None)
1552+
if settings != {"rust-version": "${{ env.RUST_VERSION }}"}:
1553+
raise ValueError("setup-builder must use the shared RUST_VERSION without extra inputs")
1554+
if java == "${{ inputs.java }}":
1555+
target = f"./.github/workflows/{filename}"
1556+
callers = []
1557+
declared = 0
1558+
for text in sources.values():
1559+
declared += sum(bool(re.search(rf"uses:\s*['\"]?{re.escape(target)}(?:['\"\s,}}]|$)", line.split(" #", 1)[0]))
1560+
for line in text.splitlines() if not line.lstrip().startswith("#"))
1561+
for caller in _native_jobs(text).values():
1562+
use = re.search(r"(?m)^ uses:\s*(.+)$", caller)
1563+
if use and _native_scalar(use[1]) == target:
1564+
callers.append(caller)
1565+
bindings = [re.search(r"(?m)^ java:\s*(.+)$", caller) for caller in callers]
1566+
if len(callers) != declared or not bindings or any(not value or _native_scalar(value[1]) != "17" for value in bindings):
1567+
raise ValueError("every reusable native caller must bind java to 17")
1568+
elif java == "${{ steps.resolve.outputs.java }}":
1569+
resolver = [step for step in before if re.search(r"(?m)^ id: resolve\s*$", step)]
1570+
options = set(re.findall(r"(?m)^ - ['\"]([\d.]+)['\"]\s*$", source))
1571+
arms = dict(re.findall(r"(?m)^\s+([\d.]+)\)\s*(.+);;\s*$", resolver[0])) if len(resolver) == 1 else {}
1572+
if (not options or arms.keys() != options
1573+
or any(re.findall(r"(?:^|[;\s])java=([^\s;]+)", arm) != ["17"] for arm in arms.values())
1574+
or set(re.findall(r"(?:^|[;\s])java=([^\s;]+)", resolver[0])) != {"17"}
1575+
or 'echo "java=$java" >> "$GITHUB_OUTPUT"' not in resolver[0]):
1576+
raise ValueError("writer resolver must select and output JDK 17")
1577+
elif java != "17":
1578+
raise ValueError("native JDK must match main's JDK 17")
1579+
except ValueError as error:
1580+
failures.append(f"{where}: {error}")
1581+
calls = sum(bool(re.search(rf"uses:\s*['\"]?{re.escape(NATIVE_ACTION)}(?:['\"\s]|$)", line))
1582+
for line in source.splitlines() if not line.lstrip().startswith("#"))
1583+
if calls != found_calls:
1584+
failures.append(f"{filename}: unsupported native caller layout; use the existing job/step indentation")
1585+
if ("pr_build_linux.yml", "build-native") not in producers:
1586+
failures.append("main's pr_build_linux.yml/build-native producer is missing")
1587+
return failures
1588+
1589+
1590+
def check_native_ci_config():
1591+
sources = {path.name: path.read_text(encoding="utf-8") for path in WORKFLOWS.glob("*.y*ml")}
1592+
failures = native_ci_failures(sources)
1593+
for failure in failures:
1594+
print(f"native builder: {failure}")
1595+
return not failures
1596+
1597+
14381598
if __name__ == "__main__":
14391599
ok = check_change_filters()
14401600
ok = check_event_policy() and ok
@@ -1450,6 +1610,7 @@ def check_cache_save_scope():
14501610
ok = check_nightly_scope() and ok
14511611
ok = check_nightly_base_fallback() and ok
14521612
ok = check_cache_save_scope() and ok
1613+
ok = check_native_ci_config() and ok
14531614
ok = check_local_ci_config() and ok
14541615
if not ok:
14551616
sys.exit(1)

0 commit comments

Comments
 (0)