Skip to content

Commit e3b9e85

Browse files
committed
ci: align native cache selection with review feedback
1 parent c035477 commit e3b9e85

5 files changed

Lines changed: 62 additions & 32 deletions

File tree

‎.github/workflows/README.md‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -344,21 +344,30 @@ The Linux, Spark SQL, Iceberg and manual writer workflows call
344344
cache hit restores `native/target/ci/libcomet.so` and skips Cargo. A miss restores
345345
an incremental cache and runs `cargo build --locked --profile ci`. Artifacts and
346346
downstream tests use the same paths in either case.
347+
`--locked` deliberately fails when a manifest change requires updating
348+
`native/Cargo.lock`; contributors must commit that lockfile update with the change.
347349

348350
`dev/ci/native-cache-key.py` snapshots tracked native/protobuf/dependency files,
349351
Cargo configuration and the native build recipes before Cargo generates source
350352
files. The key also includes Rust versions, installed system package
351353
versions, architecture, JDK release/path and compiler flags. The helper targets
352354
our official Rust container and `setup-builder`, not arbitrary local toolchains.
353355
Spark-only edits, documentation, unrelated workflows and generated files preserve
354-
the key; native/protobuf changes invalidate it. Benchmarks enter the debug cache
355-
key but not the library key. The shared action uses portable `x86-64-v3` code
356-
generation.
356+
the key; native/protobuf changes invalidate it. Optional contrib crates contribute
357+
their manifests, which Cargo resolves even with their features disabled, but not
358+
their Rust sources or standalone lockfiles. Benchmarks enter the debug cache key
359+
but not the library key. The input lists and glob matcher are shared with main's
360+
cache routing in `compute-changes.py`. The shared action uses portable `x86-64-v3`
361+
code generation.
357362

358363
The incremental cache contains the effective `CARGO_HOME` registry/git directories
359-
and `native/target`. Its dependency prefix permits reuse after source changes,
360-
but every restore still invokes Cargo. The Rust test job uses a separate debug
361-
key and continues to run all checks and tests.
364+
and `native/target`. Its dependency prefix permits reuse after source changes
365+
within the same build environment, but every restore still invokes Cargo.
366+
Environment changes also invalidate this fallback: native dependencies compile C
367+
against JNI headers and cache build-script outputs that Cargo does not fully
368+
invalidate after external compiler or JDK changes. This can miss after unrelated
369+
package updates, but prevents reusing those objects under a new library key.
370+
The Rust test job uses a separate debug key and continues to run all checks and tests.
362371

363372
Only pushes to `main` save either cache. Main always compiles to keep the
364373
incremental cache warm. Other runs consume matching entries; a cold or evicted

‎.github/workflows/pr_build_linux.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -362,6 +362,8 @@ jobs:
362362
jdk-version: 17
363363

364364
- name: Fingerprint Rust test build inputs
365+
# Default HDFS support compiles against JNI headers and links libjvm.
366+
# Cached build-script outputs also contain JDK paths (core/build.rs).
365367
id: cargo-key
366368
run: python3 dev/ci/native-cache-key.py --profile debug --github-output "$GITHUB_OUTPUT"
367369

‎dev/ci/compute-changes.py‎

Lines changed: 23 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,25 @@
3838
import sys
3939
from pathlib import Path
4040

41+
# Shared cache recipes affect every native producer. Tests exercise the recipes
42+
# but do not affect the resulting library, so they are routed separately below.
43+
NATIVE_CACHE_RECIPES = (
44+
".github/actions/build-native-ci/**",
45+
"dev/ci/native-cache-key.py", "dev/ci/compute-changes.py",
46+
)
47+
48+
# Cargo validates optional contrib manifests against native/Cargo.lock even
49+
# with their features disabled. Their Rust sources and standalone lockfiles
50+
# do not enter the default CI/debug builds.
51+
NATIVE_BUILD_INPUTS = (
52+
"native/**", "contrib/*/native/Cargo.toml", ".cargo/**",
53+
".github/actions/setup-builder/**", *NATIVE_CACHE_RECIPES,
54+
".github/workflows/pr_build_linux.yml", ".github/workflows/spark_sql_test_reusable.yml",
55+
".github/workflows/iceberg_spark_test_reusable.yml", ".github/workflows/spark_sql_writer_tests.yml",
56+
"rust-toolchain", "rust-toolchain.toml", "!**.md",
57+
)
58+
NATIVE_LIBRARY_INPUTS = (*NATIVE_BUILD_INPUTS, "!**/benches/**")
59+
4160
FILTERS = {
4261
"build_linux": [
4362
"native/**",
@@ -395,8 +414,7 @@
395414
"iceberg_1_8", "iceberg_1_9", "iceberg_1_10", "iceberg_1_11",
396415
):
397416
FILTERS[_native_consumer].extend([
398-
".github/actions/build-native-ci/**",
399-
"dev/ci/native-cache-key.py",
417+
*NATIVE_CACHE_RECIPES,
400418
"dev/ci/test-native-cache-key.py",
401419
])
402420

@@ -563,15 +581,9 @@ def compute(files, event):
563581
name: event_allows(name, event) and matches(patterns, files)
564582
for name, patterns in FILTERS.items()
565583
}
566-
# These native-key inputs have no ordinary Linux route. Warm main after
567-
# they change, without adding the full Linux pipeline to contrib-only PRs.
568-
if event.get("name") == "push" and matches([
569-
"contrib/*/native/**", ".cargo/**", "rust-toolchain",
570-
".github/workflows/spark_sql_test_reusable.yml",
571-
".github/workflows/iceberg_spark_test_reusable.yml",
572-
".github/workflows/spark_sql_writer_tests.yml",
573-
"!**.md", "!**/benches/**",
574-
], files):
584+
# Use the fingerprint's exact patterns and matcher for main's producer,
585+
# including inputs owned by other workflows, without broadening PR jobs.
586+
if event.get("name") == "push" and matches(NATIVE_LIBRARY_INPUTS, files):
575587
selected["build_linux"] = True
576588
return selected
577589

‎dev/ci/native-cache-key.py‎

Lines changed: 7 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -24,23 +24,16 @@
2424
"""
2525

2626
import argparse
27-
from fnmatch import fnmatchcase
2827
import hashlib
2928
import json
3029
import os
3130
from pathlib import Path
31+
import runpy
3232
import subprocess
3333

3434

35-
# Direct Cargo inputs and the recipes that install/configure the native build.
36-
# Unrelated workflows and JVM/Maven files do not enter the native library build.
37-
INPUT_PATTERNS = (
38-
"native/**", "contrib/*/native/**", ".cargo/**",
39-
".github/actions/setup-builder/**", ".github/actions/build-native-ci/**",
40-
".github/workflows/pr_build_linux.yml", ".github/workflows/spark_sql_test_reusable.yml",
41-
".github/workflows/iceberg_spark_test_reusable.yml", ".github/workflows/spark_sql_writer_tests.yml",
42-
"rust-toolchain", "rust-toolchain.toml", "dev/ci/native-cache-key.py",
43-
)
35+
# Share both the input patterns and their glob semantics with main's warmer.
36+
CHANGES = runpy.run_path(str(Path(__file__).with_name("compute-changes.py")))
4437

4538

4639
def digest(value):
@@ -61,16 +54,15 @@ def source_inputs(root, profile="ci"):
6154
builds omit benchmarks; debug checks compile them. Trust only this checkout
6255
for the Git read: container steps can run as a different owner than checkout.
6356
"""
57+
patterns = CHANGES["NATIVE_LIBRARY_INPUTS" if profile == "ci" else "NATIVE_BUILD_INPUTS"]
6458
inventory = command(["git", "-c", f"safe.directory={root}",
6559
"ls-files", "--stage", "-z"], root)
6660
sources = {}
6761
for record in inventory.split("\0"):
6862
if not record:
6963
continue
7064
metadata, name = record.split("\t", 1)
71-
if name.endswith(".md") or (profile == "ci" and "/benches/" in name):
72-
continue
73-
if any(fnmatchcase(name, pattern) for pattern in INPUT_PATTERNS):
65+
if CHANGES["matches"](patterns, [name]):
7466
sources[name] = [metadata.split()[0],
7567
hashlib.sha256((root / name).read_bytes()).hexdigest()]
7668
dependencies = {name: value for name, value in sources.items()
@@ -108,6 +100,8 @@ def cache_keys(profile, dependencies, sources, environment):
108100
109101
Only the incremental Cargo cache has a source-independent restore prefix.
110102
The library key includes all tracked build inputs and never uses fallback.
103+
Both retain the environment: native build scripts can reuse C objects
104+
without detecting changes to external compiler binaries or JNI headers.
111105
"""
112106
prefix = f"Linux-cargo-{profile}-v3-{digest([environment, dependencies])}-"
113107
return {

‎dev/ci/test-native-cache-key.py‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,11 @@ def setUp(self):
5050
".github/workflows/README.md": "CI documentation\n",
5151
".github/workflows/spark_sql_test_reusable.yml": "jobs: {}\n",
5252
".github/workflows/check_pr_title.yml": "jobs: {}\n",
53+
"dev/ci/compute-changes.py": "# shared native input rules\n",
54+
"contrib/delta/native/Cargo.toml": '[package]\nname = "delta"\n',
55+
"contrib/delta/native/src/lib.rs": "fn delta() {}\n",
56+
"contrib/delta/native/Cargo.lock": "version = 4\n",
57+
"contrib/a/b/native/Cargo.toml": '[package]\nname = "nested"\n',
5358
"native/core/benches/perf.rs": "fn benchmark() {}\n"}
5459
for name, content in self.inputs.items():
5560
self.write(name, content)
@@ -78,6 +83,7 @@ def test_source_and_dependency_changes_invalidate_the_right_keys(self):
7883
"""Native/protobuf edits retain the dependency prefix; dependency edits replace it."""
7984
before = self.keys()
8085
for name in ("native/lib.rs", "native/proto/expr.proto", "native/Cargo.toml", "native/Cargo.lock",
86+
"contrib/delta/native/Cargo.toml", "dev/ci/compute-changes.py",
8187
".github/workflows/spark_sql_test_reusable.yml"):
8288
with self.subTest(name=name):
8389
self.write(name, self.inputs[name] + "changed\n")
@@ -100,6 +106,9 @@ def test_generated_files_and_unrelated_jvm_edits_preserve_keys(self):
100106
self.write("README.md", "updated docs")
101107
self.write(".github/workflows/README.md", "updated CI docs")
102108
self.write(".github/workflows/check_pr_title.yml", "jobs: {changed: {}}")
109+
self.write("contrib/delta/native/src/lib.rs", "fn changed_delta() {}")
110+
self.write("contrib/delta/native/Cargo.lock", "version = 3\n")
111+
self.write("contrib/a/b/native/Cargo.toml", '[package]\nname = "changed_nested"\n')
103112
self.write("native/core/benches/perf.rs", "fn changed_benchmark() {}")
104113
self.assertEqual(before, self.keys())
105114
self.assertNotEqual(debug["source-key"], self.keys("debug")["source-key"])
@@ -109,9 +118,12 @@ def test_every_binary_key_input_has_a_main_cache_warmer(self):
109118
project = Path(__file__).resolve().parents[2]
110119
route = runpy.run_path(str(project / "dev/ci/compute-changes.py"))["compute"]
111120
_, sources = CACHE.source_inputs(project)
112-
for name in [*sources, ".cargo/config.toml", "rust-toolchain", "contrib/new/native/src/lib.rs"]:
121+
for name in [*sources, ".cargo/config.toml", "rust-toolchain", "contrib/new/native/Cargo.toml"]:
113122
self.assertTrue(route([name], {"name": "push"})["build_linux"], name)
114-
self.assertFalse(route(["contrib/new/native/src/lib.rs"], {"name": "pull_request"})["build_linux"])
123+
for name in ("contrib/delta/native/src/lib.rs", "contrib/delta/native/Cargo.lock",
124+
"contrib/a/b/native/Cargo.toml", "contrib/a/b/native/x.rs"):
125+
self.assertFalse(route([name], {"name": "push"})["build_linux"], name)
126+
self.assertFalse(route(["contrib/new/native/Cargo.toml"], {"name": "pull_request"})["build_linux"])
115127

116128
def test_tools_jdk_flags_and_tracked_build_configuration_invalidate(self):
117129
"""Observed tool/package versions, Java metadata, flags and tracked configs enter keys."""
@@ -122,6 +134,7 @@ def test_tools_jdk_flags_and_tracked_build_configuration_invalidate(self):
122134
self.versions[tool] += "changed\n"
123135
self.assertNotEqual(before["source-key"], self.keys()["source-key"])
124136
self.assertNotEqual(before["binary-key"], self.keys()["binary-key"])
137+
self.assertNotEqual(before["restore-prefix"], self.keys()["restore-prefix"])
125138
self.versions[tool] = old
126139
self.write("jdk/release", 'JAVA_VERSION="17.0.2"\n')
127140
self.assertNotEqual(before["binary-key"], self.keys()["binary-key"])

0 commit comments

Comments
 (0)