1515# specific language governing permissions and limitations
1616# under the License.
1717
18- # Guards seven CI invariants that are silent when broken:
18+ # Guards CI invariants that are silent when broken:
1919#
2020# 1. Change-filter routing. dev/ci/compute-changes.py decides which heavy
2121# jobs run. A file that a job depends on but that no filter lists makes
7373# which is how every native build in the queue came to miss its cargo
7474# cache and pay a cold ~26 minute compile.
7575#
76+ # 8. Native library consumers share main's builder and declare only known
77+ # environment overrides. Caller workflows are not cache-key inputs, so
78+ # an untracked override must fail preflight rather than reuse a library.
79+ #
7680# Run from the repository root: python3 dev/ci/check-ci-config.py
7781
7882import importlib .util
123127
124128# The two contrib/UDF gates also run ./mvnw, but consume no shared artifact.
125129MVN_JOBS = BUILD_JOBS | {"delta_gate" , "pyarrow_udf" }
130+ LINUX_JOBS = {"build_linux" , "build_linux_full" , "build_linux_all_profiles" }
126131
127132ROUTING_CASES = [
128133 # The Maven wrapper and its config feed every job that runs ./mvnw: the
144149 # Helper edits also match macOS's broad dev/ci filter, as before.
145150 ([".github/actions/build-native-ci/action.yaml" ], BUILD_JOBS - {"build_macos" }),
146151 (["dev/ci/native-cache-key.py" ], BUILD_JOBS ),
152+ # Cargo validates disabled contrib manifests with native/Cargo.lock too.
153+ (["contrib/lance/native/Cargo.toml" ], LINUX_JOBS ),
154+ (["contrib/delta/native/Cargo.toml" ], LINUX_JOBS | {"delta_gate" }),
155+ ([".cargo/config.toml" ], LINUX_JOBS ),
156+ (["rust-toolchain" ], LINUX_JOBS ),
157+ (["dev/ci/test-native-cache-key.py" ], LINUX_JOBS | {"build_macos" }),
158+ (["dev/ci/compute-changes.py" ], LINUX_JOBS | {"build_macos" }),
147159 # Spot checks that the additions above did not widen unrelated routes.
148160 (["docs/source/user-guide/overview.md" ], {"docs" }),
149161 (["native/core/benches/parquet_read.rs" ], {"benchmark" }),
162+ (["native/shuffle/benches/shuffle.rs" ], LINUX_JOBS | {"build_macos" , "delta_gate" }),
150163 # The mermaid guard is run by preflight, which is unconditional, and again
151164 # by the docs deploy, which is not, so the deploy has to be routed. The
152165 # build jobs come along because `dev/ci/**` already feeds them.
@@ -1435,6 +1448,153 @@ def check_cache_save_scope():
14351448 return not failures
14361449
14371450
1451+ NATIVE_ACTION = "./.github/actions/build-native-ci"
1452+ NATIVE_ENV = {
1453+ "RUST_VERSION" : "stable" ,
1454+ "RUST_BACKTRACE" : "1" ,
1455+ "RUSTFLAGS" : "-Clink-arg=-fuse-ld=bfd" ,
1456+ }
1457+
1458+
1459+ def _native_scalar (value ):
1460+ value = value .split (" #" , 1 )[0 ].strip ()
1461+ return value [1 :- 1 ] if len (value ) > 1 and value [0 ] == value [- 1 ] and value [0 ] in "\" '" else value
1462+
1463+
1464+ def _native_map (source , key , indent ):
1465+ """Read the flat, block-style maps used by the official native callers.
1466+
1467+ Like the other guards, require the supported shape instead of adding a
1468+ YAML dependency to preflight. A new shape needs an explicit contract edit.
1469+ """
1470+ found = re .search (rf"(?m)^{ ' ' * indent } { key } :([^\n]*)$" , source )
1471+ if not found :
1472+ return {}
1473+ if _native_scalar (found [1 ]):
1474+ raise ValueError (f"{ key } must be a block mapping" )
1475+ result = {}
1476+ for line in source [found .end ():].splitlines ():
1477+ if not line .strip () or line .lstrip ().startswith ("#" ):
1478+ continue
1479+ if len (line ) - len (line .lstrip ()) <= indent :
1480+ break
1481+ entry = re .fullmatch (rf"{ ' ' * (indent + 2 )} ([\w-]+):\s*(.+)" , line )
1482+ if not entry or entry [1 ] in result :
1483+ raise ValueError (f"unsupported or duplicate { key } entry: { line .strip ()} " )
1484+ result [entry [1 ]] = _native_scalar (entry [2 ])
1485+ return result
1486+
1487+
1488+ def _native_jobs (source ):
1489+ parts = re .split (r"(?m)^ ([\w-]+):\s*$" , source .partition ("\n jobs:\n " )[2 ])
1490+ return dict (zip (parts [1 ::2 ], parts [2 ::2 ]))
1491+
1492+
1493+ def native_ci_failures (sources ):
1494+ """Check only setup through the native action; later JVM env is unrelated."""
1495+ failures , producers = [], set ()
1496+ for filename , source in sources .items ():
1497+ found_calls = 0
1498+ for job , body in _native_jobs (source ).items ():
1499+ action_calls = re .finditer (r"(?m)^\s+(?:- )?uses:\s*(.+)$" , body )
1500+ if not any (_native_scalar (use [1 ]) == NATIVE_ACTION for use in action_calls ):
1501+ continue
1502+ found_calls += 1
1503+ where = f"{ filename } /{ job } "
1504+ producers .add ((filename , job ))
1505+ try :
1506+ header , separator , steps_text = body .partition (" steps:\n " )
1507+ if not separator :
1508+ raise ValueError ("native producer must use block-style steps" )
1509+ steps = [" - " + step for step in re .split (r"(?m)^ - " , steps_text )[1 :]]
1510+ uses = [re .search (r"(?m)^\s+(?:- )?uses:\s*(\S+)" , step ) for step in steps ]
1511+ native = [index for index , use in enumerate (uses ) if use and use [1 ] == NATIVE_ACTION ]
1512+ if len (native ) != 1 :
1513+ raise ValueError ("expected one build-native-ci action" )
1514+ before = steps [:native [0 ] + 1 ]
1515+ prefix = header + "" .join (before )
1516+ runner = re .search (r"(?m)^ runs-on:\s*(.+)$" , header )
1517+ if not runner or _native_scalar (runner [1 ]) != "ubuntu-24.04" :
1518+ raise ValueError ("native runner must match main's ubuntu-24.04" )
1519+ if _native_map (header , "container" , 4 ) != {"image" : "amd64/rust" }:
1520+ raise ValueError ("native container must match main's amd64/rust" )
1521+ if _native_map (source .partition ("\n jobs:\n " )[0 ], "env" , 0 ) != NATIVE_ENV :
1522+ raise ValueError (f"workflow env must match the known native environment { NATIVE_ENV } " )
1523+ if re .search (r"(?m)^\s+(?:- )?['\"]env['\"]\s*:" , prefix ):
1524+ raise ValueError ("native env mappings must use the existing unquoted env key" )
1525+ for match in re .finditer (r"(?m)^( +)env:" , prefix ):
1526+ declared = _native_map (prefix [match .start ():], "env" , len (match [1 ]))
1527+ if any (NATIVE_ENV .get (key ) != value for key , value in declared .items ()):
1528+ raise ValueError ("unknown native env override; update the fingerprint contract first" )
1529+ if re .search (r"(?m)^\s+- env:" , prefix ):
1530+ raise ValueError ("put native step env after its name or uses key" )
1531+ if re .search (r"GITHUB_(?:ENV|PATH)" , prefix ):
1532+ raise ValueError ("caller must not write GITHUB_ENV/GITHUB_PATH before native reuse" )
1533+ setup , checkouts = [], []
1534+ for index , (step , use ) in enumerate (zip (before , uses )):
1535+ if not use :
1536+ continue
1537+ if use [1 ].startswith ("actions/checkout@" ):
1538+ checkouts .append ((index , step ))
1539+ elif use [1 ] == "./.github/actions/setup-builder" :
1540+ setup .append ((index , step ))
1541+ elif use [1 ] != NATIVE_ACTION :
1542+ raise ValueError (f"unsupported action before native reuse: { use [1 ]} " )
1543+ if len (checkouts ) != 1 or len (setup ) != 1 :
1544+ raise ValueError ("native reuse requires one checkout and one setup-builder" )
1545+ required_steps = checkouts [0 ][1 ] + setup [0 ][1 ] + before [- 1 ]
1546+ if checkouts [0 ][0 ] >= setup [0 ][0 ] or re .search (r"(?m)^\s+(?:- )?if:" , required_steps ):
1547+ raise ValueError ("checkout, setup-builder and native reuse must run unconditionally in that order" )
1548+ if _native_map (checkouts [0 ][1 ], "with" , 8 ).get ("path" , "." ) != "." :
1549+ raise ValueError ("native checkout must use the default workspace path" )
1550+ settings = _native_map (setup [0 ][1 ], "with" , 8 )
1551+ java = settings .pop ("jdk-version" , None )
1552+ if settings != {"rust-version" : "${{ env.RUST_VERSION }}" }:
1553+ raise ValueError ("setup-builder must use the shared RUST_VERSION without extra inputs" )
1554+ if java == "${{ inputs.java }}" :
1555+ target = f"./.github/workflows/{ filename } "
1556+ callers = []
1557+ declared = 0
1558+ for text in sources .values ():
1559+ declared += sum (bool (re .search (rf"uses:\s*['\"]?{ re .escape (target )} (?:['\"\s,}}]|$)" , line .split (" #" , 1 )[0 ]))
1560+ for line in text .splitlines () if not line .lstrip ().startswith ("#" ))
1561+ for caller in _native_jobs (text ).values ():
1562+ use = re .search (r"(?m)^ uses:\s*(.+)$" , caller )
1563+ if use and _native_scalar (use [1 ]) == target :
1564+ callers .append (caller )
1565+ bindings = [re .search (r"(?m)^ java:\s*(.+)$" , caller ) for caller in callers ]
1566+ if len (callers ) != declared or not bindings or any (not value or _native_scalar (value [1 ]) != "17" for value in bindings ):
1567+ raise ValueError ("every reusable native caller must bind java to 17" )
1568+ elif java == "${{ steps.resolve.outputs.java }}" :
1569+ resolver = [step for step in before if re .search (r"(?m)^ id: resolve\s*$" , step )]
1570+ options = set (re .findall (r"(?m)^ - ['\"]([\d.]+)['\"]\s*$" , source ))
1571+ arms = dict (re .findall (r"(?m)^\s+([\d.]+)\)\s*(.+);;\s*$" , resolver [0 ])) if len (resolver ) == 1 else {}
1572+ if (not options or arms .keys () != options
1573+ or any (re .findall (r"(?:^|[;\s])java=([^\s;]+)" , arm ) != ["17" ] for arm in arms .values ())
1574+ or set (re .findall (r"(?:^|[;\s])java=([^\s;]+)" , resolver [0 ])) != {"17" }
1575+ or 'echo "java=$java" >> "$GITHUB_OUTPUT"' not in resolver [0 ]):
1576+ raise ValueError ("writer resolver must select and output JDK 17" )
1577+ elif java != "17" :
1578+ raise ValueError ("native JDK must match main's JDK 17" )
1579+ except ValueError as error :
1580+ failures .append (f"{ where } : { error } " )
1581+ calls = sum (bool (re .search (rf"uses:\s*['\"]?{ re .escape (NATIVE_ACTION )} (?:['\"\s]|$)" , line ))
1582+ for line in source .splitlines () if not line .lstrip ().startswith ("#" ))
1583+ if calls != found_calls :
1584+ failures .append (f"{ filename } : unsupported native caller layout; use the existing job/step indentation" )
1585+ if ("pr_build_linux.yml" , "build-native" ) not in producers :
1586+ failures .append ("main's pr_build_linux.yml/build-native producer is missing" )
1587+ return failures
1588+
1589+
1590+ def check_native_ci_config ():
1591+ sources = {path .name : path .read_text (encoding = "utf-8" ) for path in WORKFLOWS .glob ("*.y*ml" )}
1592+ failures = native_ci_failures (sources )
1593+ for failure in failures :
1594+ print (f"native builder: { failure } " )
1595+ return not failures
1596+
1597+
14381598if __name__ == "__main__" :
14391599 ok = check_change_filters ()
14401600 ok = check_event_policy () and ok
@@ -1450,6 +1610,7 @@ def check_cache_save_scope():
14501610 ok = check_nightly_scope () and ok
14511611 ok = check_nightly_base_fallback () and ok
14521612 ok = check_cache_save_scope () and ok
1613+ ok = check_native_ci_config () and ok
14531614 ok = check_local_ci_config () and ok
14541615 if not ok :
14551616 sys .exit (1 )
0 commit comments