|
60 | 60 |
|
61 | 61 | import importlib.util |
62 | 62 | import re |
| 63 | +import subprocess |
63 | 64 | import sys |
64 | 65 | from pathlib import Path |
65 | 66 |
|
66 | 67 | WORKFLOWS = Path(".github/workflows") |
67 | 68 | ASF_YAML = Path(".asf.yaml") |
| 69 | +LOCAL_CI = Path("dev/local-ci.sh") |
68 | 70 |
|
69 | 71 | # The ci.yml job that aggregates every other job's result. |
70 | 72 | AGGREGATOR_JOB = "required_checks" |
@@ -447,6 +449,150 @@ def load_filters(): |
447 | 449 | return module |
448 | 450 |
|
449 | 451 |
|
| 452 | +def load_ci_module(name, filename): |
| 453 | + """Import one of the hyphenated dev/ci scripts as a module.""" |
| 454 | + spec = importlib.util.spec_from_file_location(name, f"dev/ci/{filename}") |
| 455 | + module = importlib.util.module_from_spec(spec) |
| 456 | + spec.loader.exec_module(module) |
| 457 | + return module |
| 458 | + |
| 459 | + |
| 460 | +def ci_job_inputs(): |
| 461 | + """The `with:` inputs of every ci.yml job, parsed independently of the shell. |
| 462 | +
|
| 463 | + dev/local-ci.sh reads the same blocks with awk. Two implementations that |
| 464 | + have to agree is the whole point: neither one silently drifts. |
| 465 | + """ |
| 466 | + jobs = {} |
| 467 | + job = None |
| 468 | + in_with = False |
| 469 | + for line in (WORKFLOWS / "ci.yml").read_text(encoding="utf-8").splitlines(): |
| 470 | + match = re.match(r"^ ([A-Za-z0-9_-]+):\s*$", line) |
| 471 | + if match: |
| 472 | + job, in_with = match.group(1), False |
| 473 | + continue |
| 474 | + if job and re.match(r"^ with:\s*$", line): |
| 475 | + in_with = True |
| 476 | + continue |
| 477 | + if in_with: |
| 478 | + match = re.match(r"^ ([a-z][a-z0-9-]*):\s*(\S.*?)\s*$", line) |
| 479 | + if match: |
| 480 | + # YAML accepts either quote style, and so does the shell parser. |
| 481 | + jobs.setdefault(job, {})[match.group(1)] = match.group(2).strip("'\"") |
| 482 | + elif re.match(r"^ [a-z]", line): |
| 483 | + in_with = False |
| 484 | + return jobs |
| 485 | + |
| 486 | + |
| 487 | +def check_local_ci_config(): |
| 488 | + """dev/local-ci.sh parses ci.yml and dev/ci; make that drift fail here. |
| 489 | +
|
| 490 | + The script derives the Spark and Iceberg versions, the JDK, the Spark SQL |
| 491 | + matrix rows, the sbt projects those rows need, the Iceberg shard count and |
| 492 | + Scala default, and the version the DEDICATED_JVM_SBT_TESTS isolation is |
| 493 | + gated on. All of it comes from parsing the workflow files with awk and sed, |
| 494 | + so requoting `spark-full: '4.1.3'` or reindenting a `with:` block breaks it. |
| 495 | + Nothing else in preflight looks at that script -- actionlint runs with |
| 496 | + --shellcheck=off and only covers workflow files -- so without this the |
| 497 | + breakage surfaces the next time somebody actually needs the script, which is |
| 498 | + the worst moment. `--print-config` reports what it read; this asserts it |
| 499 | + against the same sources parsed independently, and syntax-checks the script. |
| 500 | + """ |
| 501 | + if not LOCAL_CI.exists(): |
| 502 | + return True |
| 503 | + |
| 504 | + syntax = subprocess.run( |
| 505 | + ["bash", "-n", str(LOCAL_CI)], capture_output=True, text=True, check=False |
| 506 | + ) |
| 507 | + if syntax.returncode != 0: |
| 508 | + print(f"local-ci: {LOCAL_CI} is not valid bash: {syntax.stderr.strip()}") |
| 509 | + return False |
| 510 | + |
| 511 | + printed = subprocess.run( |
| 512 | + ["bash", str(LOCAL_CI), "--print-config"], capture_output=True, text=True, check=False |
| 513 | + ) |
| 514 | + if printed.returncode != 0: |
| 515 | + print(f"local-ci: `{LOCAL_CI} --print-config` failed: {printed.stderr.strip()}") |
| 516 | + return False |
| 517 | + |
| 518 | + inputs = ci_job_inputs() |
| 519 | + policy = load_filters().POLICY |
| 520 | + rows = load_ci_module("spark_sql_modules", "spark-sql-modules.py").select("all") |
| 521 | + shards = load_ci_module("iceberg_shards", "check-iceberg-shards.py").SHARD_COUNT |
| 522 | + |
| 523 | + def newest_queue(prefix): |
| 524 | + # Only purely version-shaped keys: `spark_4_1_hive` is queue-tier too, |
| 525 | + # and the shell's `[0-9_]*` pattern excludes it for the same reason. |
| 526 | + versions = [ |
| 527 | + match.group(1).replace("_", ".") |
| 528 | + for key, events in policy.items() |
| 529 | + for match in [re.fullmatch(rf"{prefix}_(\d+(?:_\d+)*)", key)] |
| 530 | + if match and "queue" in events |
| 531 | + ] |
| 532 | + return max(versions, key=lambda v: [int(part) for part in v.split(".")]) |
| 533 | + |
| 534 | + expected = set() |
| 535 | + for job, given in inputs.items(): |
| 536 | + version = job.split("_", 1)[1].replace("_", ".") if "_" in job else job |
| 537 | + if job.startswith("spark_") and "spark-full" in given: |
| 538 | + expected.add(f"spark {version} full={given['spark-full']} java={given['java']}") |
| 539 | + elif job.startswith("iceberg_") and "iceberg-full" in given: |
| 540 | + expected.add( |
| 541 | + f"iceberg {version} full={given['iceberg-full']} " |
| 542 | + f"spark={given['spark-short']} java={given['java']}" |
| 543 | + ) |
| 544 | + expected.add(f"spark default={newest_queue('spark')}") |
| 545 | + expected.add(f"iceberg default={newest_queue('iceberg')}") |
| 546 | + expected.add("spark rows=" + ",".join(row["name"] for row in rows)) |
| 547 | + expected.add( |
| 548 | + "spark projects=" |
| 549 | + + ",".join(sorted({(row["args1"] or row["args2"]).split("/")[0] for row in rows})) |
| 550 | + ) |
| 551 | + expected.add(f"iceberg shards={shards}") |
| 552 | + |
| 553 | + reusable = (WORKFLOWS / "spark_sql_test_reusable.yml").read_text(encoding="utf-8") |
| 554 | + gate = re.search(r"DEDICATED_JVM_SBT_TESTS:.*?spark-short == '([^']*)'", reusable) |
| 555 | + expected.add(f"spark dedicated-jvm-gate={gate.group(1) if gate else ''}") |
| 556 | + |
| 557 | + iceberg_reusable = (WORKFLOWS / "iceberg_spark_test_reusable.yml").read_text(encoding="utf-8") |
| 558 | + scala = re.search(r"^ scala:.*?^ default:\s*'?([0-9.]+)'?", iceberg_reusable, re.S | re.M) |
| 559 | + expected.add(f"iceberg scala={scala.group(1) if scala else ''}") |
| 560 | + |
| 561 | + got = {line for line in printed.stdout.splitlines() if line.strip()} |
| 562 | + failures = [] |
| 563 | + for line in sorted(expected - got): |
| 564 | + failures.append(f"{LOCAL_CI} did not report `{line}`") |
| 565 | + for line in sorted(got - expected): |
| 566 | + failures.append(f"{LOCAL_CI} reported `{line}`, which does not match ci.yml or dev/ci") |
| 567 | + |
| 568 | + # Shape checks on top of the comparison above. Two parsers can share a blind |
| 569 | + # spot and agree on a broken value: both used to strip only single quotes, so |
| 570 | + # requoting `spark-full: "4.1.3"` gave them both `"4.1.3"` and the diff was |
| 571 | + # clean while the script built a URL with quotes in it. Assert the values |
| 572 | + # look like versions regardless of what either parser produced. |
| 573 | + shapes = [ |
| 574 | + r"^spark \d+\.\d+ full=\d+\.\d+\.\d+ java=\d+$", |
| 575 | + r"^iceberg \d+\.\d+ full=\d+\.\d+\.\d+ spark=\d+\.\d+ java=\d+$", |
| 576 | + r"^(spark|iceberg) default=\d+\.\d+$", |
| 577 | + r"^spark rows=[\w.-]+(,[\w.-]+)*$", |
| 578 | + r"^spark projects=[\w.-]+(,[\w.-]+)*$", |
| 579 | + r"^spark dedicated-jvm-gate=\d+\.\d+$", |
| 580 | + r"^iceberg scala=\d+\.\d+$", |
| 581 | + r"^iceberg shards=[1-9]\d*$", |
| 582 | + ] |
| 583 | + for line in sorted(got): |
| 584 | + if not any(re.match(shape, line) for shape in shapes): |
| 585 | + failures.append( |
| 586 | + f"{LOCAL_CI} reported `{line}`, which is not shaped like a " |
| 587 | + f"version. Two parsers can share a blind spot and agree on a " |
| 588 | + f"value the script cannot actually use, so the comparison above " |
| 589 | + f"is not enough on its own" |
| 590 | + ) |
| 591 | + for failure in failures: |
| 592 | + print(f"local-ci: {failure}") |
| 593 | + return not failures |
| 594 | + |
| 595 | + |
450 | 596 | def check_spark_sql_modules(): |
451 | 597 | """`--modules core` and `--modules hive` must partition `--modules all`. |
452 | 598 |
|
@@ -1016,6 +1162,7 @@ def check_nightly_base_fallback(): |
1016 | 1162 | ok = check_cache_refresh_scope() and ok |
1017 | 1163 | ok = check_nightly_scope() and ok |
1018 | 1164 | ok = check_nightly_base_fallback() and ok |
| 1165 | + ok = check_local_ci_config() and ok |
1019 | 1166 | if not ok: |
1020 | 1167 | sys.exit(1) |
1021 | 1168 | print("CI config checks passed") |
0 commit comments