|
27 | 27 | # could not be tested; POLICY_CASES below is the test it never had. The |
28 | 28 | # expected sets are transcribed from the `if:` expressions ci.yml carried |
29 | 29 | # before the policy moved, so a regression here is a behaviour change. |
| 30 | +# The event reaches the script as environment variables set by the |
| 31 | +# `Compute outputs` step of ci.yml's `Detect changes` job, and a variable |
| 32 | +# dropped there reads as an empty string, which is a valid value (no |
| 33 | +# label, no base branch), so that wiring is pinned as well. So is the |
| 34 | +# `refs/heads/main` guard on the `docs` job: a dispatch routes every job, |
| 35 | +# docs included, and that guard is all that keeps a dispatch on a release |
| 36 | +# branch from publishing that branch's docs over the site. |
30 | 37 | # |
31 | 38 | # 3. Required-check coverage. `Required Checks` in ci.yml is the job that |
32 | 39 | # `.asf.yaml` can name in `required_status_checks` for main. A heavy job |
|
390 | 397 | }, |
391 | 398 | set(), |
392 | 399 | ), |
| 400 | + # A pull request against a release branch runs the queue and nightly tiers |
| 401 | + # as well, because a release branch has no queue and no nightly to run them |
| 402 | + # later. The site deploy still never runs from a pull request, and the |
| 403 | + # deprecated Spark 3.4 suite still waits for its label. |
| 404 | + ( |
| 405 | + {"name": "pull_request", "action": "opened", "labels": [], "base": "branch-1.1"}, |
| 406 | + QUEUE_TIER | NIGHTLY_TIER, |
| 407 | + ), |
| 408 | + ( |
| 409 | + {"name": "pull_request", "action": "synchronize", "labels": [], "base": "branch-2.0"}, |
| 410 | + QUEUE_TIER | NIGHTLY_TIER, |
| 411 | + ), |
| 412 | + ( |
| 413 | + { |
| 414 | + "name": "pull_request", |
| 415 | + "action": "synchronize", |
| 416 | + "labels": ["run-spark-3.4-tests"], |
| 417 | + "base": "branch-1.1", |
| 418 | + }, |
| 419 | + QUEUE_TIER | NIGHTLY_TIER | SPARK_DEPRECATED, |
| 420 | + ), |
| 421 | + # There the commit run already covered everything a label gates except |
| 422 | + # Spark 3.4, so a `labeled` run adds Spark 3.4 or nothing. |
| 423 | + ( |
| 424 | + { |
| 425 | + "name": "pull_request", |
| 426 | + "action": "labeled", |
| 427 | + "label": "run-spark-3.4-tests", |
| 428 | + "labels": ["run-spark-3.4-tests"], |
| 429 | + "base": "branch-1.1", |
| 430 | + }, |
| 431 | + SPARK_DEPRECATED, |
| 432 | + ), |
| 433 | + ( |
| 434 | + { |
| 435 | + "name": "pull_request", |
| 436 | + "action": "labeled", |
| 437 | + "label": "run-iceberg-tests", |
| 438 | + "labels": ["run-iceberg-tests"], |
| 439 | + "base": "branch-1.1", |
| 440 | + }, |
| 441 | + set(), |
| 442 | + ), |
| 443 | + # Only `branch-N.M` is a release branch. A pull request against main, or |
| 444 | + # one stacked on another branch in the repository, keeps the PR tier. |
| 445 | + ({"name": "pull_request", "action": "synchronize", "labels": [], "base": "main"}, PR_TIER), |
| 446 | + ({"name": "pull_request", "action": "synchronize", "labels": [], "base": "pr-5654"}, PR_TIER), |
| 447 | + ( |
| 448 | + { |
| 449 | + "name": "pull_request", |
| 450 | + "action": "synchronize", |
| 451 | + "labels": [], |
| 452 | + "base": "branch-1.1-backports", |
| 453 | + }, |
| 454 | + PR_TIER, |
| 455 | + ), |
393 | 456 | ] |
394 | 457 |
|
395 | 458 |
|
@@ -654,6 +717,8 @@ def check_event_policy(): |
654 | 717 | label += f"/{event['action']}" |
655 | 718 | if event.get("label"): |
656 | 719 | label += f" +{event['label']}" |
| 720 | + if event.get("base"): |
| 721 | + label += f" base={event['base']}" |
657 | 722 | failures.append( |
658 | 723 | f"{label} labels={event.get('labels', [])}: " |
659 | 724 | f"unexpectedly allowed {sorted(actual - expected) or 'nothing'}, " |
@@ -688,6 +753,98 @@ def check_event_policy(): |
688 | 753 | return not failures |
689 | 754 |
|
690 | 755 |
|
| 756 | +# The environment variables `event_from_env` in compute-changes.py reads, and |
| 757 | +# the `github` context expression ci.yml's `Compute outputs` step must set each |
| 758 | +# one from. A typo in an expression is as silent as a missing variable: both |
| 759 | +# arrive as an empty string. |
| 760 | +EVENT_ENV_SOURCES = { |
| 761 | + "EVENT_NAME": "github.event_name", |
| 762 | + "EVENT_ACTION": "github.event.action", |
| 763 | + "LABEL_NAME": "github.event.label.name", |
| 764 | + "PR_LABELS": "toJSON(github.event.pull_request.labels.*.name)", |
| 765 | + "PR_BASE_REF": "github.event.pull_request.base.ref", |
| 766 | +} |
| 767 | +ENV_READ = re.compile(r'os\.environ\.get\("([A-Z_]+)"') |
| 768 | +ENV_SET = re.compile(r"^\s+([A-Z_]+):\s+\$\{\{\s*(.+?)\s*\}\}\s*$") |
| 769 | + |
| 770 | + |
| 771 | +def compute_step_env(): |
| 772 | + """{variable: expression} from the `env:` of ci.yml's `Compute outputs` step.""" |
| 773 | + env, in_step, in_env = {}, False, False |
| 774 | + for line in CI_WORKFLOW.read_text(encoding="utf-8").splitlines(): |
| 775 | + if re.match(r"^\s+- name: Compute outputs\s*$", line): |
| 776 | + in_step = True |
| 777 | + elif in_step and not in_env: |
| 778 | + if re.match(r"^\s+- ", line): |
| 779 | + break # the next step; this one has no `env:` |
| 780 | + in_env = bool(re.match(r"^\s+env:\s*$", line)) |
| 781 | + elif in_env: |
| 782 | + match = ENV_SET.match(line) |
| 783 | + if not match: |
| 784 | + break |
| 785 | + env[match.group(1)] = match.group(2) |
| 786 | + return env |
| 787 | + |
| 788 | + |
| 789 | +def check_event_env(): |
| 790 | + """Every variable POLICY reads is set by `Detect changes`, from the right context. |
| 791 | +
|
| 792 | + Without PR_BASE_REF, for instance, every pull request against a release |
| 793 | + branch would quietly fall back to the PR tier and every check would pass. |
| 794 | + """ |
| 795 | + failures = [] |
| 796 | + source = Path("dev/ci/compute-changes.py").read_text(encoding="utf-8") |
| 797 | + body = source.split("\ndef event_from_env", 1)[1].split("\ndef ", 1)[0] |
| 798 | + read = set(ENV_READ.findall(body)) |
| 799 | + env = compute_step_env() |
| 800 | + for name in sorted(read - set(EVENT_ENV_SOURCES)): |
| 801 | + failures.append( |
| 802 | + f"event_from_env reads {name}, which EVENT_ENV_SOURCES does not " |
| 803 | + f"list; add the `github` context expression it comes from" |
| 804 | + ) |
| 805 | + for name in sorted(set(EVENT_ENV_SOURCES) - read): |
| 806 | + failures.append(f"EVENT_ENV_SOURCES lists {name}, which event_from_env no longer reads") |
| 807 | + for name in sorted(read & set(EVENT_ENV_SOURCES)): |
| 808 | + expected = EVENT_ENV_SOURCES[name] |
| 809 | + if name not in env: |
| 810 | + failures.append( |
| 811 | + f"{CI_WORKFLOW}: the `Compute outputs` step does not set {name}, " |
| 812 | + f"so compute-changes.py reads it as empty" |
| 813 | + ) |
| 814 | + elif env[name] != expected: |
| 815 | + failures.append( |
| 816 | + f"{CI_WORKFLOW}: the `Compute outputs` step sets {name} from " |
| 817 | + f"`{env[name]}`, not `{expected}`" |
| 818 | + ) |
| 819 | + for failure in failures: |
| 820 | + print(f"event env: {failure}") |
| 821 | + return not failures |
| 822 | + |
| 823 | + |
| 824 | +# The site deploy's job-level `if:` in ci.yml. It has to be on the job rather |
| 825 | +# than in POLICY, because a dispatch routes every job (see POLICY_CASES). |
| 826 | +DOCS_JOB = "docs" |
| 827 | +DOCS_MAIN_GUARD = re.compile(r"^ if:.*github\.ref\s*==\s*'refs/heads/main'") |
| 828 | + |
| 829 | + |
| 830 | +def check_docs_deploy_guard(): |
| 831 | + """The site deploy runs from main only, whatever the event. |
| 832 | +
|
| 833 | + docs.yaml rsyncs the built site over asf-site with --delete and falls back |
| 834 | + to `git push --force`, and the release process dispatches ci.yml on the |
| 835 | + release branch before every release candidate. |
| 836 | + """ |
| 837 | + _, guarded = guarded_jobs(CI_WORKFLOW, DOCS_MAIN_GUARD) |
| 838 | + if DOCS_JOB in guarded: |
| 839 | + return True |
| 840 | + print( |
| 841 | + f"docs deploy: the `{DOCS_JOB}` job in {CI_WORKFLOW} must require " |
| 842 | + f"github.ref == 'refs/heads/main' in its `if:`, or a dispatch on a " |
| 843 | + f"release branch publishes that branch's docs over the site" |
| 844 | + ) |
| 845 | + return False |
| 846 | + |
| 847 | + |
691 | 848 | def artifact_names(path): |
692 | 849 | """Return ([upload names], [download names]) for one workflow file.""" |
693 | 850 | uploads, downloads = [], [] |
@@ -1277,6 +1434,8 @@ def check_cache_save_scope(): |
1277 | 1434 | if __name__ == "__main__": |
1278 | 1435 | ok = check_change_filters() |
1279 | 1436 | ok = check_event_policy() and ok |
| 1437 | + ok = check_event_env() and ok |
| 1438 | + ok = check_docs_deploy_guard() and ok |
1280 | 1439 | ok = check_spark_sql_modules() and ok |
1281 | 1440 | ok = check_linux_test_profiles() and ok |
1282 | 1441 | ok = check_artifact_names() and ok |
|
0 commit comments