@@ -75,6 +75,19 @@ class PlanTable(FixtureCase):
7575 ("a package's own test" , ["python/frameworks/alpha/tests/test_x.py" ], ["alpha" ], False , False ),
7676 ("the traceai package" , ["python/traceai/traceai/__init__.py" ], ["traceai" ], False , False ),
7777 ("a core file" , ["python/fi_instrumentation/otel.py" ], ALL_PACKAGES , False , True ),
78+ ("a nested core file" , ["python/fi_instrumentation/instrumentation/context_attributes.py" ], ALL_PACKAGES , False , True ),
79+ (
80+ "CI files and a core file together (core_install runs in every package job)" ,
81+ [
82+ ".github/workflows/python-tests.yml" ,
83+ ".github/scripts/known_broken_verdict.sh" ,
84+ ".github/scripts/test_python_test_plan.py" ,
85+ "python/fi_instrumentation/instrumentation/context_attributes.py" ,
86+ ],
87+ ALL_PACKAGES ,
88+ True ,
89+ True ,
90+ ),
7891 ("poetry.lock" , ["python/poetry.lock" ], ALL_PACKAGES , False , True ),
7992 ("python/pyproject.toml" , ["python/pyproject.toml" ], ALL_PACKAGES , False , True ),
8093 ("conftest.py" , ["python/tests/conftest.py" ], ["gamma" ], False , False ),
@@ -367,6 +380,9 @@ def test_a_diff_with_a_change_plans_it(self):
367380VERDICT = SCRIPTS_DIR / "known_broken_verdict.sh"
368381
369382
383+ INSTALL_FAILS_THE_CHECK = "a failed install fails the check, listed or not"
384+
385+
370386class KnownBrokenVerdict (unittest .TestCase ):
371387 BASE = {
372388 "PKG_NAME" : "pkg" ,
@@ -385,20 +401,44 @@ class KnownBrokenVerdict(unittest.TestCase):
385401 "tests pass after a failed install" ,
386402 {"TESTS" : "success" , "INSTALL_POETRY" : "failure" },
387403 1 ,
388- "failed in this run : install (poetry)" ,
404+ f"pkg : install (poetry) failed; { INSTALL_FAILS_THE_CHECK } " ,
389405 ),
390406 ("tests fail" , {"TESTS" : "failure" }, 0 , "known broken (the listed reason)" ),
391407 (
392408 "tests fail after failed poetry and core installs" ,
393409 {"TESTS" : "failure" , "INSTALL_POETRY" : "failure" , "CORE_INSTALL" : "failure" },
394- 0 ,
395- " install (poetry), fi_instrumentation install" ,
410+ 1 ,
411+ f"pkg: install (poetry), fi_instrumentation install failed; { INSTALL_FAILS_THE_CHECK } " ,
396412 ),
397413 (
398414 "tests fail after a failed hatch install" ,
399415 {"TESTS" : "failure" , "INSTALL_POETRY" : "skipped" , "INSTALL_HATCH" : "failure" },
400- 0 ,
401- "install (hatch)" ,
416+ 1 ,
417+ f"pkg: install (hatch) failed; { INSTALL_FAILS_THE_CHECK } " ,
418+ ),
419+ (
420+ "tests fail after a failed core install" ,
421+ {"TESTS" : "failure" , "CORE_INSTALL" : "failure" },
422+ 1 ,
423+ f"pkg: fi_instrumentation install failed; { INSTALL_FAILS_THE_CHECK } " ,
424+ ),
425+ (
426+ "tests skipped after a failed install (the real case: a failed install skips the Tests step)" ,
427+ {"TESTS" : "skipped" , "INSTALL_POETRY" : "failure" },
428+ 1 ,
429+ f"pkg: install (poetry) failed; { INSTALL_FAILS_THE_CHECK } " ,
430+ ),
431+ (
432+ "cancelled after a failed install" ,
433+ {"TESTS" : "cancelled" , "INSTALL_HATCH" : "failure" },
434+ 1 ,
435+ f"pkg: install (hatch) failed; { INSTALL_FAILS_THE_CHECK } " ,
436+ ),
437+ (
438+ "no tests, the tests pass, and a failed install" ,
439+ {"TESTS" : "success" , "OWN_TESTS" : "false" , "CORE_TESTS" : "" , "CORE_INSTALL" : "failure" },
440+ 1 ,
441+ INSTALL_FAILS_THE_CHECK ,
402442 ),
403443 ("tests skipped" , {"TESTS" : "skipped" }, 1 , "did not run" ),
404444 (
@@ -414,6 +454,7 @@ class KnownBrokenVerdict(unittest.TestCase):
414454 "its tests pass" ,
415455 ),
416456 ("cancelled" , {"TESTS" : "cancelled" }, 0 , "known broken" ),
457+ ("tests fail, no install failed (skipped installs)" , {"TESTS" : "failure" , "INSTALL_POETRY" : "skipped" , "CORE_INSTALL" : "skipped" }, 0 , "known broken (the listed reason)" ),
417458 ("no Tests outcome" , {"TESTS" : "" }, 1 , "unexpected Tests outcome" ),
418459 ("a misspelled outcome" , {"TESTS" : "succes" }, 1 , "unexpected Tests outcome" ),
419460 ]
@@ -436,6 +477,9 @@ def test_cases(self):
436477 output = done .stdout + done .stderr
437478 self .assertEqual (done .returncode , code , output )
438479 self .assertIn (phrase , output )
480+ # The old "may not reflect the listed reason" warning and "(failed in this run: ...)" note are gone.
481+ self .assertNotIn ("may not reflect" , output )
482+ self .assertNotIn ("failed in this run" , output )
439483
440484 def test_the_syntax_is_valid (self ):
441485 subprocess .run (["bash" , "-n" , str (VERDICT )], check = True )
@@ -581,6 +625,155 @@ def test_the_planner_tests_step_cannot_be_skipped_or_ignored(self):
581625 for line in step :
582626 self .assertFalse (line .startswith (("continue-on-error" , "if:" )), line )
583627
628+ def test_every_run_step_uses_bash_with_pipefail (self ):
629+ # `shell: bash` under the workflow's `defaults: run:` makes GitHub run every run step as
630+ # `bash --noprofile --norc -eo pipefail {0}` (a bare run step gets `bash -e {0}`, with no pipefail,
631+ # so `cmd | tee log` would hide a failed cmd). The block is exactly that, and the file has
632+ # exactly one `shell:` anywhere (flow style and comments included), so no job or step overrides it.
633+ lines = self .text .splitlines ()
634+ self .assertEqual (lines .count ("defaults:" ), 1 )
635+ start = lines .index ("defaults:" )
636+ self .assertEqual (lines [start : start + 3 ], ["defaults:" , " run:" , " shell: bash" ])
637+ self .assertFalse (lines [start + 3 ].startswith (" " ), "another key under defaults" )
638+ self .assertEqual (len (re .findall (r"\bshell[\"']?\s*:" , self .text )), 1 )
639+
640+ INSTALL_STEPS = (
641+ # (step id, step name, its if:, the install command that must stand alone on a line)
642+ (
643+ "install_poetry" ,
644+ "Install the package and its dev dependencies (poetry)" ,
645+ "if: matrix.package.backend == 'poetry'" ,
646+ "poetry install --no-interaction" ,
647+ ),
648+ (
649+ "install_hatch" ,
650+ "Install the package and its dev dependencies (hatch)" ,
651+ "if: matrix.package.backend == 'hatch'" ,
652+ '.venv/bin/pip install -e ".[dev]"' ,
653+ ),
654+ (
655+ "core_install" ,
656+ "Test against this repo's fi_instrumentation" ,
657+ "if: needs.plan.outputs.core_changed == 'true'" ,
658+ '"$GITHUB_WORKSPACE/$PKG_DIR/.venv/bin/pip" install -e "$GITHUB_WORKSPACE/python"' ,
659+ ),
660+ )
661+
662+ # The commands of each install step, exactly as reviewed (comments and indentation aside). A line
663+ # filter cannot see a multi-line wrapper that exempts the install from errexit (`{ ...; } && ...`,
664+ # a function called from `if`, `trap ... ERR`, `bash -c '...'`), so the whole script is pinned:
665+ # changing an install step means changing this table, in the same diff.
666+ INSTALL_SCRIPTS = {
667+ "install_poetry" : [
668+ 'pip install "poetry==$POETRY_VERSION"' ,
669+ "poetry install --no-interaction" ,
670+ "for pin in pytest==8.4.2 pytest-asyncio==0.24.0 pytest-mock==3.14.0; do" ,
671+ 'module=$(echo "${pin%%==*}" | tr - _)' ,
672+ 'if poetry run python -c "import $module" 2>/dev/null; then' ,
673+ 'echo "$module: already present"' ,
674+ "else" ,
675+ 'echo "$module: added $pin"' ,
676+ 'poetry run pip install "$pin"' ,
677+ "fi" ,
678+ "done" ,
679+ ],
680+ "install_hatch" : [
681+ "python -m venv .venv" ,
682+ '.venv/bin/pip install -e ".[dev]"' ,
683+ "for pin in pytest==8.4.2 pytest-asyncio==0.24.0 pytest-mock==3.14.0; do" ,
684+ 'module=$(echo "${pin%%==*}" | tr - _)' ,
685+ 'if .venv/bin/python -c "import $module" 2>/dev/null; then' ,
686+ 'echo "$module: already present"' ,
687+ "else" ,
688+ 'echo "$module: added $pin"' ,
689+ '.venv/bin/pip install "$pin"' ,
690+ "fi" ,
691+ "done" ,
692+ ],
693+ "core_install" : [
694+ '"$GITHUB_WORKSPACE/$PKG_DIR/.venv/bin/pip" install -e "$GITHUB_WORKSPACE/python"' ,
695+ ],
696+ }
697+
698+ def test_the_install_scripts_are_exactly_as_reviewed (self ):
699+ for step_id , name , _condition , _command in self .INSTALL_STEPS :
700+ with self .subTest (step_id ):
701+ step = self .step (name , containing = f"id: { step_id } " )
702+ commands = [line for line in self .run_lines (step ) if line and not line .startswith ("#" )]
703+ self .assertEqual (commands , self .INSTALL_SCRIPTS [step_id ])
704+
705+ @staticmethod
706+ def run_lines (step ):
707+ """The lines of the step's `run: |` block (the run block is the last key of the install steps)."""
708+ index = step .index ("run: |" )
709+ return step [index + 1 :]
710+
711+ def test_a_failed_install_fails_its_step_for_a_listed_package_too (self ):
712+ # With continue-on-error on an install step, a failed install of a listed package would not
713+ # fail the job, and the Tests step would run on a broken install. So, for each install step:
714+ # - no continue-on-error, no `shell:` and exactly the pinned `if:`;
715+ # - over its run lines (comments included): no `||`, no `|` outside a `$( ... )` span (so no
716+ # `cmd | tee log`; a `$(...)` is stripped before looking, nested ones are not handled), and
717+ # no `set +` (so no `set +e` and no `set +o errexit`);
718+ # - the install command is a line of its own, exactly: an `if ! ...; then` wrapper, a trailing
719+ # comment or a pipe on that line is not the exact line, so it fails.
720+ for step_id , name , condition , command in self .INSTALL_STEPS :
721+ with self .subTest (step_id ):
722+ step = self .step (name , containing = f"id: { step_id } " )
723+ self .assertEqual ([line for line in step if line .startswith ("continue-on-error" )], [], step )
724+ self .assertEqual ([line for line in step if re .search (r"\bshell[\"']?\s*:" , line )], [], step )
725+ self .assertEqual ([line for line in step if line .startswith ("if:" )], [condition ], step )
726+ run = self .run_lines (step )
727+ self .assertIn (command , run )
728+ self .assertEqual (run .count (command ), 1 , run )
729+ self .assertEqual ([line for line in run if "||" in line ], [], run )
730+ outside = [re .sub (r"\$\([^)]*\)" , "" , line ) for line in run ]
731+ self .assertEqual ([line for line in outside if "|" in line ], [], run )
732+ self .assertEqual ([line for line in run if "set +" in line ], [], run )
733+ # Only the Tests step may be allowed to fail (for a listed package).
734+ tests = self .step ("Tests" , containing = "id: tests" )
735+ self .assertEqual (tests .count ("continue-on-error: ${{ matrix.package.known_broken }}" ), 1 )
736+ self .assertEqual (sum (line .startswith ("continue-on-error" ) for line in self .lines ()), 1 )
737+
738+ def test_the_tests_step_is_skipped_after_a_failed_install (self ):
739+ # The Tests step (id: tests) has no `if:`, so GitHub gives it the default success(): a failed
740+ # install step skips it, and a broken install never gets its tests run.
741+ tests = self .step ("Tests" , containing = "id: tests" )
742+ self .assertEqual ([line for line in tests if line .startswith ("if:" )], [], tests )
743+
744+ def test_every_job_has_a_time_limit (self ):
745+ # Every job under `jobs:` has exactly one job-level `timeout-minutes: N` (4 spaces, N a positive
746+ # integer; a step's own timeout does not bound the job) and an entry in this table, so a new job
747+ # must get both.
748+ limits = {"plan" : 10 , "fi-instrumentation" : 30 , "package" : 45 , "result" : 5 }
749+ lines = self .text .splitlines ()
750+
751+ def is_comment (line ):
752+ return line .lstrip ().startswith ("#" )
753+
754+ # A comment line (at any indent) neither ends the jobs block nor a job, and a job key may carry
755+ # a trailing comment.
756+ start = lines .index ("jobs:" )
757+ end = next ((i for i in range (start + 1 , len (lines )) if re .match (r"^\S" , lines [i ]) and not is_comment (lines [i ])), len (lines ))
758+ found_keys = {}
759+ for i in range (start + 1 , end ):
760+ match = re .match (r"^ ([A-Za-z0-9_-]+):\s*(#.*)?$" , lines [i ])
761+ if match :
762+ found_keys [match .group (1 )] = i
763+ keys = list (found_keys )
764+ self .assertEqual (sorted (keys ), sorted (limits ), "a job without an entry in the table, or an entry without a job" )
765+ for key in keys :
766+ with self .subTest (key ):
767+ first = found_keys [key ]
768+ last = next (
769+ (i for i in range (first + 1 , len (lines )) if re .match (r"^ \S|^\S" , lines [i ]) and not is_comment (lines [i ])),
770+ len (lines ),
771+ )
772+ found = [line for line in lines [first :last ] if re .match (r"^ timeout-minutes:" , line )]
773+ self .assertEqual (len (found ), 1 , found )
774+ self .assertRegex (found [0 ], r"^ timeout-minutes: [1-9][0-9]*$" )
775+ self .assertEqual (found , [f" timeout-minutes: { limits [key ]} " ])
776+
584777 def test_the_verdict_step_runs_always_for_known_broken_packages (self ):
585778 step = self .step ("Known-broken verdict" )
586779 self .assertIn ("if: always() && matrix.package.known_broken" , step )
@@ -598,7 +791,7 @@ def test_core_tests_reach_the_tests_and_verdict_steps(self):
598791 def test_a_pr_base_missing_from_the_clone_is_fetched_then_everything_runs (self ):
599792 text = self .text
600793 check = 'if ! git cat-file -e "$PR_BASE_SHA^{commit}" 2>/dev/null; then'
601- fetch = 'git fetch --no-tags origin "$PR_BASE_SHA" || true'
794+ fetch = 'timeout 120 git fetch --no-tags origin "$PR_BASE_SHA" || true'
602795 message = 'echo "PR base $PR_BASE_SHA is not in the clone: treating every file as changed" >&2'
603796 diff = 'python3 .github/scripts/python_test_plan.py --diff "$PR_BASE_SHA...$PR_HEAD_SHA"'
604797 lines = self .lines ()
0 commit comments