Skip to content

Commit 17e573e

Browse files
Merge pull request #278 from future-agi/ci/TH-8682-install-fails-check
ci: a failed install fails the check, and every job gets a time limit (TH-8682)
2 parents 708a623 + 55275fb commit 17e573e

6 files changed

Lines changed: 235 additions & 26 deletions

File tree

‎.github/python-tests-known-broken.txt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
# Python packages whose tests are known to fail today (see .github/workflows/python-tests.yml).
2-
# A listed package still runs in CI and shows a warning, but its failures do not fail the required "Python tests" check.
2+
# A listed package still runs in CI and shows a warning, but its test failures do not fail the required "Python tests" check.
3+
# A failed install fails the check, listed or not.
34
# If a listed package's tests all pass, CI fails until its line is removed from this file.
45
# So a PR that makes a listed package pass must remove its line in the same PR, or CI fails
56
# (for example, a LangChain change that fixes its tests). A listed package with no tests at all also fails: remove its line.

‎.github/scripts/known_broken_verdict.sh‎

Lines changed: 11 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -16,24 +16,27 @@ case "$TESTS" in
1616
exit 1
1717
;;
1818
esac
19-
# The Tests step is skipped only when an earlier step (checkout or setup) failed outright, or
19+
# A failed install fails the check, listed or not: the package's tests never ran against a working
20+
# install, so neither a pass nor a failure of them says anything about the listed reason.
21+
# The install steps have no continue-on-error, so a failed install skips the Tests step; this
22+
# check comes before the "skipped" one so the error names the install, whatever TESTS says.
23+
if [ -n "$install_failed" ]; then
24+
echo "::error::$PKG_NAME: $install_failed failed; a failed install fails the check, listed or not"
25+
exit 1
26+
fi
27+
# The Tests step is skipped when checkout or setup failed outright (a failed install is handled above), or
2028
# the job was cancelled: that is not a pass, and the job fails with it.
2129
if [ "$TESTS" = "skipped" ]; then
2230
echo "::error::$PKG_NAME is known broken ($REASON), but its tests did not run (an earlier step failed, or the job was cancelled)"
2331
exit 1
2432
fi
25-
# The tests passed (whatever an install step did): the line has to come off the list.
33+
# The tests passed: the line has to come off the list.
2634
if [ "$TESTS" = "success" ]; then
2735
if [ "$OWN_TESTS" != "true" ] && [ -z "$CORE_TESTS" ]; then
2836
echo "::error::$PKG_NAME is on the known-broken list but has no tests; remove its line from .github/python-tests-known-broken.txt"
2937
else
30-
echo "::error::$PKG_NAME is on the known-broken list but its tests pass; remove its line from .github/python-tests-known-broken.txt${install_failed:+ (failed in this run: $install_failed)}"
38+
echo "::error::$PKG_NAME is on the known-broken list but its tests pass; remove its line from .github/python-tests-known-broken.txt"
3139
fi
3240
exit 1
3341
fi
34-
# The tests failed. After a failed install that may not be the listed reason: say so.
35-
if [ -n "$install_failed" ]; then
36-
echo "::warning::$PKG_NAME is known broken, but in this run these steps failed: $install_failed; its test results may not reflect the listed reason ($REASON)"
37-
exit 0
38-
fi
3942
echo "::warning::$PKG_NAME is known broken ($REASON): its failures do not fail the check"

‎.github/scripts/python_test_plan.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,8 @@
3636
``known_broken`` and ``known_broken_reason`` (from ``.github/python-tests-known-broken.txt``:
3737
one ``<package dir name> | <reason>`` line per package; ``#`` comments and blank lines are
3838
ignored; the reason is "" for a package that is not listed). The workflow lets a listed
39-
package fail without failing the check, and fails when a listed package passes.
39+
package's tests fail without failing the check, fails when a listed package passes, and fails
40+
on any failed install, listed or not.
4041
4142
Paths from git are used exactly as written (stdin lines are stripped). Standard library only,
4243
plus git for --diff and --all. It fails loudly (non-zero exit, message on stderr) on a

‎.github/scripts/test_python_test_plan.py‎

Lines changed: 199 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,19 @@ class PlanTable(FixtureCase):
7575
("a package's own test", ["python/frameworks/alpha/tests/test_x.py"], ["alpha"], False, False),
7676
("the traceai package", ["python/traceai/traceai/__init__.py"], ["traceai"], False, False),
7777
("a core file", ["python/fi_instrumentation/otel.py"], ALL_PACKAGES, False, True),
78+
("a nested core file", ["python/fi_instrumentation/instrumentation/context_attributes.py"], ALL_PACKAGES, False, True),
79+
(
80+
"CI files and a core file together (core_install runs in every package job)",
81+
[
82+
".github/workflows/python-tests.yml",
83+
".github/scripts/known_broken_verdict.sh",
84+
".github/scripts/test_python_test_plan.py",
85+
"python/fi_instrumentation/instrumentation/context_attributes.py",
86+
],
87+
ALL_PACKAGES,
88+
True,
89+
True,
90+
),
7891
("poetry.lock", ["python/poetry.lock"], ALL_PACKAGES, False, True),
7992
("python/pyproject.toml", ["python/pyproject.toml"], ALL_PACKAGES, False, True),
8093
("conftest.py", ["python/tests/conftest.py"], ["gamma"], False, False),
@@ -367,6 +380,9 @@ def test_a_diff_with_a_change_plans_it(self):
367380
VERDICT = SCRIPTS_DIR / "known_broken_verdict.sh"
368381

369382

383+
INSTALL_FAILS_THE_CHECK = "a failed install fails the check, listed or not"
384+
385+
370386
class KnownBrokenVerdict(unittest.TestCase):
371387
BASE = {
372388
"PKG_NAME": "pkg",
@@ -385,20 +401,44 @@ class KnownBrokenVerdict(unittest.TestCase):
385401
"tests pass after a failed install",
386402
{"TESTS": "success", "INSTALL_POETRY": "failure"},
387403
1,
388-
"failed in this run: install (poetry)",
404+
f"pkg: install (poetry) failed; {INSTALL_FAILS_THE_CHECK}",
389405
),
390406
("tests fail", {"TESTS": "failure"}, 0, "known broken (the listed reason)"),
391407
(
392408
"tests fail after failed poetry and core installs",
393409
{"TESTS": "failure", "INSTALL_POETRY": "failure", "CORE_INSTALL": "failure"},
394-
0,
395-
"install (poetry), fi_instrumentation install",
410+
1,
411+
f"pkg: install (poetry), fi_instrumentation install failed; {INSTALL_FAILS_THE_CHECK}",
396412
),
397413
(
398414
"tests fail after a failed hatch install",
399415
{"TESTS": "failure", "INSTALL_POETRY": "skipped", "INSTALL_HATCH": "failure"},
400-
0,
401-
"install (hatch)",
416+
1,
417+
f"pkg: install (hatch) failed; {INSTALL_FAILS_THE_CHECK}",
418+
),
419+
(
420+
"tests fail after a failed core install",
421+
{"TESTS": "failure", "CORE_INSTALL": "failure"},
422+
1,
423+
f"pkg: fi_instrumentation install failed; {INSTALL_FAILS_THE_CHECK}",
424+
),
425+
(
426+
"tests skipped after a failed install (the real case: a failed install skips the Tests step)",
427+
{"TESTS": "skipped", "INSTALL_POETRY": "failure"},
428+
1,
429+
f"pkg: install (poetry) failed; {INSTALL_FAILS_THE_CHECK}",
430+
),
431+
(
432+
"cancelled after a failed install",
433+
{"TESTS": "cancelled", "INSTALL_HATCH": "failure"},
434+
1,
435+
f"pkg: install (hatch) failed; {INSTALL_FAILS_THE_CHECK}",
436+
),
437+
(
438+
"no tests, the tests pass, and a failed install",
439+
{"TESTS": "success", "OWN_TESTS": "false", "CORE_TESTS": "", "CORE_INSTALL": "failure"},
440+
1,
441+
INSTALL_FAILS_THE_CHECK,
402442
),
403443
("tests skipped", {"TESTS": "skipped"}, 1, "did not run"),
404444
(
@@ -414,6 +454,7 @@ class KnownBrokenVerdict(unittest.TestCase):
414454
"its tests pass",
415455
),
416456
("cancelled", {"TESTS": "cancelled"}, 0, "known broken"),
457+
("tests fail, no install failed (skipped installs)", {"TESTS": "failure", "INSTALL_POETRY": "skipped", "CORE_INSTALL": "skipped"}, 0, "known broken (the listed reason)"),
417458
("no Tests outcome", {"TESTS": ""}, 1, "unexpected Tests outcome"),
418459
("a misspelled outcome", {"TESTS": "succes"}, 1, "unexpected Tests outcome"),
419460
]
@@ -436,6 +477,9 @@ def test_cases(self):
436477
output = done.stdout + done.stderr
437478
self.assertEqual(done.returncode, code, output)
438479
self.assertIn(phrase, output)
480+
# The old "may not reflect the listed reason" warning and "(failed in this run: ...)" note are gone.
481+
self.assertNotIn("may not reflect", output)
482+
self.assertNotIn("failed in this run", output)
439483

440484
def test_the_syntax_is_valid(self):
441485
subprocess.run(["bash", "-n", str(VERDICT)], check=True)
@@ -581,6 +625,155 @@ def test_the_planner_tests_step_cannot_be_skipped_or_ignored(self):
581625
for line in step:
582626
self.assertFalse(line.startswith(("continue-on-error", "if:")), line)
583627

628+
def test_every_run_step_uses_bash_with_pipefail(self):
629+
# `shell: bash` under the workflow's `defaults: run:` makes GitHub run every run step as
630+
# `bash --noprofile --norc -eo pipefail {0}` (a bare run step gets `bash -e {0}`, with no pipefail,
631+
# so `cmd | tee log` would hide a failed cmd). The block is exactly that, and the file has
632+
# exactly one `shell:` anywhere (flow style and comments included), so no job or step overrides it.
633+
lines = self.text.splitlines()
634+
self.assertEqual(lines.count("defaults:"), 1)
635+
start = lines.index("defaults:")
636+
self.assertEqual(lines[start : start + 3], ["defaults:", " run:", " shell: bash"])
637+
self.assertFalse(lines[start + 3].startswith(" "), "another key under defaults")
638+
self.assertEqual(len(re.findall(r"\bshell[\"']?\s*:", self.text)), 1)
639+
640+
INSTALL_STEPS = (
641+
# (step id, step name, its if:, the install command that must stand alone on a line)
642+
(
643+
"install_poetry",
644+
"Install the package and its dev dependencies (poetry)",
645+
"if: matrix.package.backend == 'poetry'",
646+
"poetry install --no-interaction",
647+
),
648+
(
649+
"install_hatch",
650+
"Install the package and its dev dependencies (hatch)",
651+
"if: matrix.package.backend == 'hatch'",
652+
'.venv/bin/pip install -e ".[dev]"',
653+
),
654+
(
655+
"core_install",
656+
"Test against this repo's fi_instrumentation",
657+
"if: needs.plan.outputs.core_changed == 'true'",
658+
'"$GITHUB_WORKSPACE/$PKG_DIR/.venv/bin/pip" install -e "$GITHUB_WORKSPACE/python"',
659+
),
660+
)
661+
662+
# The commands of each install step, exactly as reviewed (comments and indentation aside). A line
663+
# filter cannot see a multi-line wrapper that exempts the install from errexit (`{ ...; } && ...`,
664+
# a function called from `if`, `trap ... ERR`, `bash -c '...'`), so the whole script is pinned:
665+
# changing an install step means changing this table, in the same diff.
666+
INSTALL_SCRIPTS = {
667+
"install_poetry": [
668+
'pip install "poetry==$POETRY_VERSION"',
669+
"poetry install --no-interaction",
670+
"for pin in pytest==8.4.2 pytest-asyncio==0.24.0 pytest-mock==3.14.0; do",
671+
'module=$(echo "${pin%%==*}" | tr - _)',
672+
'if poetry run python -c "import $module" 2>/dev/null; then',
673+
'echo "$module: already present"',
674+
"else",
675+
'echo "$module: added $pin"',
676+
'poetry run pip install "$pin"',
677+
"fi",
678+
"done",
679+
],
680+
"install_hatch": [
681+
"python -m venv .venv",
682+
'.venv/bin/pip install -e ".[dev]"',
683+
"for pin in pytest==8.4.2 pytest-asyncio==0.24.0 pytest-mock==3.14.0; do",
684+
'module=$(echo "${pin%%==*}" | tr - _)',
685+
'if .venv/bin/python -c "import $module" 2>/dev/null; then',
686+
'echo "$module: already present"',
687+
"else",
688+
'echo "$module: added $pin"',
689+
'.venv/bin/pip install "$pin"',
690+
"fi",
691+
"done",
692+
],
693+
"core_install": [
694+
'"$GITHUB_WORKSPACE/$PKG_DIR/.venv/bin/pip" install -e "$GITHUB_WORKSPACE/python"',
695+
],
696+
}
697+
698+
def test_the_install_scripts_are_exactly_as_reviewed(self):
699+
for step_id, name, _condition, _command in self.INSTALL_STEPS:
700+
with self.subTest(step_id):
701+
step = self.step(name, containing=f"id: {step_id}")
702+
commands = [line for line in self.run_lines(step) if line and not line.startswith("#")]
703+
self.assertEqual(commands, self.INSTALL_SCRIPTS[step_id])
704+
705+
@staticmethod
706+
def run_lines(step):
707+
"""The lines of the step's `run: |` block (the run block is the last key of the install steps)."""
708+
index = step.index("run: |")
709+
return step[index + 1 :]
710+
711+
def test_a_failed_install_fails_its_step_for_a_listed_package_too(self):
712+
# With continue-on-error on an install step, a failed install of a listed package would not
713+
# fail the job, and the Tests step would run on a broken install. So, for each install step:
714+
# - no continue-on-error, no `shell:` and exactly the pinned `if:`;
715+
# - over its run lines (comments included): no `||`, no `|` outside a `$( ... )` span (so no
716+
# `cmd | tee log`; a `$(...)` is stripped before looking, nested ones are not handled), and
717+
# no `set +` (so no `set +e` and no `set +o errexit`);
718+
# - the install command is a line of its own, exactly: an `if ! ...; then` wrapper, a trailing
719+
# comment or a pipe on that line is not the exact line, so it fails.
720+
for step_id, name, condition, command in self.INSTALL_STEPS:
721+
with self.subTest(step_id):
722+
step = self.step(name, containing=f"id: {step_id}")
723+
self.assertEqual([line for line in step if line.startswith("continue-on-error")], [], step)
724+
self.assertEqual([line for line in step if re.search(r"\bshell[\"']?\s*:", line)], [], step)
725+
self.assertEqual([line for line in step if line.startswith("if:")], [condition], step)
726+
run = self.run_lines(step)
727+
self.assertIn(command, run)
728+
self.assertEqual(run.count(command), 1, run)
729+
self.assertEqual([line for line in run if "||" in line], [], run)
730+
outside = [re.sub(r"\$\([^)]*\)", "", line) for line in run]
731+
self.assertEqual([line for line in outside if "|" in line], [], run)
732+
self.assertEqual([line for line in run if "set +" in line], [], run)
733+
# Only the Tests step may be allowed to fail (for a listed package).
734+
tests = self.step("Tests", containing="id: tests")
735+
self.assertEqual(tests.count("continue-on-error: ${{ matrix.package.known_broken }}"), 1)
736+
self.assertEqual(sum(line.startswith("continue-on-error") for line in self.lines()), 1)
737+
738+
def test_the_tests_step_is_skipped_after_a_failed_install(self):
739+
# The Tests step (id: tests) has no `if:`, so GitHub gives it the default success(): a failed
740+
# install step skips it, and a broken install never gets its tests run.
741+
tests = self.step("Tests", containing="id: tests")
742+
self.assertEqual([line for line in tests if line.startswith("if:")], [], tests)
743+
744+
def test_every_job_has_a_time_limit(self):
745+
# Every job under `jobs:` has exactly one job-level `timeout-minutes: N` (4 spaces, N a positive
746+
# integer; a step's own timeout does not bound the job) and an entry in this table, so a new job
747+
# must get both.
748+
limits = {"plan": 10, "fi-instrumentation": 30, "package": 45, "result": 5}
749+
lines = self.text.splitlines()
750+
751+
def is_comment(line):
752+
return line.lstrip().startswith("#")
753+
754+
# A comment line (at any indent) neither ends the jobs block nor a job, and a job key may carry
755+
# a trailing comment.
756+
start = lines.index("jobs:")
757+
end = next((i for i in range(start + 1, len(lines)) if re.match(r"^\S", lines[i]) and not is_comment(lines[i])), len(lines))
758+
found_keys = {}
759+
for i in range(start + 1, end):
760+
match = re.match(r"^ ([A-Za-z0-9_-]+):\s*(#.*)?$", lines[i])
761+
if match:
762+
found_keys[match.group(1)] = i
763+
keys = list(found_keys)
764+
self.assertEqual(sorted(keys), sorted(limits), "a job without an entry in the table, or an entry without a job")
765+
for key in keys:
766+
with self.subTest(key):
767+
first = found_keys[key]
768+
last = next(
769+
(i for i in range(first + 1, len(lines)) if re.match(r"^ \S|^\S", lines[i]) and not is_comment(lines[i])),
770+
len(lines),
771+
)
772+
found = [line for line in lines[first:last] if re.match(r"^ timeout-minutes:", line)]
773+
self.assertEqual(len(found), 1, found)
774+
self.assertRegex(found[0], r"^ timeout-minutes: [1-9][0-9]*$")
775+
self.assertEqual(found, [f" timeout-minutes: {limits[key]}"])
776+
584777
def test_the_verdict_step_runs_always_for_known_broken_packages(self):
585778
step = self.step("Known-broken verdict")
586779
self.assertIn("if: always() && matrix.package.known_broken", step)
@@ -598,7 +791,7 @@ def test_core_tests_reach_the_tests_and_verdict_steps(self):
598791
def test_a_pr_base_missing_from_the_clone_is_fetched_then_everything_runs(self):
599792
text = self.text
600793
check = 'if ! git cat-file -e "$PR_BASE_SHA^{commit}" 2>/dev/null; then'
601-
fetch = 'git fetch --no-tags origin "$PR_BASE_SHA" || true'
794+
fetch = 'timeout 120 git fetch --no-tags origin "$PR_BASE_SHA" || true'
602795
message = 'echo "PR base $PR_BASE_SHA is not in the clone: treating every file as changed" >&2'
603796
diff = 'python3 .github/scripts/python_test_plan.py --diff "$PR_BASE_SHA...$PR_HEAD_SHA"'
604797
lines = self.lines()

0 commit comments

Comments
 (0)