Skip to content

Commit e7f8017

Browse files
committed
Recognize skipped pytest subtests in CI logs
1 parent 0d565c1 commit e7f8017

2 files changed

Lines changed: 58 additions & 2 deletions

File tree

‎toolshed/find_skipped_tests.py‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,10 @@
3636

3737
ANSI_ESCAPE = re.compile(r"\x1B\[[0-9;]*[A-Za-z]")
3838
PYTEST_NODE_ID = re.compile(r"tests/\S+\.py::\S+")
39-
PYTEST_TEST_OUTCOME = re.compile(r"(tests/\S+\.py::\S+)\s+(PASSED|FAILED|ERROR|SKIPPED|XFAIL|XPASS)\b")
39+
PYTEST_TEST_OUTCOME = re.compile(
40+
r"(tests/\S+\.py::\S+)\s+"
41+
r"(PASSED|FAILED|ERROR|SKIPPED|XFAIL|XPASS|SUBPASSED|SUBFAILED|SUBERROR|SUBSKIPPED|SUBXFAIL|SUBXPASS)\b"
42+
)
4043

4144
# GHA log format markers used to identify which test suite is active.
4245
# `gh api` logs: ##[group]<step-name> opens a section, ##[endgroup] closes it.
@@ -194,6 +197,8 @@ def extract_test_status_sets(text: str) -> tuple[set[str], set[str], dict[str, s
194197
"""Parse pytest output and return (skipped, non_skipped, test_id->suite)."""
195198
skipped: set[str] = set()
196199
non_skipped: set[str] = set()
200+
passed: set[str] = set()
201+
subtest_seen: set[str] = set()
197202
test_suites: dict[str, str] = {}
198203
current_suite = ""
199204

@@ -216,10 +221,20 @@ def extract_test_status_sets(text: str) -> tuple[set[str], set[str], dict[str, s
216221

217222
# Parse per-test outcomes first so PASS/FAIL lines disqualify tests.
218223
for test_id, outcome in PYTEST_TEST_OUTCOME.findall(line):
219-
if outcome == "SKIPPED":
224+
if outcome.startswith("SUB"):
225+
subtest_seen.add(test_id)
226+
if outcome == "SUBSKIPPED":
227+
skipped.add(test_id)
228+
if current_suite:
229+
test_suites.setdefault(test_id, current_suite)
230+
else:
231+
non_skipped.add(test_id)
232+
elif outcome == "SKIPPED":
220233
skipped.add(test_id)
221234
if current_suite:
222235
test_suites.setdefault(test_id, current_suite)
236+
elif outcome == "PASSED":
237+
passed.add(test_id)
223238
else:
224239
non_skipped.add(test_id)
225240

@@ -233,6 +248,11 @@ def extract_test_status_sets(text: str) -> tuple[set[str], set[str], dict[str, s
233248
if current_suite:
234249
test_suites.setdefault(test_id, current_suite)
235250

251+
# Pytest reports a passing parent after its subtests even when every
252+
# subtest skipped. Only treat that parent pass as execution evidence when
253+
# the test did not emit subtest outcomes of its own.
254+
non_skipped.update(passed - subtest_seen)
255+
236256
return skipped, non_skipped, test_suites
237257

238258

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
#
3+
# SPDX-License-Identifier: Apache-2.0
4+
5+
import pytest
6+
7+
from toolshed.find_skipped_tests import extract_test_status_sets
8+
9+
10+
@pytest.mark.agent_authored(model="gpt-5")
11+
def test_extract_test_status_sets_understands_subtest_parent_passes():
12+
log = """
13+
##[group]run-tests core
14+
tests/system/test_device.py::test_all_skipped SUBSKIPPED(i=0)
15+
tests/system/test_device.py::test_all_skipped SUBSKIPPED(i=1)
16+
tests/system/test_device.py::test_all_skipped PASSED
17+
tests/system/test_device.py::test_mixed SUBSKIPPED(i=0)
18+
tests/system/test_device.py::test_mixed SUBPASSED(i=1)
19+
tests/system/test_device.py::test_mixed PASSED
20+
tests/system/test_device.py::test_ordinary PASSED
21+
"""
22+
23+
skipped, non_skipped, test_suites = extract_test_status_sets(log)
24+
25+
assert skipped == {
26+
"tests/system/test_device.py::test_all_skipped",
27+
"tests/system/test_device.py::test_mixed",
28+
}
29+
assert non_skipped == {
30+
"tests/system/test_device.py::test_mixed",
31+
"tests/system/test_device.py::test_ordinary",
32+
}
33+
assert test_suites == {
34+
"tests/system/test_device.py::test_all_skipped": "cuda_core",
35+
"tests/system/test_device.py::test_mixed": "cuda_core",
36+
}

0 commit comments

Comments
 (0)