Skip to content

Commit a5ecda6

Browse files
yinli-systemsjohn-b-yang
authored andcommitted
Fix pytest skipped test grading (#598)
1 parent d2d91b1 commit a5ecda6

4 files changed

Lines changed: 149 additions & 2 deletions

File tree

‎swebench/harness/grading.py‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,11 +52,22 @@ def test_passed(case: str, sm: dict[str, str]) -> bool:
5252
]
5353

5454

55+
def test_maintained(case: str, sm: dict[str, str]) -> bool:
56+
"""P2P semantics: a skipped test is not a regression, unlike for F2P."""
57+
key = _resolve_case(case, sm)
58+
return test_passed(case, sm) or (
59+
key is not None and sm[key] == TestStatus.SKIPPED.value
60+
)
61+
62+
5563
def test_failed(case: str, sm: dict[str, str]) -> bool:
5664
key = _resolve_case(case, sm)
5765
return key is None or sm[key] in [
5866
TestStatus.FAILED.value,
5967
TestStatus.ERROR.value,
68+
# a skipped F2P test is not a resolution; without this, a patch that makes
69+
# every F2P test skip lands in neither list and scores RESOLVED_FULL
70+
TestStatus.SKIPPED.value,
6071
]
6172

6273

@@ -136,6 +147,12 @@ def check_pass_and_fail(test_case, eval_status_map, success, failed):
136147
elif test_failed(test_case, eval_status_map):
137148
failed.append(test_case)
138149

150+
def check_maintained(test_case, eval_status_map, success, failed):
151+
if test_maintained(test_case, eval_status_map):
152+
success.append(test_case)
153+
elif test_failed(test_case, eval_status_map):
154+
failed.append(test_case)
155+
139156
def check_fail_only(test_case, eval_status_map, success, failed):
140157
if (
141158
test_case in eval_status_map
@@ -156,10 +173,13 @@ def check_fail_only(test_case, eval_status_map, success, failed):
156173
check_test_case(test_case, eval_status_map, f2p_success, f2p_failure)
157174

158175
# Calculate maintenance metrics
176+
check_p2p = (
177+
check_maintained if eval_type == EvalType.PASS_AND_FAIL else check_fail_only
178+
)
159179
p2p_success = []
160180
p2p_failure = []
161181
for test_case in gold_results[PASS_TO_PASS]:
162-
check_test_case(test_case, eval_status_map, p2p_success, p2p_failure)
182+
check_p2p(test_case, eval_status_map, p2p_success, p2p_failure)
163183

164184
results = {
165185
FAIL_TO_PASS: {

‎swebench/harness/log_parsers/python.py‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,20 @@
33
from swebench.harness.constants import TestStatus
44
from swebench.types import TestSpec
55

6+
_SKIP_SUMMARY_COUNT = re.compile(r"^\[\d+\]$")
7+
8+
9+
def _is_skip_summary(status: str, name: str) -> bool:
10+
"""True for pytest's ``SKIPPED [N] path:line: reason`` summary, where [N] is not a test.
11+
12+
Deliberately scoped to SKIPPED. Wrapped progress lines produce a similar
13+
``PASSED [100%]`` artifact, but PASS_TO_PASS for pytest-dev__pytest-5262 and
14+
-7521 literally expects ``[100%]``, so dropping it fails those instances.
15+
TODO(john-b-yang): repair those two P2P lists, then widen this to any
16+
bare bracketed count.
17+
"""
18+
return status == TestStatus.SKIPPED.value and bool(_SKIP_SUMMARY_COUNT.match(name))
19+
620

721
def parse_log_pytest(log: str, test_spec: TestSpec) -> dict[str, str]:
822
"""
@@ -22,6 +36,8 @@ def parse_log_pytest(log: str, test_spec: TestSpec) -> dict[str, str]:
2236
test_case = line.split()
2337
if len(test_case) <= 1:
2438
continue
39+
if _is_skip_summary(test_case[0], test_case[1]):
40+
continue
2541
test_status_map[test_case[1]] = test_case[0]
2642
return test_status_map
2743

@@ -45,6 +61,8 @@ def parse_log_pytest_options(log: str, test_spec: TestSpec) -> dict[str, str]:
4561
test_case = line.split()
4662
if len(test_case) <= 1:
4763
continue
64+
if _is_skip_summary(test_case[0], test_case[1]):
65+
continue
4866
has_option = option_pattern.search(test_case[1])
4967
if has_option:
5068
main, option = has_option.groups()
@@ -161,7 +179,7 @@ def parse_log_pytest_v2(log: str, test_spec: TestSpec) -> dict[str, str]:
161179
# drop the trailing " - <assertion message>" so it can't enter the id
162180
line = line.split(" - ", 1)[0]
163181
test_case = line.split()
164-
if len(test_case) >= 2:
182+
if len(test_case) >= 2 and not _is_skip_summary(test_case[0], test_case[1]):
165183
test_status_map[" ".join(test_case[1:])] = test_case[0]
166184
# Support older pytest versions by checking if the line ends with the test status
167185
elif any([line.endswith(x.value) for x in TestStatus]):
@@ -248,6 +266,8 @@ def parse_log_matplotlib(log: str, test_spec: TestSpec) -> dict[str, str]:
248266
test_case = line.split()
249267
if len(test_case) <= 1:
250268
continue
269+
if _is_skip_summary(test_case[0], test_case[1]):
270+
continue
251271
test_status_map[test_case[1]] = test_case[0]
252272
return test_status_map
253273

‎tests/test_grading_skipped.py‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
"""Tests for skipped-test grading (#545, PR #598).
2+
3+
A skipped FAIL_TO_PASS test is not a resolution; a skipped PASS_TO_PASS test is
4+
not a regression. Before this, skipped tests landed in neither the success nor
5+
failure list, so an all-skipped F2P set scored RESOLVED_FULL off an empty ratio.
6+
"""
7+
8+
from swebench.harness.constants import (
9+
FAIL_TO_FAIL,
10+
FAIL_TO_PASS,
11+
PASS_TO_FAIL,
12+
PASS_TO_PASS,
13+
EvalType,
14+
ResolvedStatus,
15+
TestStatus,
16+
)
17+
from swebench.harness.grading import get_eval_tests_report, get_resolution_status
18+
from swebench.harness.grading import test_failed as grade_failed
19+
from swebench.harness.grading import test_maintained as grade_maintained
20+
21+
PASSED = TestStatus.PASSED.value
22+
FAILED = TestStatus.FAILED.value
23+
SKIPPED = TestStatus.SKIPPED.value
24+
25+
26+
def _gold(f2p, p2p):
27+
return {
28+
FAIL_TO_PASS: f2p,
29+
PASS_TO_PASS: p2p,
30+
FAIL_TO_FAIL: [],
31+
PASS_TO_FAIL: [],
32+
}
33+
34+
35+
def test_skipped_f2p_is_a_failure():
36+
assert grade_failed("t", {"t": SKIPPED})
37+
38+
39+
def test_skipped_p2p_is_maintained():
40+
assert grade_maintained("t", {"t": SKIPPED})
41+
assert grade_maintained("t", {"t": PASSED})
42+
assert not grade_maintained("t", {"t": FAILED})
43+
44+
45+
def test_all_skipped_f2p_is_not_resolved():
46+
"""Regression: skipping every F2P test used to score RESOLVED_FULL."""
47+
sm = {"t_a": SKIPPED, "t_b": SKIPPED, "t_c": PASSED}
48+
report = get_eval_tests_report(
49+
sm, _gold(["t_a", "t_b"], ["t_c"]), eval_type=EvalType.PASS_AND_FAIL
50+
)
51+
assert report[FAIL_TO_PASS]["success"] == []
52+
assert report[FAIL_TO_PASS]["failure"] == ["t_a", "t_b"]
53+
assert get_resolution_status(report) == ResolvedStatus.NO.value
54+
55+
56+
def test_skipped_p2p_does_not_block_resolution():
57+
sm = {"t_a": PASSED, "t_c": SKIPPED}
58+
report = get_eval_tests_report(
59+
sm, _gold(["t_a"], ["t_c"]), eval_type=EvalType.PASS_AND_FAIL
60+
)
61+
assert report[PASS_TO_PASS]["success"] == ["t_c"]
62+
assert get_resolution_status(report) == ResolvedStatus.FULL.value
Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
"""pytest's ``SKIPPED [N] path:line: reason`` summary is not a test (#598).
2+
3+
All four pytest-family parsers in log_parsers/python.py had their own copy of the
4+
loop, so all four recorded ``[N]`` as a fake test id.
5+
"""
6+
7+
from swebench.harness.constants import TestStatus
8+
from swebench.harness.log_parsers.python import (
9+
parse_log_matplotlib,
10+
parse_log_pytest,
11+
parse_log_pytest_options,
12+
parse_log_pytest_v2,
13+
)
14+
15+
PASSED = TestStatus.PASSED.value
16+
17+
LOG = """SKIPPED [1] tests/test_a.py:12: needs network
18+
PASSED tests/test_a.py::test_real
19+
SKIPPED [23] tests/test_b.py:4: unsupported platform
20+
"""
21+
22+
23+
def test_skip_summary_excluded_from_all_pytest_parsers():
24+
for parser in (
25+
parse_log_pytest,
26+
parse_log_pytest_options,
27+
parse_log_pytest_v2,
28+
parse_log_matplotlib,
29+
):
30+
result = parser(LOG, None)
31+
assert result == {"tests/test_a.py::test_real": PASSED}, parser.__name__
32+
33+
34+
def test_genuine_skipped_test_is_still_recorded():
35+
"""A real ``SKIPPED <nodeid>`` line must survive the guard."""
36+
log = "SKIPPED tests/test_a.py::test_flaky"
37+
assert parse_log_pytest(log, None) == {
38+
"tests/test_a.py::test_flaky": TestStatus.SKIPPED.value
39+
}
40+
41+
42+
def test_wrapped_progress_artifact_is_preserved():
43+
"""``PASSED [100%]`` must survive: PASS_TO_PASS for pytest-dev__pytest-5262
44+
and -7521 literally expects ``[100%]``, so dropping it fails those gold instances."""
45+
assert parse_log_pytest("PASSED [100%]", None) == {"[100%]": PASSED}

0 commit comments

Comments
 (0)