@@ -52,11 +52,22 @@ def test_passed(case: str, sm: dict[str, str]) -> bool:
5252 ]
5353
5454
55+ def test_maintained (case : str , sm : dict [str , str ]) -> bool :
56+ """P2P semantics: a skipped test is not a regression, unlike for F2P."""
57+ key = _resolve_case (case , sm )
58+ return test_passed (case , sm ) or (
59+ key is not None and sm [key ] == TestStatus .SKIPPED .value
60+ )
61+
62+
5563def test_failed (case : str , sm : dict [str , str ]) -> bool :
5664 key = _resolve_case (case , sm )
5765 return key is None or sm [key ] in [
5866 TestStatus .FAILED .value ,
5967 TestStatus .ERROR .value ,
68+ # a skipped F2P test is not a resolution; without this, a patch that makes
69+ # every F2P test skip lands in neither list and scores RESOLVED_FULL
70+ TestStatus .SKIPPED .value ,
6071 ]
6172
6273
@@ -136,6 +147,12 @@ def check_pass_and_fail(test_case, eval_status_map, success, failed):
136147 elif test_failed (test_case , eval_status_map ):
137148 failed .append (test_case )
138149
150+ def check_maintained (test_case , eval_status_map , success , failed ):
151+ if test_maintained (test_case , eval_status_map ):
152+ success .append (test_case )
153+ elif test_failed (test_case , eval_status_map ):
154+ failed .append (test_case )
155+
139156 def check_fail_only (test_case , eval_status_map , success , failed ):
140157 if (
141158 test_case in eval_status_map
@@ -156,10 +173,13 @@ def check_fail_only(test_case, eval_status_map, success, failed):
156173 check_test_case (test_case , eval_status_map , f2p_success , f2p_failure )
157174
158175 # Calculate maintenance metrics
176+ check_p2p = (
177+ check_maintained if eval_type == EvalType .PASS_AND_FAIL else check_fail_only
178+ )
159179 p2p_success = []
160180 p2p_failure = []
161181 for test_case in gold_results [PASS_TO_PASS ]:
162- check_test_case (test_case , eval_status_map , p2p_success , p2p_failure )
182+ check_p2p (test_case , eval_status_map , p2p_success , p2p_failure )
163183
164184 results = {
165185 FAIL_TO_PASS : {
0 commit comments