fix(evals): exclude api_req_rate_limit_wait from isApiUnstable check

roomote · roomote · commit a77b429a158b · 2025-12-30T02:10:38.000Z
User-configured rate limiting is not an error/retry signal and should not
mark the API as unstable in eval runner.
diff --git a/packages/evals/src/cli/runTask.ts b/packages/evals/src/cli/runTask.ts
@@ -315,7 +315,7 @@ export const runTask = async ({ run, task, publish, logger, jobToken }: RunTaskO
 		if (
 			eventName === RooCodeEventName.Message &&
 			payload[0].message.say &&
-			["api_req_rate_limit_wait", "api_req_retry_delayed", "api_req_retried"].includes(payload[0].message.say)
+			["api_req_retry_delayed", "api_req_retried"].includes(payload[0].message.say)
 		) {
 			isApiUnstable = true
 		}

Original file line number	Diff line number	Diff line change
`@@ -315,7 +315,7 @@ export const runTask = async ({ run, task, publish, logger, jobToken }: RunTaskO`
`315`	`315`	`if (`
`316`	`316`	`eventName === RooCodeEventName.Message &&`
`317`	`317`	`payload[0].message.say &&`
`318`		`- ["api_req_rate_limit_wait", "api_req_retry_delayed", "api_req_retried"].includes(payload[0].message.say)`
	`318`	`+ ["api_req_retry_delayed", "api_req_retried"].includes(payload[0].message.say)`
`319`	`319`	`) {`
`320`	`320`	`isApiUnstable = true`
`321`	`321`	`}`