Skip to content

Commit 53d39fa

Browse files
committed
fix(runtime): repair reconcile contract, sticky cancelling, bound constants, blocking log tail
- Client reconcile() accepted only Ack while the server answers Jobs; accept Jobs and return the reconciled job list. - Late Progress(Running) after a cancel flipped cancelling back to running; mirror the Accepted-branch cancel_requested guard and add a Postgres-gated regression test. - Use MAX_PENDING_PER_CHANNEL / MAX_INBOX_RETRIES instead of drifting 500 / 10 literals in enqueue, requeue, and recover. - Wrap tail_rotating_log in spawn_blocking so the async control handler no longer blocks a tokio worker on sync file reads. Signed-off-by: Jacques Wainwright <[email protected]>
1 parent b57e247 commit 53d39fa

4 files changed

Lines changed: 81 additions & 19 deletions

File tree

‎crates/buzz-acp/src/job_supervisor.rs‎

Lines changed: 9 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -884,17 +884,15 @@ impl JobSupervisor {
884884
let lines = lines.unwrap_or(100).min(MAX_LOG_TAIL_LINES);
885885
let attempt_dir = job_attempt_dir(&self.inner.runtime.state_dir, job.job_id, job.attempt)
886886
.map_err(|error| control("logs_unavailable", error.to_string()))?;
887-
let stdout = tail_rotating_log(
888-
attempt_dir.join("stdout.log"),
889-
lines,
890-
LOG_TAIL_STREAM_BUDGET,
891-
)
892-
.map_err(|error| control("logs_unavailable", error.to_string()))?;
893-
let stderr = tail_rotating_log(
894-
attempt_dir.join("stderr.log"),
895-
lines,
896-
LOG_TAIL_STREAM_BUDGET,
897-
)
887+
let stdout_path = attempt_dir.join("stdout.log");
888+
let stderr_path = attempt_dir.join("stderr.log");
889+
let (stdout, stderr) = tokio::task::spawn_blocking(move || {
890+
let stdout = tail_rotating_log(stdout_path, lines, LOG_TAIL_STREAM_BUDGET)?;
891+
let stderr = tail_rotating_log(stderr_path, lines, LOG_TAIL_STREAM_BUDGET)?;
892+
Ok::<_, std::io::Error>((stdout, stderr))
893+
})
894+
.await
895+
.map_err(|error| control("logs_unavailable", error.to_string()))?
898896
.map_err(|error| control("logs_unavailable", error.to_string()))?;
899897
let mut output = Vec::with_capacity(stdout.len() + stderr.len());
900898
output.extend(stdout.into_iter().map(|line| format!("stdout: {line}")));

‎crates/buzz-relay/src/handlers/agent_jobs.rs‎

Lines changed: 64 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -468,9 +468,13 @@ async fn persist_lifecycle(
468468
{
469469
return Err(reject("agent job progress seq must be strictly monotonic"));
470470
}
471-
let state = match payload.state {
472-
AgentJobProgressState::Running => "running",
473-
AgentJobProgressState::Cancelling => "cancelling",
471+
let state = if job.cancel_requested {
472+
"cancelling"
473+
} else {
474+
match payload.state {
475+
AgentJobProgressState::Running => "running",
476+
AgentJobProgressState::Cancelling => "cancelling",
477+
}
474478
};
475479
sqlx::query(
476480
r#"
@@ -1344,4 +1348,61 @@ mod tests {
13441348
assert!(!lookup.status.cancel_requested);
13451349
assert_eq!(lookup.chain.len(), 1);
13461350
}
1351+
1352+
#[tokio::test]
1353+
#[ignore = "requires Postgres"]
1354+
async fn late_running_progress_cannot_flip_cancelling_back_to_running() {
1355+
let (pool, db, community, channel, requester, target) = seed_job_test().await;
1356+
let job = Uuid::new_v4();
1357+
let request = request_event(&requester, &target, channel, job, "sticky cancelling");
1358+
persist_agent_job_event(&db, community, &request)
1359+
.await
1360+
.expect("request");
1361+
let accepted = signed_event(
1362+
&target,
1363+
KIND_JOB_ACCEPTED,
1364+
serde_json::json!({
1365+
"schema": 1, "job": job, "attempt": 1,
1366+
"state": "accepted", "accepted_at": Utc::now()
1367+
}),
1368+
lifecycle_tags(&requester, channel, job, &request),
1369+
);
1370+
persist_agent_job_event(&db, community, &accepted)
1371+
.await
1372+
.expect("accepted");
1373+
let cancel = signed_event(
1374+
&requester,
1375+
KIND_JOB_CANCEL,
1376+
serde_json::json!({"schema": 1, "job": job, "reason": "stop"}),
1377+
lifecycle_tags(&target, channel, job, &request),
1378+
);
1379+
persist_agent_job_event(&db, community, &cancel)
1380+
.await
1381+
.expect("cancel");
1382+
1383+
// Late runner progress claiming Running must not reopen the job.
1384+
let mut late_tags = lifecycle_tags(&requester, channel, job, &request);
1385+
late_tags.push(vec!["seq".into(), "1".into()]);
1386+
let late = signed_event(
1387+
&target,
1388+
KIND_JOB_PROGRESS,
1389+
serde_json::json!({
1390+
"schema": 1, "job": job, "attempt": 1, "seq": 1,
1391+
"state": "running", "summary": "late running", "artifacts": []
1392+
}),
1393+
late_tags,
1394+
);
1395+
persist_agent_job_event(&db, community, &late)
1396+
.await
1397+
.expect("late progress accepted");
1398+
1399+
let lookup = lookup_agent_job(&db, community, job)
1400+
.await
1401+
.expect("lookup")
1402+
.expect("job");
1403+
assert_eq!(lookup.status.state, "cancelling");
1404+
assert!(lookup.status.cancel_requested);
1405+
1406+
drop(pool);
1407+
}
13471408
}

‎crates/buzz-runtime/src/client.rs‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -160,10 +160,13 @@ impl RuntimeClient {
160160
}
161161
}
162162
/// Requests privileged runner reconciliation.
163-
pub async fn reconcile(&self) -> Result<(), ClientError> {
163+
///
164+
/// Returns the post-reconciliation job list; the server answers
165+
/// `Reconcile` with `ControlPayload::Jobs`, not `Ack`.
166+
pub async fn reconcile(&self) -> Result<Vec<JobStatus>, ClientError> {
164167
self.require_controller()?;
165168
match self.call(ControlOperation::Reconcile).await? {
166-
ControlPayload::Ack => Ok(()),
169+
ControlPayload::Jobs(jobs) => Ok(jobs),
167170
_ => Err(ClientError::UnexpectedResponse),
168171
}
169172
}

‎crates/buzz-runtime/src/store.rs‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1339,7 +1339,7 @@ fn enqueue(c: &mut Connection, v: InboxEvent) -> Result<EnqueueOutcome, StoreErr
13391339
[v.channel_id.to_string()],
13401340
|r| r.get(0),
13411341
)?;
1342-
let (state, error, outcome) = if n >= 500 {
1342+
let (state, error, outcome) = if n >= MAX_PENDING_PER_CHANNEL as i64 {
13431343
(
13441344
"dead_letter",
13451345
Some("queue_capacity"),
@@ -1448,7 +1448,7 @@ fn requeue(
14481448
[t],
14491449
|r| r.get::<_, i64>(0),
14501450
)? as u32;
1451-
if a > 10 {
1451+
if a > MAX_INBOX_RETRIES {
14521452
tx.execute("UPDATE inbox_events SET state='dead_letter',attempt=?1,turn_id=NULL,available_at=NULL,last_error=?2 WHERE turn_id=?3 AND state='in_turn'",params![a,e,t])?;
14531453
tx.commit()?;
14541454
return Ok(RequeueOutcome::DeadLettered { attempt: a });
@@ -1473,7 +1473,7 @@ fn recover(c: &mut Connection, now: DateTime<Utc>) -> Result<RecoveryOutcome, St
14731473
let mut out = RecoveryOutcome::default();
14741474
for (id, old) in rows {
14751475
let a = old + 1;
1476-
if a > 10 {
1476+
if a > MAX_INBOX_RETRIES {
14771477
tx.execute("UPDATE inbox_events SET state='dead_letter',attempt=?1,turn_id=NULL,available_at=NULL,last_error='recovery_retry_exhausted' WHERE event_id=?2",params![a,id])?;
14781478
out.dead_lettered += 1
14791479
} else {

0 commit comments

Comments
 (0)