Skip to content

Commit 64ff53d

Browse files
committed
hardwork: fix restart resilience and parallel agent hangs
- Give each parallel plan agent its own isolated OPENCODE_DB to avoid SQLite lock contention that caused subagents to hang mid tool-call when running multiple agents in parallel (anomalyco/opencode#21215) - Add a stall watchdog to runOpencodeStreaming (OPENCODE_STALL_TIMEOUT_MS, default 10m) so a hung opencode process is killed and fails cleanly instead of hanging the job forever - Persist each plan agent's result incrementally via new hardworkPlanProgress mutation + hardworkIndividualPlans column, so a worker restart mid-hardwork-flow doesn't lose completed work - On resume, recover already-completed plans from disk (.opencode/plans/plan-hardwork-*.md) and skip straight to synthesis - Add Prisma migration for hardworkIndividualPlans
1 parent c38a27e commit 64ff53d

9 files changed

Lines changed: 239 additions & 15 deletions

File tree

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
-- AlterTable
2+
ALTER TABLE "Job" ADD COLUMN "hardworkIndividualPlans" TEXT;

‎packages/bot/prisma/schema.prisma‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ model Job {
7272
hardwork Boolean @default(false)
7373
parallelPlanCount Int @default(3)
7474
hardworkPlans String?
75+
hardworkIndividualPlans String?
7576
selectedPlanIndex Int?
7677
pendingSuggestion String?
7778
planEditToken String?

‎packages/bot/src/trpc/router.ts‎

Lines changed: 34 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ import {
2727
CreateReviewMergeJobInput,
2828
CloseJobThreadInput,
2929
HardworkPlansReadyInput,
30+
HardworkPlanProgressInput,
3031
ConfirmHardworkPlanInput,
3132
WaitForSelectionOutput,
3233
SavePlanRevisionInput,
@@ -80,6 +81,7 @@ function toJobOutput(job: Job) {
8081
hardwork: job.hardwork,
8182
parallelPlanCount: job.parallelPlanCount,
8283
hardworkPlans: job.hardworkPlans ?? null,
84+
hardworkIndividualPlans: job.hardworkIndividualPlans ?? null,
8385
selectedPlanIndex: job.selectedPlanIndex ?? null,
8486
pendingSuggestion: job.pendingSuggestion,
8587
planEditToken: job.planEditToken ?? null,
@@ -514,6 +516,35 @@ export const appRouter = t.router({
514516
return { success: true };
515517
}),
516518

519+
// Persists a single completed parallel plan-agent result as soon as it
520+
// finishes, so a worker restart mid-hardwork-flow doesn't lose completed
521+
// work and doesn't have to re-run agents that already succeeded.
522+
hardworkPlanProgress: t.procedure
523+
.input(HardworkPlanProgressInput)
524+
.output(StatusResult)
525+
.mutation(async ({ input }) => {
526+
const job = await prisma.job.findUnique({ where: { id: input.jobId } });
527+
if (!job) return { success: false };
528+
529+
let existing: { index: number; planMd: string; sessionId: string }[] = [];
530+
try {
531+
const parsed = JSON.parse(job.hardworkIndividualPlans ?? "[]");
532+
if (Array.isArray(parsed)) existing = parsed;
533+
} catch {
534+
existing = [];
535+
}
536+
537+
const withoutIndex = existing.filter(p => p.index !== input.index);
538+
withoutIndex.push({ index: input.index, planMd: input.planMd, sessionId: input.sessionId });
539+
540+
await prisma.job.update({
541+
where: { id: input.jobId },
542+
data: { hardworkIndividualPlans: JSON.stringify(withoutIndex) },
543+
});
544+
545+
return { success: true };
546+
}),
547+
517548
hardworkPlansReady: t.procedure
518549
.input(HardworkPlansReadyInput)
519550
.output(StatusResult)
@@ -612,8 +643,9 @@ export const appRouter = t.router({
612643
status: "pending",
613644
workerId: null,
614645
// Intentionally NOT clearing: planMd, opencodeSessionId, buildSessionId,
615-
// hardworkPlans, selectedPlanIndex, branch, worktreePath — these are
616-
// durable progress checkpoints used to resume correctly after a restart.
646+
// hardworkPlans, hardworkIndividualPlans, selectedPlanIndex, branch,
647+
// worktreePath — these are durable progress checkpoints used to
648+
// resume correctly after a restart.
617649
pendingSuggestion: null,
618650
planEditToken: null,
619651
pendingQuestions: null,

‎packages/shared/src/router.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import {
2626
CreateReviewMergeJobInput,
2727
CloseJobThreadInput,
2828
HardworkPlansReadyInput,
29+
HardworkPlanProgressInput,
2930
ConfirmHardworkPlanInput,
3031
WaitForSelectionOutput,
3132
SavePlanRevisionInput,
@@ -178,6 +179,13 @@ export const appRouter = t.router({
178179
throw new Error("Not implemented in shared package");
179180
}),
180181

182+
hardworkPlanProgress: t.procedure
183+
.input(HardworkPlanProgressInput)
184+
.output(StatusResult)
185+
.mutation(async () => {
186+
throw new Error("Not implemented in shared package");
187+
}),
188+
181189
confirmHardworkPlan: t.procedure
182190
.input(ConfirmHardworkPlanInput)
183191
.output(StatusResult)

‎packages/shared/src/schemas.ts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,7 @@ export const JobSchema = z.object({
6060
hardwork: z.boolean(),
6161
parallelPlanCount: z.number(),
6262
hardworkPlans: z.string().nullable(),
63+
hardworkIndividualPlans: z.string().nullable(),
6364
selectedPlanIndex: z.number().nullable(),
6465
pendingSuggestion: z.string().nullable(),
6566
planEditToken: z.string().nullish(),
@@ -225,6 +226,13 @@ export const HardworkPlanEntry = z.object({
225226
label: z.string(),
226227
});
227228

229+
export const HardworkPlanProgressInput = z.object({
230+
jobId: z.number(),
231+
index: z.number(),
232+
planMd: z.string(),
233+
sessionId: z.string(),
234+
});
235+
228236
export const HardworkPlansReadyInput = z.object({
229237
jobId: z.number(),
230238
plans: z.array(HardworkPlanEntry),

‎packages/worker/src/env.ts‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ const {
99
GIT_BOT_EMAIL = "[email protected]",
1010
GIT_COAUTHOR_NAME,
1111
GIT_COAUTHOR_EMAIL,
12+
OPENCODE_STALL_TIMEOUT_MS,
1213
PATH: ENV_PATH = process.env.PATH ?? "",
1314
} = process.env;
1415

@@ -22,6 +23,18 @@ const gitCoauthorName = GIT_COAUTHOR_NAME || "";
2223
const gitCoauthorEmail = GIT_COAUTHOR_EMAIL || "";
2324
const hasCoauthor = !!(gitCoauthorName && gitCoauthorEmail);
2425

26+
// How long an `opencode` subprocess may go without producing any stdout
27+
// events before the worker considers it stuck and kills it. This converts
28+
// a silent infinite hang (e.g. SQLite lock contention between parallel
29+
// `opencode run` processes sharing the same session DB, or a stuck
30+
// subagent/tool call) into a clean failure that can be retried, instead of
31+
// blocking the job — and the worker — forever.
32+
// Set to 0 to disable.
33+
const DEFAULT_OPENCODE_STALL_TIMEOUT_MS = 10 * 60 * 1000; // 10 minutes
34+
const opencodeStallTimeoutMs = OPENCODE_STALL_TIMEOUT_MS !== undefined
35+
? Number.parseInt(OPENCODE_STALL_TIMEOUT_MS, 10) || 0
36+
: DEFAULT_OPENCODE_STALL_TIMEOUT_MS;
37+
2538
if (!SHARED_SECRET) throw new Error("SHARED_SECRET is required");
2639

2740
// ── PATH augmentation ──────────────────────────────────────────────────────
@@ -91,6 +104,7 @@ export {
91104
gitCoauthorName,
92105
gitCoauthorEmail,
93106
hasCoauthor,
107+
opencodeStallTimeoutMs,
94108
AUGMENTED_PATH,
95109
REQUIRED_BINARIES,
96110
requireBinaries,

‎packages/worker/src/hardwork.ts‎

Lines changed: 123 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import os from "node:os";
12
import path from "node:path";
23
import { WORKER_ID } from "./env";
34
import { client, postInfo } from "./trpc";
@@ -6,6 +7,12 @@ import { jobLog } from "./logging";
67
import { runPlanAgent } from "./plan";
78
import { runOpencodeStreaming } from "./opencode";
89

10+
interface IndividualPlan {
11+
index: number;
12+
planMd: string;
13+
sessionId: string;
14+
}
15+
916
function buildSynthesisPrompt(
1017
job: Job,
1118
plans: { index: number; planMd: string }[],
@@ -52,31 +59,136 @@ async function waitForHardworkSelection(
5259
}
5360
}
5461

62+
/** Parse the `hardworkIndividualPlans` JSON column into a typed array, tolerating bad/missing data. */
63+
function parseIndividualPlans(raw: string | null | undefined): IndividualPlan[] {
64+
if (!raw) return [];
65+
try {
66+
const parsed = JSON.parse(raw);
67+
if (Array.isArray(parsed)) {
68+
return parsed.filter((p): p is IndividualPlan =>
69+
!!p && typeof p.index === "number" && typeof p.planMd === "string");
70+
}
71+
} catch {
72+
// ignore malformed JSON
73+
}
74+
return [];
75+
}
76+
77+
/** Persist a single completed plan-agent result immediately, so it survives a worker restart. */
78+
async function savePlanProgress(jobId: number, plan: IndividualPlan): Promise<void> {
79+
await client.hardworkPlanProgress.mutate({
80+
jobId,
81+
index: plan.index,
82+
planMd: plan.planMd,
83+
sessionId: plan.sessionId,
84+
}).catch((err) => {
85+
jobLog(jobId, `Failed to persist hardwork plan ${plan.index} progress (non-fatal): ${err}`);
86+
});
87+
}
88+
89+
/**
90+
* Runs the full hardwork flow: N parallel plan agents followed by a
91+
* synthesis pass. Restart-safe — already-completed individual plans (whether
92+
* checkpointed in the DB via `hardworkIndividualPlans`, or recovered from
93+
* plan files left on disk by a previous run) are reused rather than re-run.
94+
*/
5595
async function runHardworkFlow(
5696
job: Job,
5797
worktreePath: string,
5898
issueNumber: number | null,
5999
helperPath: string,
60100
): Promise<{ planMd: string; sessionId: string } | null> {
61-
await postInfo(job.id, `Running ${job.parallelPlanCount} parallel plan agents...`);
101+
const total = job.parallelPlanCount;
102+
const planDir = path.join(worktreePath, ".opencode", "plans");
62103

63-
const planPromises = Array.from(
64-
{ length: job.parallelPlanCount },
65-
(_, i) => runPlanAgent(job, worktreePath, issueNumber, helperPath, `hardwork-${job.id}-${i}`, false),
66-
);
104+
// ── Recover already-completed individual plans ─────────────────────────
105+
const completed = new Map<number, IndividualPlan>();
106+
for (const p of parseIndividualPlans(job.hardworkIndividualPlans)) {
107+
completed.set(p.index, p);
108+
}
109+
110+
const recoveredFromDisk: number[] = [];
111+
for (let i = 0; i < total; i++) {
112+
if (completed.has(i)) continue;
113+
114+
// Fall back to plan files left on disk by a previous run that crashed
115+
// before its progress could be persisted to the DB (or before this
116+
// checkpointing existed at all).
117+
const planFilePath = path.join(planDir, `plan-hardwork-${job.id}-${i}.md`);
118+
const file = Bun.file(planFilePath);
119+
if (await file.exists()) {
120+
const content = await file.text();
121+
if (content.trim().length > 0) {
122+
completed.set(i, { index: i, planMd: content, sessionId: "" });
123+
recoveredFromDisk.push(i);
124+
}
125+
}
126+
}
127+
128+
if (recoveredFromDisk.length > 0) {
129+
jobLog(job.id, `Recovered ${recoveredFromDisk.length} plan(s) from disk (indices: ${recoveredFromDisk.join(", ")})`);
130+
// Persist immediately so a future restart doesn't need to re-scan disk.
131+
await Promise.all(recoveredFromDisk.map(i => savePlanProgress(job.id, completed.get(i)!)));
132+
}
67133

68-
const results = await Promise.allSettled(planPromises);
69-
const successfulPlans = results
70-
.filter((r): r is PromiseFulfilledResult<{ planMd: string; sessionId: string }> => r.status === "fulfilled")
71-
.map((r, i) => ({ index: i, planMd: r.value.planMd, sessionId: r.value.sessionId }));
134+
const pending: number[] = [];
135+
for (let i = 0; i < total; i++) {
136+
if (!completed.has(i)) pending.push(i);
137+
}
138+
139+
if (pending.length === 0 && completed.size > 0) {
140+
jobLog(job.id, `All ${total} hardwork plan agents already completed in a previous run — skipping straight to synthesis`);
141+
await postInfo(job.id, `All ${total} plan agents already completed in a previous run — proceeding to synthesis...`);
142+
} else {
143+
if (completed.size > 0) {
144+
jobLog(job.id, `Resuming hardwork: ${completed.size}/${total} plan agent(s) already done, running remaining ${pending.length}`);
145+
await postInfo(job.id, `Resuming — ${completed.size}/${total} plan agents already done, running remaining ${pending.length}...`);
146+
} else {
147+
await postInfo(job.id, `Running ${total} parallel plan agents...`);
148+
}
72149

73-
if (successfulPlans.length === 0) {
150+
// Each parallel plan agent gets its own isolated opencode session
151+
// database. Running multiple `opencode run` processes concurrently
152+
// against the same project shares a single SQLite session DB, which
153+
// under contention can cause a process to hang silently (e.g. stuck
154+
// mid tool-call while writing session state for a `read`). Giving each
155+
// agent its own DB file avoids that contention entirely. These agents
156+
// are fire-and-forget (no later `--session --continue`), so isolation
157+
// is safe here.
158+
const tmpDir = os.tmpdir();
159+
const results = await Promise.allSettled(pending.map(async (i) => {
160+
const dbPath = path.join(tmpDir, `opencode-hardwork-${job.id}-${i}.db`);
161+
const result = await runPlanAgent(
162+
job, worktreePath, issueNumber, helperPath, `hardwork-${job.id}-${i}`, false,
163+
{ OPENCODE_DB: dbPath },
164+
);
165+
const plan: IndividualPlan = { index: i, planMd: result.planMd, sessionId: result.sessionId };
166+
await savePlanProgress(job.id, plan);
167+
return plan;
168+
}));
169+
170+
for (let idx = 0; idx < pending.length; idx++) {
171+
const i = pending[idx]!;
172+
const r = results[idx]!;
173+
if (r.status === "fulfilled") {
174+
completed.set(i, r.value);
175+
} else {
176+
jobLog(job.id, `Hardwork plan agent ${i} failed: ${r.reason instanceof Error ? r.reason.message : String(r.reason)}`);
177+
}
178+
}
179+
}
180+
181+
if (completed.size === 0) {
74182
throw new Error("All parallel plan agents failed");
75183
}
76184

185+
const successfulPlans = [...completed.values()].toSorted((a, b) => a.index - b.index);
186+
if (successfulPlans.length < total) {
187+
jobLog(job.id, `Proceeding to synthesis with ${successfulPlans.length}/${total} successful plan(s)`);
188+
}
189+
77190
await postInfo(job.id, "Synthesizing plans into final plan...");
78191

79-
const planDir = path.join(worktreePath, ".opencode", "plans");
80192
const synthesisPlanFile = path.join(planDir, `plan-synthesis-${job.id}.md`);
81193
const synthesisPrompt = buildSynthesisPrompt(job, successfulPlans, issueNumber, synthesisPlanFile);
82194

0 commit comments

Comments
 (0)