Skip to content

Commit e7978b0

Browse files
Afloat16thedotmackclaude
authored
fix(server): lock operator generation state transitions (#5650)
* fix(server): lock operator generation state transitions * build: synchronize verified runtime artifacts * chore: drop generated bundles (rebuilt at release) Co-Authored-By: Claude Opus 5.5 <[email protected]> Claude-Session: https://claude.ai/code/session_019FbY8a4P6RjaWFnrGVZXnd --------- Co-authored-by: Alex Newman <[email protected]> Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent 43ab43b commit e7978b0

2 files changed

Lines changed: 328 additions & 107 deletions

File tree

‎src/server/routes/v1/ServerV1PostgresRoutes.ts‎

Lines changed: 111 additions & 107 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import { z, type ZodTypeAny } from 'zod';
55
import type { RouteHandler } from '../../../services/server/Server.js';
66
import { AppError } from '../../../services/server/ErrorHandler.js';
77
import { CreateAgentEventSchema } from '../../../core/schemas/agent-event.js';
8-
import type { PostgresPool } from '../../../storage/postgres/pool.js';
8+
import { withPostgresTransaction, type PostgresPool } from '../../../storage/postgres/pool.js';
99
import {
1010
PostgresAgentEventsRepository,
1111
type CreatePostgresAgentEventInput,
@@ -1755,104 +1755,99 @@ export class ServerV1PostgresRoutes implements RouteHandler {
17551755
}
17561756
// Scope check first — same NotFound disclosure as the rest of the routes.
17571757
const repo = new PostgresObservationGenerationJobRepository(this.options.pool);
1758-
const current = await this.loadScopedById(req, res, {
1758+
const visible = await this.loadScopedById(req, res, {
17591759
id,
17601760
teamId,
17611761
table: 'observation_generation_jobs',
17621762
notFound: 'Generation job not found',
17631763
scopeMismatch: 'not-found',
17641764
load: (projectId) => repo.getByIdForScope({ id, projectId, teamId }),
17651765
});
1766-
if (!current) return null;
1767-
1768-
// Idempotent fast-path: already queued -> emit audit only, no DB writes.
1769-
if (current.status === 'queued') {
1766+
if (!visible) return null;
1767+
1768+
// Keep the state check and update under the same row lock used by the
1769+
// generation commit path. Queue I/O and audit publication happen after
1770+
// this transaction, never while it holds a generation row.
1771+
const transition = await withPostgresTransaction(this.options.pool, async client => {
1772+
const current = await new PostgresObservationGenerationJobRepository(client).getByIdForScope({
1773+
id, projectId: visible.projectId, teamId, forUpdate: true,
1774+
});
1775+
if (!current) return { kind: 'missing' as const };
1776+
if (current.status === 'queued') return { kind: 'unchanged' as const, current };
1777+
if (current.status === 'processing' || current.status === 'completed') {
1778+
return { kind: 'conflict' as const, current };
1779+
}
1780+
// Reset to queued, clear lock + lifecycle timestamps, increment
1781+
// retried_count for audit. Preserve the existing explicit operator retry
1782+
// policy: an exhausted job receives one additional attempt.
1783+
//
1784+
// current.payload is the canonical BullMQ payload persisted at outbox
1785+
// create time (kind/team_id/project_id/source_type/source_id/
1786+
// generation_job_id/api_key_id/actor_id/source_adapter/request_id).
1787+
// The retry adds operator metadata to the persisted row but enqueues
1788+
// ONLY the BullMQ payload — the worker calls
1789+
// assertServerGenerationJobPayload(job.data) on receipt and would reject
1790+
// the metadata-only object the previous implementation handed it.
1791+
const retriedCount = extractRetriedCount(current.payload) + 1;
1792+
const persistedBullmqPayload = (current.payload && typeof current.payload === 'object'
1793+
? current.payload
1794+
: {}) as Record<string, unknown>;
1795+
const newPayload = {
1796+
...persistedBullmqPayload,
1797+
retried_count: retriedCount,
1798+
last_retried_by_actor: req.authContext?.apiKeyId ?? null,
1799+
last_retried_request_id: req.requestId ?? null,
1800+
};
1801+
// The payload we re-publish to BullMQ on retry: refresh request_id (so
1802+
// the worker logs/audit attribute this run to the operator's request)
1803+
// but keep all canonical job context that the worker validates against.
1804+
const retryBullmqPayload = {
1805+
...persistedBullmqPayload,
1806+
request_id: req.requestId ?? (persistedBullmqPayload as { request_id?: unknown }).request_id ?? null,
1807+
};
1808+
const updated = await client.query(
1809+
`
1810+
UPDATE observation_generation_jobs
1811+
SET status = 'queued',
1812+
locked_at = NULL,
1813+
locked_by = NULL,
1814+
failed_at = NULL,
1815+
cancelled_at = NULL,
1816+
completed_at = NULL,
1817+
last_error = NULL,
1818+
attempts = LEAST(attempts, max_attempts - 1),
1819+
payload = $4::jsonb,
1820+
updated_at = now()
1821+
WHERE id = $1 AND project_id = $2 AND team_id = $3
1822+
RETURNING *
1823+
`,
1824+
[id, current.projectId, teamId, JSON.stringify(newPayload)],
1825+
);
1826+
return { kind: 'updated' as const, current, updatedRow: updated.rows[0], retriedCount, retryBullmqPayload };
1827+
});
1828+
if (transition.kind === 'missing') {
1829+
res.status(404).json({ error: 'NotFound', message: 'Generation job not found' });
1830+
return null;
1831+
}
1832+
if (transition.kind === 'unchanged') {
1833+
const current = transition.current;
17701834
await this.auditWrite(req, 'generation_job.retried_by_operator', current.id, current.projectId, {
17711835
outcome: 'noop_already_queued',
17721836
currentAttempts: current.attempts,
17731837
requestId: req.requestId ?? null,
17741838
});
17751839
return { job: current, retriedCount: extractRetriedCount(current.payload), alreadyQueued: true };
17761840
}
1777-
1778-
if (current.status === 'processing') {
1779-
// Refuse retry on in-flight jobs — the running worker MUST be allowed
1780-
// to finish or fail through its normal lifecycle. Operator can wait
1781-
// or cancel, then retry.
1782-
res.status(409).json({
1783-
error: 'Conflict',
1784-
message: 'Generation job is currently processing; cancel or wait for completion before retrying',
1785-
});
1786-
return null;
1787-
}
1788-
1789-
if (current.status === 'completed') {
1790-
// Refuse retry on already-completed jobs. The deduplication index on
1791-
// observations (generation_key = job_id + index + content) does NOT
1792-
// protect against re-running the provider, because LLM output is
1793-
// non-deterministic and the second run almost always produces a
1794-
// different content string. Replaying would persist a parallel set
1795-
// of observations attributed to the same generation_job_id.
1796-
// cancelGenerationJob applies the same 409 guard for the same reason.
1841+
if (transition.kind === 'conflict') {
17971842
res.status(409).json({
17981843
error: 'Conflict',
1799-
message: 'Generation job already completed; retrying would duplicate observations',
1844+
message: transition.current.status === 'processing'
1845+
? 'Generation job is currently processing; cancel or wait for completion before retrying'
1846+
: 'Generation job already completed; retrying would duplicate observations',
18001847
});
18011848
return null;
18021849
}
1803-
1804-
// Reset to queued, clear lock + lifecycle timestamps, increment
1805-
// retried_count for audit. attempts is intentionally preserved so the
1806-
// BullMQ attempt cap is not bypassed; if the job hit max_attempts the
1807-
// operator must lift the cap explicitly via a separate flow.
1808-
//
1809-
// current.payload is the canonical BullMQ payload persisted at outbox
1810-
// create time (kind/team_id/project_id/source_type/source_id/
1811-
// generation_job_id/api_key_id/actor_id/source_adapter/request_id).
1812-
// The retry adds operator metadata to the persisted row but enqueues
1813-
// ONLY the BullMQ payload — the worker calls
1814-
// assertServerGenerationJobPayload(job.data) on receipt and would reject
1815-
// the metadata-only object the previous implementation handed it.
1816-
const retriedCount = extractRetriedCount(current.payload) + 1;
1817-
const persistedBullmqPayload = (current.payload && typeof current.payload === 'object'
1818-
? current.payload
1819-
: {}) as Record<string, unknown>;
1820-
const newPayload = {
1821-
...persistedBullmqPayload,
1822-
retried_count: retriedCount,
1823-
last_retried_by_actor: req.authContext?.apiKeyId ?? null,
1824-
last_retried_request_id: req.requestId ?? null,
1825-
};
1826-
// The payload we re-publish to BullMQ on retry: refresh request_id (so
1827-
// the worker logs/audit attribute this run to the operator's request)
1828-
// but keep all canonical job context that the worker validates against.
1829-
const retryBullmqPayload = {
1830-
...persistedBullmqPayload,
1831-
request_id: req.requestId ?? (persistedBullmqPayload as { request_id?: unknown }).request_id ?? null,
1832-
};
1833-
const updated = await this.options.pool.query(
1834-
`
1835-
UPDATE observation_generation_jobs
1836-
SET status = 'queued',
1837-
locked_at = NULL,
1838-
locked_by = NULL,
1839-
failed_at = NULL,
1840-
cancelled_at = NULL,
1841-
completed_at = NULL,
1842-
last_error = NULL,
1843-
attempts = LEAST(attempts, max_attempts - 1),
1844-
payload = $4::jsonb,
1845-
updated_at = now()
1846-
WHERE id = $1 AND project_id = $2 AND team_id = $3
1847-
RETURNING *
1848-
`,
1849-
[id, current.projectId, teamId, JSON.stringify(newPayload)],
1850-
);
1851-
const updatedRow = updated.rows[0];
1852-
if (!updatedRow) {
1853-
res.status(404).json({ error: 'NotFound', message: 'Generation job not found' });
1854-
return null;
1855-
}
1850+
const { current, updatedRow, retriedCount, retryBullmqPayload } = transition;
18561851

18571852
// Append lifecycle event so the audit chain mirrors the lifecycle tracker.
18581853
const eventsRepo = new PostgresObservationGenerationJobEventsRepository(this.options.pool);
@@ -1923,46 +1918,55 @@ export class ServerV1PostgresRoutes implements RouteHandler {
19231918
return null;
19241919
}
19251920
const repo = new PostgresObservationGenerationJobRepository(this.options.pool);
1926-
const current = await this.loadScopedById(req, res, {
1921+
const visible = await this.loadScopedById(req, res, {
19271922
id,
19281923
teamId,
19291924
table: 'observation_generation_jobs',
19301925
notFound: 'Generation job not found',
19311926
scopeMismatch: 'not-found',
19321927
load: (projectId) => repo.getByIdForScope({ id, projectId, teamId }),
19331928
});
1934-
if (!current) return null;
1935-
if (current.status === 'cancelled') {
1929+
if (!visible) return null;
1930+
const transition = await withPostgresTransaction(this.options.pool, async client => {
1931+
const current = await new PostgresObservationGenerationJobRepository(client).getByIdForScope({
1932+
id, projectId: visible.projectId, teamId, forUpdate: true,
1933+
});
1934+
if (!current) return { kind: 'missing' as const };
1935+
if (current.status === 'cancelled' || current.status === 'completed') {
1936+
return { kind: 'unchanged' as const, current };
1937+
}
1938+
const updateResult = await client.query(
1939+
`
1940+
UPDATE observation_generation_jobs
1941+
SET status = 'cancelled',
1942+
cancelled_at = now(),
1943+
updated_at = now()
1944+
WHERE id = $1 AND project_id = $2 AND team_id = $3
1945+
RETURNING *
1946+
`,
1947+
[id, current.projectId, teamId],
1948+
);
1949+
return { kind: 'updated' as const, current, updatedRow: updateResult.rows[0] };
1950+
});
1951+
if (transition.kind === 'missing') {
1952+
res.status(404).json({ error: 'NotFound', message: 'Generation job not found' });
1953+
return null;
1954+
}
1955+
const { current } = transition;
1956+
if (transition.kind === 'unchanged') {
1957+
if (current.status === 'completed') {
1958+
res.status(409).json({
1959+
error: 'Conflict', message: 'Generation job already completed; cannot cancel',
1960+
});
1961+
return null;
1962+
}
19361963
await this.auditWrite(req, 'generation_job.cancelled_by_operator', current.id, current.projectId, {
19371964
outcome: 'noop_already_cancelled',
19381965
requestId: req.requestId ?? null,
19391966
});
19401967
return { job: current, alreadyCancelled: true };
19411968
}
1942-
if (current.status === 'completed') {
1943-
res.status(409).json({
1944-
error: 'Conflict',
1945-
message: 'Generation job already completed; cannot cancel',
1946-
});
1947-
return null;
1948-
}
1949-
1950-
const updateResult = await this.options.pool.query(
1951-
`
1952-
UPDATE observation_generation_jobs
1953-
SET status = 'cancelled',
1954-
cancelled_at = now(),
1955-
updated_at = now()
1956-
WHERE id = $1 AND project_id = $2 AND team_id = $3
1957-
RETURNING *
1958-
`,
1959-
[id, current.projectId, teamId],
1960-
);
1961-
const updatedRow = updateResult.rows[0];
1962-
if (!updatedRow) {
1963-
res.status(404).json({ error: 'NotFound', message: 'Generation job not found' });
1964-
return null;
1965-
}
1969+
const { updatedRow } = transition;
19661970

19671971
const eventsRepo = new PostgresObservationGenerationJobEventsRepository(this.options.pool);
19681972
await eventsRepo.append({

0 commit comments

Comments
 (0)