@@ -5,7 +5,7 @@ import { z, type ZodTypeAny } from 'zod';
55import type { RouteHandler } from '../../../services/server/Server.js' ;
66import { AppError } from '../../../services/server/ErrorHandler.js' ;
77import { CreateAgentEventSchema } from '../../../core/schemas/agent-event.js' ;
8- import type { PostgresPool } from '../../../storage/postgres/pool.js' ;
8+ import { withPostgresTransaction , type PostgresPool } from '../../../storage/postgres/pool.js' ;
99import {
1010 PostgresAgentEventsRepository ,
1111 type CreatePostgresAgentEventInput ,
@@ -1755,104 +1755,99 @@ export class ServerV1PostgresRoutes implements RouteHandler {
17551755 }
17561756 // Scope check first — same NotFound disclosure as the rest of the routes.
17571757 const repo = new PostgresObservationGenerationJobRepository ( this . options . pool ) ;
1758- const current = await this . loadScopedById ( req , res , {
1758+ const visible = await this . loadScopedById ( req , res , {
17591759 id,
17601760 teamId,
17611761 table : 'observation_generation_jobs' ,
17621762 notFound : 'Generation job not found' ,
17631763 scopeMismatch : 'not-found' ,
17641764 load : ( projectId ) => repo . getByIdForScope ( { id, projectId, teamId } ) ,
17651765 } ) ;
1766- if ( ! current ) return null ;
1767-
1768- // Idempotent fast-path: already queued -> emit audit only, no DB writes.
1769- if ( current . status === 'queued' ) {
1766+ if ( ! visible ) return null ;
1767+
1768+ // Keep the state check and update under the same row lock used by the
1769+ // generation commit path. Queue I/O and audit publication happen after
1770+ // this transaction, never while it holds a generation row.
1771+ const transition = await withPostgresTransaction ( this . options . pool , async client => {
1772+ const current = await new PostgresObservationGenerationJobRepository ( client ) . getByIdForScope ( {
1773+ id, projectId : visible . projectId , teamId, forUpdate : true ,
1774+ } ) ;
1775+ if ( ! current ) return { kind : 'missing' as const } ;
1776+ if ( current . status === 'queued' ) return { kind : 'unchanged' as const , current } ;
1777+ if ( current . status === 'processing' || current . status === 'completed' ) {
1778+ return { kind : 'conflict' as const , current } ;
1779+ }
1780+ // Reset to queued, clear lock + lifecycle timestamps, increment
1781+ // retried_count for audit. Preserve the existing explicit operator retry
1782+ // policy: an exhausted job receives one additional attempt.
1783+ //
1784+ // current.payload is the canonical BullMQ payload persisted at outbox
1785+ // create time (kind/team_id/project_id/source_type/source_id/
1786+ // generation_job_id/api_key_id/actor_id/source_adapter/request_id).
1787+ // The retry adds operator metadata to the persisted row but enqueues
1788+ // ONLY the BullMQ payload — the worker calls
1789+ // assertServerGenerationJobPayload(job.data) on receipt and would reject
1790+ // the metadata-only object the previous implementation handed it.
1791+ const retriedCount = extractRetriedCount ( current . payload ) + 1 ;
1792+ const persistedBullmqPayload = ( current . payload && typeof current . payload === 'object'
1793+ ? current . payload
1794+ : { } ) as Record < string , unknown > ;
1795+ const newPayload = {
1796+ ...persistedBullmqPayload ,
1797+ retried_count : retriedCount ,
1798+ last_retried_by_actor : req . authContext ?. apiKeyId ?? null ,
1799+ last_retried_request_id : req . requestId ?? null ,
1800+ } ;
1801+ // The payload we re-publish to BullMQ on retry: refresh request_id (so
1802+ // the worker logs/audit attribute this run to the operator's request)
1803+ // but keep all canonical job context that the worker validates against.
1804+ const retryBullmqPayload = {
1805+ ...persistedBullmqPayload ,
1806+ request_id : req . requestId ?? ( persistedBullmqPayload as { request_id ?: unknown } ) . request_id ?? null ,
1807+ } ;
1808+ const updated = await client . query (
1809+ `
1810+ UPDATE observation_generation_jobs
1811+ SET status = 'queued',
1812+ locked_at = NULL,
1813+ locked_by = NULL,
1814+ failed_at = NULL,
1815+ cancelled_at = NULL,
1816+ completed_at = NULL,
1817+ last_error = NULL,
1818+ attempts = LEAST(attempts, max_attempts - 1),
1819+ payload = $4::jsonb,
1820+ updated_at = now()
1821+ WHERE id = $1 AND project_id = $2 AND team_id = $3
1822+ RETURNING *
1823+ ` ,
1824+ [ id , current . projectId , teamId , JSON . stringify ( newPayload ) ] ,
1825+ ) ;
1826+ return { kind : 'updated' as const , current, updatedRow : updated . rows [ 0 ] , retriedCount, retryBullmqPayload } ;
1827+ } ) ;
1828+ if ( transition . kind === 'missing' ) {
1829+ res . status ( 404 ) . json ( { error : 'NotFound' , message : 'Generation job not found' } ) ;
1830+ return null ;
1831+ }
1832+ if ( transition . kind === 'unchanged' ) {
1833+ const current = transition . current ;
17701834 await this . auditWrite ( req , 'generation_job.retried_by_operator' , current . id , current . projectId , {
17711835 outcome : 'noop_already_queued' ,
17721836 currentAttempts : current . attempts ,
17731837 requestId : req . requestId ?? null ,
17741838 } ) ;
17751839 return { job : current , retriedCount : extractRetriedCount ( current . payload ) , alreadyQueued : true } ;
17761840 }
1777-
1778- if ( current . status === 'processing' ) {
1779- // Refuse retry on in-flight jobs — the running worker MUST be allowed
1780- // to finish or fail through its normal lifecycle. Operator can wait
1781- // or cancel, then retry.
1782- res . status ( 409 ) . json ( {
1783- error : 'Conflict' ,
1784- message : 'Generation job is currently processing; cancel or wait for completion before retrying' ,
1785- } ) ;
1786- return null ;
1787- }
1788-
1789- if ( current . status === 'completed' ) {
1790- // Refuse retry on already-completed jobs. The deduplication index on
1791- // observations (generation_key = job_id + index + content) does NOT
1792- // protect against re-running the provider, because LLM output is
1793- // non-deterministic and the second run almost always produces a
1794- // different content string. Replaying would persist a parallel set
1795- // of observations attributed to the same generation_job_id.
1796- // cancelGenerationJob applies the same 409 guard for the same reason.
1841+ if ( transition . kind === 'conflict' ) {
17971842 res . status ( 409 ) . json ( {
17981843 error : 'Conflict' ,
1799- message : 'Generation job already completed; retrying would duplicate observations' ,
1844+ message : transition . current . status === 'processing'
1845+ ? 'Generation job is currently processing; cancel or wait for completion before retrying'
1846+ : 'Generation job already completed; retrying would duplicate observations' ,
18001847 } ) ;
18011848 return null ;
18021849 }
1803-
1804- // Reset to queued, clear lock + lifecycle timestamps, increment
1805- // retried_count for audit. attempts is intentionally preserved so the
1806- // BullMQ attempt cap is not bypassed; if the job hit max_attempts the
1807- // operator must lift the cap explicitly via a separate flow.
1808- //
1809- // current.payload is the canonical BullMQ payload persisted at outbox
1810- // create time (kind/team_id/project_id/source_type/source_id/
1811- // generation_job_id/api_key_id/actor_id/source_adapter/request_id).
1812- // The retry adds operator metadata to the persisted row but enqueues
1813- // ONLY the BullMQ payload — the worker calls
1814- // assertServerGenerationJobPayload(job.data) on receipt and would reject
1815- // the metadata-only object the previous implementation handed it.
1816- const retriedCount = extractRetriedCount ( current . payload ) + 1 ;
1817- const persistedBullmqPayload = ( current . payload && typeof current . payload === 'object'
1818- ? current . payload
1819- : { } ) as Record < string , unknown > ;
1820- const newPayload = {
1821- ...persistedBullmqPayload ,
1822- retried_count : retriedCount ,
1823- last_retried_by_actor : req . authContext ?. apiKeyId ?? null ,
1824- last_retried_request_id : req . requestId ?? null ,
1825- } ;
1826- // The payload we re-publish to BullMQ on retry: refresh request_id (so
1827- // the worker logs/audit attribute this run to the operator's request)
1828- // but keep all canonical job context that the worker validates against.
1829- const retryBullmqPayload = {
1830- ...persistedBullmqPayload ,
1831- request_id : req . requestId ?? ( persistedBullmqPayload as { request_id ?: unknown } ) . request_id ?? null ,
1832- } ;
1833- const updated = await this . options . pool . query (
1834- `
1835- UPDATE observation_generation_jobs
1836- SET status = 'queued',
1837- locked_at = NULL,
1838- locked_by = NULL,
1839- failed_at = NULL,
1840- cancelled_at = NULL,
1841- completed_at = NULL,
1842- last_error = NULL,
1843- attempts = LEAST(attempts, max_attempts - 1),
1844- payload = $4::jsonb,
1845- updated_at = now()
1846- WHERE id = $1 AND project_id = $2 AND team_id = $3
1847- RETURNING *
1848- ` ,
1849- [ id , current . projectId , teamId , JSON . stringify ( newPayload ) ] ,
1850- ) ;
1851- const updatedRow = updated . rows [ 0 ] ;
1852- if ( ! updatedRow ) {
1853- res . status ( 404 ) . json ( { error : 'NotFound' , message : 'Generation job not found' } ) ;
1854- return null ;
1855- }
1850+ const { current, updatedRow, retriedCount, retryBullmqPayload } = transition ;
18561851
18571852 // Append lifecycle event so the audit chain mirrors the lifecycle tracker.
18581853 const eventsRepo = new PostgresObservationGenerationJobEventsRepository ( this . options . pool ) ;
@@ -1923,46 +1918,55 @@ export class ServerV1PostgresRoutes implements RouteHandler {
19231918 return null ;
19241919 }
19251920 const repo = new PostgresObservationGenerationJobRepository ( this . options . pool ) ;
1926- const current = await this . loadScopedById ( req , res , {
1921+ const visible = await this . loadScopedById ( req , res , {
19271922 id,
19281923 teamId,
19291924 table : 'observation_generation_jobs' ,
19301925 notFound : 'Generation job not found' ,
19311926 scopeMismatch : 'not-found' ,
19321927 load : ( projectId ) => repo . getByIdForScope ( { id, projectId, teamId } ) ,
19331928 } ) ;
1934- if ( ! current ) return null ;
1935- if ( current . status === 'cancelled' ) {
1929+ if ( ! visible ) return null ;
1930+ const transition = await withPostgresTransaction ( this . options . pool , async client => {
1931+ const current = await new PostgresObservationGenerationJobRepository ( client ) . getByIdForScope ( {
1932+ id, projectId : visible . projectId , teamId, forUpdate : true ,
1933+ } ) ;
1934+ if ( ! current ) return { kind : 'missing' as const } ;
1935+ if ( current . status === 'cancelled' || current . status === 'completed' ) {
1936+ return { kind : 'unchanged' as const , current } ;
1937+ }
1938+ const updateResult = await client . query (
1939+ `
1940+ UPDATE observation_generation_jobs
1941+ SET status = 'cancelled',
1942+ cancelled_at = now(),
1943+ updated_at = now()
1944+ WHERE id = $1 AND project_id = $2 AND team_id = $3
1945+ RETURNING *
1946+ ` ,
1947+ [ id , current . projectId , teamId ] ,
1948+ ) ;
1949+ return { kind : 'updated' as const , current, updatedRow : updateResult . rows [ 0 ] } ;
1950+ } ) ;
1951+ if ( transition . kind === 'missing' ) {
1952+ res . status ( 404 ) . json ( { error : 'NotFound' , message : 'Generation job not found' } ) ;
1953+ return null ;
1954+ }
1955+ const { current } = transition ;
1956+ if ( transition . kind === 'unchanged' ) {
1957+ if ( current . status === 'completed' ) {
1958+ res . status ( 409 ) . json ( {
1959+ error : 'Conflict' , message : 'Generation job already completed; cannot cancel' ,
1960+ } ) ;
1961+ return null ;
1962+ }
19361963 await this . auditWrite ( req , 'generation_job.cancelled_by_operator' , current . id , current . projectId , {
19371964 outcome : 'noop_already_cancelled' ,
19381965 requestId : req . requestId ?? null ,
19391966 } ) ;
19401967 return { job : current , alreadyCancelled : true } ;
19411968 }
1942- if ( current . status === 'completed' ) {
1943- res . status ( 409 ) . json ( {
1944- error : 'Conflict' ,
1945- message : 'Generation job already completed; cannot cancel' ,
1946- } ) ;
1947- return null ;
1948- }
1949-
1950- const updateResult = await this . options . pool . query (
1951- `
1952- UPDATE observation_generation_jobs
1953- SET status = 'cancelled',
1954- cancelled_at = now(),
1955- updated_at = now()
1956- WHERE id = $1 AND project_id = $2 AND team_id = $3
1957- RETURNING *
1958- ` ,
1959- [ id , current . projectId , teamId ] ,
1960- ) ;
1961- const updatedRow = updateResult . rows [ 0 ] ;
1962- if ( ! updatedRow ) {
1963- res . status ( 404 ) . json ( { error : 'NotFound' , message : 'Generation job not found' } ) ;
1964- return null ;
1965- }
1969+ const { updatedRow } = transition ;
19661970
19671971 const eventsRepo = new PostgresObservationGenerationJobEventsRepository ( this . options . pool ) ;
19681972 await eventsRepo . append ( {
0 commit comments