Skip to content

Commit f55d433

Browse files
anandgupta42claude
andcommitted
Merge feat/analyst-harness-dab-2026-10 into feat/analyst-v10-commit
Resolve `processor.ts`: idle is published once, after committed text, for both deferred echoes and requested aborts. Fix `run-output.ts` joining partial-answer objects as strings, and stub `beginRequest` in the run accounting harness. Co-Authored-By: Claude Opus 5.5 <[email protected]>
2 parents 4e41ff9 + 5891314 commit f55d433

93 files changed

Lines changed: 5448 additions & 1015 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎docs/docs/data-engineering/tools/sql-tools.md‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,8 @@ Run SQL queries against your connected warehouse.
2323
```
2424

2525
**Parameters:**
26-
- `query` (required): SQL to execute
26+
- `query` (required unless `query_from` is set): SQL to execute
27+
- `query_from` (optional): `{warehouse, sql}` - a single read-only SELECT, run on `warehouse`, that returns exactly one row and one non-empty SQL string column (max 1 MiB). That SQL is validated as a read-only SELECT and run on the target `warehouse` without passing through the conversation. `query_from.warehouse` is the source; the top-level `warehouse` is then required and names the target; provide exactly one of `query` or `query_from`
2728
- `warehouse` (optional): Connection name from config. Uses default if omitted
2829
- `limit` (optional, default: 100): Max rows returned
2930

‎packages/drivers/src/duckdb.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -167,10 +167,11 @@ export async function connect(
167167
timeoutMs: number | undefined,
168168
maxResultBytes?: number,
169169
onDispatch?: () => void,
170+
signal?: AbortSignal,
170171
): Promise<any[]> {
171172
if (!shared) throw new Error("DuckDB connection not open")
172173
try {
173-
return await shared.engine.call<any[]>("all", { sql, params, connectionId, maxResultBytes }, timeoutMs, onDispatch)
174+
return await shared.engine.call<any[]>("all", { sql, params, connectionId, maxResultBytes }, timeoutMs, onDispatch, signal)
174175
} catch (err) {
175176
throw err instanceof Error ? wrapDuckDBError(err) : err
176177
}
@@ -322,6 +323,7 @@ export async function connect(
322323
resolveQueryTimeout(config, options).ms,
323324
options?.maxResultBytes,
324325
options?.onDispatch,
326+
options?.signal,
325327
)
326328
// altimate_change end
327329
const columns =

‎packages/drivers/src/local-engine.ts‎

Lines changed: 111 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,8 @@
1414
* and transactions. In-memory engines remain private to their connector.
1515
* - Calls run one at a time in arrival order. The deadline covers execution
1616
* only (not time queued behind another call), so a caller is never cancelled
17-
* by someone else's timeout. A restart rejects every stale queued call,
18-
* including the timed-out caller's, and discloses peers' lost state once
19-
* before accepting fresh calls.
17+
* by someone else's timeout. A restart rejects stale queued calls for
18+
* sessions that may have lost state; stateless file readers can continue.
2019
* - After a kill the handle is reopened with the same arguments and open deadline on the next
2120
* call. All sessions on that engine lose their state, and the timeout error
2221
* says so.
@@ -30,14 +29,17 @@ import * as os from "os"
3029
import * as path from "path"
3130
import { LOCAL_ENGINE_CHILD_SOURCE } from "./local-engine-child-source"
3231
import { pack, unpack } from "./local-engine-codec"
33-
import { QueryTimeoutError } from "./types"
32+
import { LocalEngineProcessError, QueryTimeoutError } from "./types"
3433

3534
/** Grace for a killed process to be reaped before we move on. */
3635
const REAP_TIMEOUT_MS = 2_000
3736
/** Grace for a polite close before the process is killed. */
3837
const CLOSE_TIMEOUT_MS = 2_000
3938
/** Short grace when close must bypass an in-flight query. */
4039
const ACTIVE_CLOSE_TIMEOUT_MS = 500
40+
/** Keep diagnostics bounded, including when another process inherits the stderr pipe. */
41+
const STDERR_TAIL_CHARS = 8_192
42+
const STDERR_DRAIN_TIMEOUT_MS = 100
4143

4244
type Engine = "sqlite" | "duckdb"
4345

@@ -49,7 +51,9 @@ interface ChildProc {
4951
// altimate_change end
5052
stdin: { write(data: string): unknown; flush(): unknown }
5153
stdout: AsyncIterable<Uint8Array>
54+
stderr: AsyncIterable<Uint8Array>
5255
exited: Promise<number>
56+
signalCode: string | null
5357
kill(signal?: string): void
5458
}
5559

@@ -66,10 +70,22 @@ interface Child {
6670
retired: boolean
6771
/** We ended it on purpose; its exit is not a crash. */
6872
killedByUs: boolean
73+
stderrTail: string
6974
}
7075

7176
interface Session {
7277
generation: number
78+
stateful: boolean
79+
lostGeneration: number
80+
}
81+
82+
/** Prove simple reads only; unfamiliar SQL and batches conservatively retain the state-loss guard. */
83+
function isStatelessRead(sql: unknown): boolean {
84+
if (typeof sql !== "string") return false
85+
const statement = sql.trim().replace(/;$/, "")
86+
return /^(SELECT|WITH|VALUES)\b/i.test(statement) &&
87+
!statement.includes(";") &&
88+
!/\b(INSERT|UPDATE|DELETE|MERGE|CREATE|DROP|ALTER|COPY|CALL|EXECUTE|BEGIN|START|COMMIT|ROLLBACK|SAVEPOINT|RELEASE|SET|RESET|USE|ATTACH|DETACH|PRAGMA|INSTALL|LOAD|SETSEED|NEXTVAL|LOAD_EXTENSION)\b/i.test(statement)
7389
}
7490

7591
const liveChildren = new Set<Child>()
@@ -252,6 +268,7 @@ export class LocalEngine {
252268
private tail: Promise<unknown> = Promise.resolve()
253269
private closed = false
254270
private generation = 0
271+
private sharedStateful = false
255272
private readonly sessions = new Map<number | undefined, Session>()
256273
private readonly cwd: string
257274

@@ -283,18 +300,24 @@ export class LocalEngine {
283300
})
284301
}
285302

286-
private sendOpen(child: Child, request: OpenRequest): Promise<void> {
303+
private sendOpen(child: Child, request: OpenRequest, signal?: AbortSignal): Promise<void> {
287304
return this.send(child, "open", { ...request.payload, engine: this.opts.engine }, request.timeoutMs, () =>
288305
request.onTimeout
289306
? request.onTimeout()
290307
: new Error(`${this.opts.label} store did not finish opening in ${request.timeoutMs}ms`),
308+
undefined,
309+
signal,
291310
)
292311
}
293312

294313
/** Register even idle shared-engine connectors so a restart reaches every peer. */
295314
registerSession(connectionId?: number): void {
296315
if (!this.sessions.has(connectionId)) {
297-
this.sessions.set(connectionId, { generation: this.generation })
316+
this.sessions.set(connectionId, {
317+
generation: this.generation,
318+
stateful: this.opts.storePath === ":memory:" || this.opts.storePath === "",
319+
lostGeneration: 0,
320+
})
298321
}
299322
}
300323

@@ -304,6 +327,7 @@ export class LocalEngine {
304327
payload: Record<string, unknown>,
305328
timeoutMs?: number,
306329
onDispatch?: () => void,
330+
signal?: AbortSignal,
307331
): Promise<T> {
308332
const connectionId = payload["connectionId"] as number | undefined
309333
this.registerSession(connectionId)
@@ -318,22 +342,35 @@ export class LocalEngine {
318342
if (this.closed || !this.openRequest) throw new Error(`${this.opts.label} connection not open`)
319343
if (op !== "disconnect") {
320344
if (this.sessions.get(connectionId) !== session) throw new Error(`${this.opts.label} connection not open`)
321-
// Reject calls queued before a restart or before this session acknowledged
322-
// it, even if an earlier rejection has since acknowledged it.
323-
if (enqueued !== this.generation || acknowledged !== this.generation) {
345+
// Reject the entire stale burst only when this session may have lost
346+
// state, even if an earlier rejection has since acknowledged that loss.
347+
if (enqueued < session.lostGeneration || acknowledged < session.lostGeneration) {
324348
session.generation = this.generation
325349
throw this.sessionStateLostError()
326350
}
327351
}
328-
const child = await this.ensureChild()
352+
const child = await this.ensureChild(signal)
329353
if (this.closed) throw new Error(`${this.opts.label} connection closed`)
354+
// Mark before dispatch: a batch may change state before it times out.
355+
if (op !== "disconnect" && !isStatelessRead(payload["sql"])) {
356+
session.stateful = true
357+
// SQLite shares one native session. DuckDB configuration, attachments
358+
// and loaded extensions can affect every connection in the helper.
359+
if (this.opts.engine === "sqlite" || /\b(SET|RESET|USE|ATTACH|DETACH|PRAGMA|INSTALL|LOAD|LOAD_EXTENSION|SETSEED)\b/i.test(String(payload["sql"])) || /\b(CREATE|DROP|ALTER)\s+(OR\s+REPLACE\s+)?(PERSISTENT\s+|TEMPORARY\s+)?(IF\s+(NOT\s+)?EXISTS\s+)?SECRET\b/i.test(String(payload["sql"]))) {
360+
this.sharedStateful = true
361+
}
362+
}
363+
// An aborted active call retires the helper. The session stays unacknowledged
364+
// so its next statement reports SESSION_STATE_LOST rather than silently
365+
// running outside the discarded transaction.
330366
return this.send(child, op, payload, op === "disconnect" ? CLOSE_TIMEOUT_MS : timeoutMs, () =>
331367
op === "disconnect"
332368
? new Error(`${this.opts.label} session did not finish closing`)
333369
: this.discloseTimeout(session, timeoutMs!),
334370
onDispatch,
371+
signal,
335372
) as Promise<T>
336-
})
373+
}, signal)
337374
}
338375

339376
async close(): Promise<void> {
@@ -363,10 +400,20 @@ export class LocalEngine {
363400

364401
// -- internals -------------------------------------------------------------
365402

366-
private enqueue<T>(fn: () => Promise<T>): Promise<T> {
367-
const run = this.tail.then(fn, fn)
368-
this.tail = run.catch(() => {})
369-
return run
403+
private enqueue<T>(fn: () => Promise<T>, signal?: AbortSignal): Promise<T> {
404+
return new Promise<T>((resolve, reject) => {
405+
const abort = () => reject(signal!.reason)
406+
signal?.addEventListener("abort", abort, { once: true })
407+
if (signal?.aborted) abort()
408+
const start = () => {
409+
signal?.removeEventListener("abort", abort)
410+
signal?.throwIfAborted()
411+
return fn()
412+
}
413+
const run = this.tail.then(start, start)
414+
this.tail = run.catch(() => {})
415+
run.then(resolve, reject)
416+
})
370417
}
371418

372419
private timeoutQueryError(timeoutMs: number): QueryTimeoutError {
@@ -401,6 +448,9 @@ export class LocalEngine {
401448
if (child.retired) return
402449
child.retired = true
403450
this.generation++
451+
for (const session of this.sessions.values()) {
452+
if (session.stateful || this.sharedStateful) session.lostGeneration = this.generation
453+
}
404454
}
405455

406456
private spawn(request: OpenRequest): Child {
@@ -413,16 +463,17 @@ export class LocalEngine {
413463
env: buildHelperEnv(dir, process.env, typeof request.payload["sandboxDirectory"] === "string"),
414464
stdin: "pipe",
415465
stdout: "pipe",
416-
stderr: "ignore",
466+
stderr: "pipe",
417467
})
418468
// altimate_change start — remember this process's spill path across helper respawns
419469
const spillDirectory = this.opts.engine === "duckdb" ? path.join(dir, `duckdb-${proc.pid}`) : undefined
420470
// altimate_change end
421-
const child: Child = { proc, retired: false, killedByUs: false }
471+
const child: Child = { proc, retired: false, killedByUs: false, stderrTail: "" }
422472
liveChildren.add(child)
423473
this.child = child
424474
void this.readLoop(child)
425-
void proc.exited.then(() => {
475+
const stderrDrained = this.readStderr(child)
476+
void proc.exited.then(async (exitCode) => {
426477
// altimate_change start — SIGKILL cannot let DuckDB clean up; wait for actual exit, not the reap timeout
427478
if (spillDirectory) {
428479
try {
@@ -434,13 +485,37 @@ export class LocalEngine {
434485
if (this.child === child) this.child = undefined
435486
const pending = child.pending
436487
child.pending = undefined
437-
if (pending && !child.killedByUs)
438-
pending.reject(new Error(`${this.opts.label} helper process exited unexpectedly`))
488+
if (pending && !child.killedByUs) {
489+
let timer: ReturnType<typeof setTimeout> | undefined
490+
await Promise.race([
491+
stderrDrained,
492+
new Promise<void>((resolve) => { timer = setTimeout(resolve, STDERR_DRAIN_TIMEOUT_MS) }),
493+
])
494+
if (timer) clearTimeout(timer)
495+
pending.reject(new LocalEngineProcessError({
496+
engine: this.opts.label,
497+
exitCode,
498+
signal: proc.signalCode,
499+
stderr: child.stderrTail,
500+
}))
501+
}
439502
// altimate_change end
440503
})
441504
return child
442505
}
443506

507+
private async readStderr(child: Child): Promise<void> {
508+
const decoder = new TextDecoder()
509+
try {
510+
for await (const chunk of child.proc.stderr) {
511+
child.stderrTail = (child.stderrTail + decoder.decode(chunk, { stream: true })).slice(-STDERR_TAIL_CHARS)
512+
}
513+
child.stderrTail = (child.stderrTail + decoder.decode()).slice(-STDERR_TAIL_CHARS)
514+
} catch {
515+
// Preserve diagnostics already received if the pipe closes with an error.
516+
}
517+
}
518+
444519
private async readLoop(child: Child): Promise<void> {
445520
const decoder = new TextDecoder()
446521
// altimate_change start — scan each chunk once, joining a line only at its terminator
@@ -484,11 +559,12 @@ export class LocalEngine {
484559
}
485560

486561
/** The live process, respawned and reopened after a kill or crash. */
487-
private async ensureChild(): Promise<Child> {
562+
private async ensureChild(signal?: AbortSignal): Promise<Child> {
563+
signal?.throwIfAborted()
488564
if (this.child && !this.child.retired) return this.child
489565
const child = this.spawn(this.openRequest!)
490566
try {
491-
await this.sendOpen(child, this.openRequest!)
567+
await this.sendOpen(child, this.openRequest!, signal)
492568
} catch (err) {
493569
await this.retire(child)
494570
throw err
@@ -503,16 +579,20 @@ export class LocalEngine {
503579
timeoutMs: number | undefined,
504580
onTimeout: () => Error,
505581
onDispatch?: () => void,
582+
signal?: AbortSignal,
506583
): Promise<any> {
507584
return new Promise((resolve, reject) => {
508585
if (child.retired) return reject(new Error(`${this.opts.label} helper process is not running`))
586+
if (signal?.aborted) return reject(signal.reason)
509587
const id = ++this.seq
510588
let timer: ReturnType<typeof setTimeout> | undefined
511589
let settled = false
590+
let stopping = false
512591
const settle = (fn: () => void) => {
513592
if (settled) return
514593
settled = true
515594
if (timer) clearTimeout(timer)
595+
signal?.removeEventListener("abort", abort)
516596
if (child.pending?.id === id) child.pending = undefined
517597
fn()
518598
}
@@ -521,12 +601,19 @@ export class LocalEngine {
521601
resolve: (v) => settle(() => resolve(v)),
522602
reject: (e) => settle(() => reject(e)),
523603
}
604+
const stop = (error: () => unknown) => {
605+
if (settled || stopping) return
606+
stopping = true
607+
void this.retire(child).then(() => settle(() => reject(error())))
608+
}
609+
const abort = () => stop(() => signal!.reason)
610+
signal?.addEventListener("abort", abort, { once: true })
524611
if (timeoutMs !== undefined) {
525612
timer = setTimeout(() => {
526-
if (settled) return
613+
if (settled || child.retired) return
527614
// Stop the work first, then report: the caller must not see the
528615
// timeout while the process still holds the store.
529-
void this.retire(child).then(() => settle(() => reject(onTimeout())))
616+
stop(onTimeout)
530617
}, timeoutMs)
531618
}
532619
try {

0 commit comments

Comments
 (0)