Skip to content

Commit 91281f9

Browse files
anandgupta42claude
andcommitted
fix: scope text_label fatal errors and reject NULL text or empty populations
- Stop the whole run immediately only on 401/403/404, or when permanent 4xx errors repeat with no successful batch; a batch-specific 4xx fails only that batch. - Mark NULL source text as failed with an explicit error instead of labeling "". - Validate id/text columns and reject an empty population before creating a table. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 979ff37 commit 91281f9

3 files changed

Lines changed: 75 additions & 8 deletions

File tree

‎packages/opencode/src/altimate/tools/text-label.ts‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ const MAX_BYTES = 64 * 1024 * 1024
2424
const IDENTIFIER = /^[A-Za-z_][A-Za-z0-9_]{0,62}$/
2525

2626
export type Label = { label: string; description: string }
27-
export type Row = { id: string | number | bigint; text: string }
27+
export type Row = { id: string | number | bigint; text: string; missing?: boolean }
2828
export type Status = "labeled" | "failed" | "unparseable"
2929
export type Labeller = (system: string, user: string, abort: AbortSignal) => Promise<string>
3030

@@ -87,7 +87,7 @@ export function clampInt(value: number | undefined, fallback: number, min: numbe
8787
return Math.min(max, Math.max(min, Math.floor(value)))
8888
}
8989

90-
function errorDetails(error: unknown): { message: string; transient: boolean; permanentHttp: boolean } {
90+
function errorDetails(error: unknown): { message: string; transient: boolean; permanentHttp: boolean; status?: number } {
9191
// AI SDK retry errors can wrap the provider's APICallError in lastError/cause.
9292
let current = error
9393
const seen = new Set<unknown>()
@@ -109,6 +109,7 @@ function errorDetails(error: unknown): { message: string; transient: boolean; pe
109109
message: `HTTP ${status}: ${e.message ?? String(current)}`,
110110
transient,
111111
permanentHttp: status >= 400 && status < 500 && !transient,
112+
status,
112113
}
113114
}
114115
const nested = e.lastError ?? e.cause
@@ -168,6 +169,11 @@ export async function labelRows(input: {
168169
let chars = 2
169170
const size = clampInt(input.batchSize, DEFAULT_BATCH, 1, MAX_BATCH)
170171
for (let index = 0; index < input.rows.length; index++) {
172+
if (input.rows[index].missing) {
173+
rowErrors[index] = "Source text is NULL; nothing to classify."
174+
record(rowErrors[index]!)
175+
continue
176+
}
171177
const length = JSON.stringify({ row: size, text: input.rows[index].text }).length + 1
172178
if (length > BATCH_CHARS) {
173179
rowErrors[index] = "Text exceeds the model batch size limit; shorten it explicitly before retrying."
@@ -193,6 +199,8 @@ export async function labelRows(input: {
193199
let next = 0
194200
let batches = 0
195201
let fatal: string | undefined
202+
let succeeded = 0
203+
let permanentFailures = 0
196204
async function worker() {
197205
while (next < jobs.length && !fatal && !signal.aborted && Date.now() < input.deadline) {
198206
let pending = jobs[next++]
@@ -213,7 +221,13 @@ export async function labelRows(input: {
213221
status[i] = "failed"
214222
rowErrors[i] = detail.message
215223
}
216-
if (detail.permanentHttp) fatal = detail.message
224+
if (detail.permanentHttp) {
225+
// Credential/model errors affect every batch. Other 4xx (e.g. a content filter) may be batch-specific,
226+
// so stop the run only when they repeat with no batch having succeeded.
227+
permanentFailures++
228+
if ([401, 403, 404].includes(detail.status ?? 0) || (permanentFailures >= 3 && succeeded === 0))
229+
fatal = detail.message
230+
}
217231
if (!detail.transient || transientRetries >= 2 || signal.aborted || fatal) break
218232
try {
219233
await delay(250 * 2 ** transientRetries++, undefined, { signal })
@@ -222,6 +236,7 @@ export async function labelRows(input: {
222236
}
223237
continue
224238
}
239+
succeeded++
225240
const parsed = parseReply(reply, input.labels, pending.length)
226241
pending = pending.filter((index, position) => {
227242
const value = parsed.get(position + 1)
@@ -333,8 +348,9 @@ export async function labelScratch(input: TextLabelInput, context: LabelContext,
333348
const columns = data.columns.map((column) => column.toLowerCase())
334349
const idIndex = columns.indexOf(args.id_column.toLowerCase())
335350
const textIndex = columns.indexOf(args.text_column.toLowerCase())
336-
if (data.rows.length && (idIndex < 0 || textIndex < 0 || idIndex === textIndex))
351+
if (idIndex < 0 || textIndex < 0 || idIndex === textIndex)
337352
return fail("id_column and text_column must identify distinct columns in the query result.")
353+
if (!data.rows.length) return fail("The query returned no rows; nothing to label.")
338354
const rows: Row[] = []
339355
const ids = new Set<string>()
340356
let numeric: boolean | undefined
@@ -352,7 +368,7 @@ export async function labelScratch(input: TextLabelInput, context: LabelContext,
352368
const text = row[textIndex]
353369
if (text !== null && typeof text !== "string")
354370
return fail("text_column must be text or NULL; cast other types to VARCHAR in SQL.")
355-
rows.push({ id, text: text ?? "" })
371+
rows.push({ id, text: text ?? "", missing: text === null })
356372
}
357373
const deadline = Date.now() + (args.max_seconds ?? DEFAULT_SECONDS) * 1000
358374
const concurrency = clampInt(Number(process.env.ALTIMATE_TEXT_LABEL_CONCURRENCY), DEFAULT_CONCURRENCY, 1, 16)

‎packages/opencode/test/altimate/text-label-model.test.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,7 @@ it.instance("uses the turn's session model with small calls and preserves stream
127127
})
128128
expect(result.output).toContain("HTTP 400: Provider rejected unsupported thinkingBudget")
129129
expect(result.output).not.toContain("No output generated")
130-
expect(stream).toHaveBeenCalledTimes(1)
130+
expect(stream).toHaveBeenCalledTimes(3)
131131
expect(getModel).toHaveBeenCalledWith(model.providerID, model.id)
132132
expect(defaultModel).not.toHaveBeenCalled()
133133
expect(getSmallModel).not.toHaveBeenCalled()

‎packages/opencode/test/altimate/tools/text-label.test.ts‎

Lines changed: 53 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -187,7 +187,7 @@ describe("text_label batching and coverage", () => {
187187
expect(result.errors.join("\n")).toContain("provider temporarily unavailable")
188188
})
189189

190-
test("a provider 400 preserves status and message, never retries, and stops new batches", async () => {
190+
test("a repeated provider 400 preserves status and message, never retries a batch, and stops new batches", async () => {
191191
let calls = 0
192192
const result = await run({
193193
concurrency: 1,
@@ -196,13 +196,56 @@ describe("text_label batching and coverage", () => {
196196
throw providerError(400, "Unsupported thinkingBudget for this model")
197197
},
198198
})
199-
expect(calls).toBe(1)
199+
expect(calls).toBe(3)
200200
expect(result.status).toEqual(Array(rows.length).fill("failed"))
201201
expect(result.errors.join("\n")).toContain("400")
202202
expect(result.errors.join("\n")).toContain("Unsupported thinkingBudget for this model")
203203
expect(result.errors.join("\n")).not.toContain("No output generated")
204204
})
205205

206+
test("an auth failure stops the run after one call", async () => {
207+
let calls = 0
208+
const result = await run({
209+
concurrency: 1,
210+
labeller: async () => {
211+
calls++
212+
throw providerError(401, "bad credentials")
213+
},
214+
})
215+
expect(calls).toBe(1)
216+
expect(result.status).toEqual(Array(rows.length).fill("failed"))
217+
})
218+
219+
test("one batch-specific 400 fails only that batch", async () => {
220+
let calls = 0
221+
const result = await run({
222+
concurrency: 1,
223+
labeller: async (system, user, abort) => {
224+
if (calls++ === 1) throw providerError(400, "content filtered")
225+
return answer(system, user)
226+
},
227+
})
228+
expect(result.status.filter((status) => status === "failed")).toHaveLength(4)
229+
expect(result.status.filter((status) => status === "labeled")).toHaveLength(rows.length - 4)
230+
})
231+
232+
test("NULL source text is failed with an explicit error and never sent to the model", async () => {
233+
const sent: string[] = []
234+
const result = await run({
235+
rows: [
236+
{ id: 1, text: "hello" },
237+
{ id: 2, text: "", missing: true },
238+
],
239+
labeller: async (system, user) => {
240+
sent.push(user)
241+
return answer(system, user)
242+
},
243+
})
244+
expect(result.status).toEqual(["labeled", "failed"])
245+
expect(result.rowErrors[1]).toContain("NULL")
246+
expect(sent.join("")).not.toContain('"row":2')
247+
})
248+
206249
test("out-of-set labels are unparseable, including after the missing-row retry", async () => {
207250
let calls = 0
208251
const result = await run({
@@ -317,6 +360,14 @@ describe("text_label scratch results", () => {
317360
])
318361
})
319362

363+
test("an empty population or wrong column names is rejected without creating a table", async () => {
364+
const empty = await labelScratch({ ...input, sql: `${input.sql} WHERE 1 = 0` }, { abort: never }, answer)
365+
expect(empty.metadata).toMatchObject({ ok: false, persisted: false })
366+
const wrong = await labelScratch({ ...input, sql: `${input.sql} WHERE 1 = 0`, text_column: "nope" }, { abort: never }, answer)
367+
expect(wrong.metadata).toMatchObject({ ok: false, persisted: false })
368+
expect(wrong.output).toContain("columns")
369+
})
370+
320371
test("failed batches remain in scratch and their IDs are retrievable, with prominent coverage warnings", async () => {
321372
const result = await labelScratch(input, { abort: never }, async (system, user) => {
322373
if (JSON.parse(user).some((row: { text: string }) => row.text === "row 3")) {

0 commit comments

Comments
 (0)