Skip to content

Commit 960229d

Browse files
anandgupta42claude
andcommitted
fix: address round-4 review comments (Codex connector, CodeRabbit, Kilo)
Abstention and fail-closed hardening: - `check --checks pii`: engine `parse_error` abstentions now produce an error finding ("PII analysis skipped: …") instead of an empty pass — `--fail-on` no longer PASSes files whose PII analysis never ran. - Composite `altimate_core.check`: a thrown `checkQueryPii` now marks the PII section as an abstention (`parse_error`) instead of leaving `{}`, which rendered a false-clean "No PII detected"; `formatCheckTitle` reports "PII check skipped" instead of PASS for abstained-but-clean runs. - `schema.detect_pii`: per-column classify failures are counted and flip `success` to false (fail closed) instead of being silently swallowed; `piiColumnsFromReport` now throws on malformed reports (missing/non-array `columns`) rather than yielding zero findings. - `altimate_core_query_pii` tool: abstentions set `metadata.success: false` so telemetry's soft-failure classification records them. Correctness: - `check --checks grade`: per-file grades (`results.grade.grades`) — the shared `gradeValue`/`gradeScore` raced across concurrent batch promises, keeping whichever file finished last; flat `grade`/`score` retained for single-file runs only. Module shape / dedup: - `engine-coerce.ts` gets the AGENTS.md-prescribed `EngineCoerce` self-reexport; all consumers import the namespace projection. - `review/runner.ts` drops its local `bandConfidence` for the shared one; shared version now bands missing/non-numeric confidence as "medium" (matching the runner's previous behavior). Tests: CLI PII-abstention and multi-file grade regression tests; "PII check skipped" title test; strict malformed-PiiReport assertions; classify-pii e2e asserts a positive `pii_count` lower bound; DuckDB e2e closes its connector and restores the telemetry env var on teardown. Co-Authored-By: Claude Fable 5 <[email protected]>
1 parent a00fb38 commit 960229d

12 files changed

Lines changed: 169 additions & 68 deletions

‎packages/opencode/src/altimate/native/altimate-core.ts‎

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
*/
1010

1111
import * as core from "@altimateai/altimate-core"
12-
import { dialectHint } from "./engine-coerce"
12+
import { EngineCoerce } from "./engine-coerce"
1313
import { register } from "./dispatcher"
1414
import { schemaOrEmpty, resolveSchema } from "./schema-resolver"
1515
import type { AltimateCoreResult } from "./types"
@@ -185,11 +185,13 @@ export function registerAll(): void {
185185
// PII exposure for the composite check — the tool has always rendered a
186186
// PII section; previously nothing populated it. Additive: a PII failure
187187
// must not fail the whole composite.
188-
let pii: Record<string, unknown> = {}
188+
let pii: Record<string, unknown>
189189
try {
190190
pii = toData(core.checkQueryPii(params.sql, schema))
191-
} catch {
192-
// validation/lint above already report unparseable SQL
191+
} catch (e) {
192+
// Mark as an abstention — an empty object would render "No PII
193+
// detected", a false-clean verdict.
194+
pii = { parse_error: String(e) }
193195
}
194196
const data: Record<string, unknown> = {
195197
validation: toData(validation),
@@ -337,7 +339,7 @@ export function registerAll(): void {
337339
// [email protected]. dialectHint coerces "" (the ReviewConfig default)
338340
// to undefined: the engine throws on an unknown dialect "", and "" must
339341
// mean auto-detect, not a real dialect.
340-
const raw = await core.checkEquivalence(params.sql1, params.sql2, schema, dialectHint(params.dialect))
342+
const raw = await core.checkEquivalence(params.sql1, params.sql2, schema, EngineCoerce.dialectHint(params.dialect))
341343
const data = toData(raw)
342344
return ok(true, data)
343345
} catch (e) {
@@ -350,7 +352,7 @@ export function registerAll(): void {
350352
try {
351353
// Build schema from old_ddl, analyze new_ddl against it. dialectHint
352354
// coerces "" to auto-detect (the engine throws on an unknown dialect "").
353-
const schema = core.Schema.fromDdl(params.old_ddl, dialectHint(params.dialect))
355+
const schema = core.Schema.fromDdl(params.old_ddl, EngineCoerce.dialectHint(params.dialect))
354356
const raw = core.analyzeMigration(params.new_ddl, schema)
355357
const data = toData(raw)
356358
return ok(true, data)
@@ -444,7 +446,7 @@ export function registerAll(): void {
444446
register("altimate_core.column_lineage", async (params) => {
445447
try {
446448
const schema = resolveSchema(params.schema_path, params.schema_context)
447-
const raw = core.columnLineage(params.sql, dialectHint(params.dialect), schema ?? undefined)
449+
const raw = core.columnLineage(params.sql, EngineCoerce.dialectHint(params.dialect), schema ?? undefined)
448450
return ok(true, toData(raw))
449451
} catch (e) {
450452
return fail(e)
@@ -465,7 +467,7 @@ export function registerAll(): void {
465467
// 22. altimate_core.format
466468
register("altimate_core.format", async (params) => {
467469
try {
468-
const raw = core.formatSql(params.sql, dialectHint(params.dialect))
470+
const raw = core.formatSql(params.sql, EngineCoerce.dialectHint(params.dialect))
469471
const data = toData(raw)
470472
return ok(true, data)
471473
} catch (e) {
@@ -476,7 +478,7 @@ export function registerAll(): void {
476478
// 23. altimate_core.metadata
477479
register("altimate_core.metadata", async (params) => {
478480
try {
479-
const raw = core.extractMetadata(params.sql, dialectHint(params.dialect))
481+
const raw = core.extractMetadata(params.sql, EngineCoerce.dialectHint(params.dialect))
480482
return ok(true, toData(raw))
481483
} catch (e) {
482484
return fail(e)
@@ -486,7 +488,7 @@ export function registerAll(): void {
486488
// 24. altimate_core.compare
487489
register("altimate_core.compare", async (params) => {
488490
try {
489-
const raw = core.compareQueries(params.left_sql, params.right_sql, dialectHint(params.dialect))
491+
const raw = core.compareQueries(params.left_sql, params.right_sql, EngineCoerce.dialectHint(params.dialect))
490492
return ok(true, toData(raw))
491493
} catch (e) {
492494
return fail(e)
@@ -540,7 +542,7 @@ export function registerAll(): void {
540542
// 29. altimate_core.import_ddl — returns Schema, must serialize
541543
register("altimate_core.import_ddl", async (params) => {
542544
try {
543-
const schema = core.importDdl(params.ddl, dialectHint(params.dialect))
545+
const schema = core.importDdl(params.ddl, EngineCoerce.dialectHint(params.dialect))
544546
const jsonObj = schema.toJson()
545547
return ok(true, { success: true, schema: toData(jsonObj) })
546548
} catch (e) {

‎packages/opencode/src/altimate/native/engine-coerce.ts‎

Lines changed: 22 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,15 @@
11
/**
22
* Shared coercions for altimate-core engine values.
33
*
4-
* The engine's napi surface has two recurring foot-guns for consumers:
4+
* The engine's napi surface has recurring foot-guns for consumers:
55
* - `PiiClassification` is a string OR `{ Custom: string }` — naive string
66
* interpolation renders `[object Object]`.
77
* - dialect parameters throw on the empty string (`unknown dialect ''`);
88
* `""` must mean "auto-detect" and be passed as undefined.
9+
* - confidence is numeric (0..1) while several consumers declare string bands.
10+
*
11+
* This module must stay free of `@altimateai/altimate-core` imports so tool
12+
* modules can use it without eagerly loading the native NAPI binding.
913
*/
1014

1115
/** PiiClassification is 'Email' | … | { Custom: string } | 'None'. */
@@ -22,15 +26,16 @@ export function dialectHint(dialect: string | undefined | null): string | undefi
2226
return dialect || undefined
2327
}
2428

25-
/** Map the engine's numeric confidence (0..1) to a string band. */
29+
/**
30+
* Map the engine's numeric confidence (0..1) to a string band.
31+
* Missing or non-numeric confidence is unknown, not low — band it "medium".
32+
*/
2633
export function bandConfidence(c: unknown): "high" | "medium" | "low" {
27-
// Missing confidence is unknown, not low.
28-
if (c == null) return "medium"
2934
if (typeof c === "string") {
3035
const s = c.toLowerCase()
3136
if (s === "high" || s === "medium" || s === "low") return s
3237
}
33-
const n = typeof c === "number" ? c : NaN
38+
const n = typeof c === "number" && Number.isFinite(c) ? c : 0.5
3439
if (n >= 0.8) return "high"
3540
if (n >= 0.5) return "medium"
3641
return "low"
@@ -40,11 +45,18 @@ export function bandConfidence(c: unknown): "high" | "medium" | "low" {
4045
* Extract the real PII rows from an engine PiiReport.
4146
*
4247
* The engine returns `{ columns, pii_count, risk_level, total_columns }` with
43-
* a row for EVERY column — classification "None" means not PII. Lives here
44-
* (not in pii-detector) so tool modules can import it without eagerly loading
45-
* the native NAPI binding at registry-load time.
48+
* a row for EVERY column — classification "None" means not PII.
49+
*
50+
* Fails closed: a report without an array `columns` field is malformed (the
51+
* engine always emits one) and throws instead of silently yielding zero
52+
* findings — silent-empty output is exactly the bug class this fixes.
4653
*/
4754
export function piiColumnsFromReport(piiData: unknown): Array<Record<string, any>> {
48-
const columns = ((piiData as Record<string, any>)?.columns ?? []) as Array<Record<string, any>>
49-
return columns.filter((c) => c.classification !== "None")
55+
const columns = (piiData as Record<string, any> | null | undefined)?.columns
56+
if (!Array.isArray(columns)) {
57+
throw new TypeError("malformed PiiReport: missing columns array")
58+
}
59+
return columns.filter((c) => c && c.classification !== "None")
5060
}
61+
62+
export * as EngineCoerce from "./engine-coerce"

‎packages/opencode/src/altimate/native/schema/pii-detector.ts‎

Lines changed: 20 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,18 @@
44
*/
55

66
import * as core from "@altimateai/altimate-core"
7-
import { bandConfidence, classificationToString, piiColumnsFromReport } from "../engine-coerce"
7+
import { EngineCoerce } from "../engine-coerce"
88
import { getCache } from "./cache"
9-
10-
export { piiColumnsFromReport }
119
import * as Registry from "../connections/registry"
1210
import type {
1311
PiiDetectParams,
1412
PiiDetectResult,
1513
PiiFinding,
1614
} from "../types"
1715

16+
/** Re-exported for tests and legacy importers; lives in engine-coerce. */
17+
export const piiColumnsFromReport = EngineCoerce.piiColumnsFromReport
18+
1819
/**
1920
* Detect PII in cached schema metadata by running altimate-core's
2021
* classifyPii() on column names and types.
@@ -46,6 +47,7 @@ export async function detectPii(params: PiiDetectParams): Promise<PiiDetectResul
4647

4748
const findings: PiiFinding[] = []
4849
let columnsScanned = 0
50+
let scanErrors = 0
4951
const tablesWithPii = new Set<string>()
5052

5153
for (const wh of targetWarehouses) {
@@ -77,21 +79,23 @@ export async function detectPii(params: PiiDetectParams): Promise<PiiDetectResul
7779

7880
// Engine PiiReport: { columns, pii_count, risk_level, total_columns };
7981
// every column gets a row — classification "None" means not PII.
80-
const piiColumns = piiColumnsFromReport(piiData)
82+
const piiColumns = EngineCoerce.piiColumnsFromReport(piiData)
8183
for (const piiCol of piiColumns) {
8284
findings.push({
8385
warehouse: col.warehouse,
8486
schema: col.schema_name,
8587
table: col.table,
8688
column: col.name,
8789
data_type: col.data_type,
88-
pii_category: classificationToString(piiCol.classification, "UNKNOWN"),
89-
confidence: bandConfidence(piiCol.confidence),
90+
pii_category: EngineCoerce.classificationToString(piiCol.classification, "UNKNOWN"),
91+
confidence: EngineCoerce.bandConfidence(piiCol.confidence),
9092
})
9193
tablesWithPii.add(`${col.warehouse}.${col.schema_name}.${col.table}`)
9294
}
9395
} catch {
94-
// classifyPii may not find PII — that is expected
96+
// classifyPii threw or returned a malformed report — record it so the
97+
// scan fails closed instead of silently reporting fewer findings.
98+
scanErrors++
9599
}
96100
}
97101
}
@@ -103,7 +107,7 @@ export async function detectPii(params: PiiDetectParams): Promise<PiiDetectResul
103107
}
104108

105109
return {
106-
success: true,
110+
success: scanErrors === 0,
107111
findings,
108112
finding_count: findings.length,
109113
columns_scanned: columnsScanned,
@@ -138,6 +142,7 @@ async function detectPiiLive(params: PiiDetectParams): Promise<PiiDetectResult>
138142

139143
const findings: PiiFinding[] = []
140144
let columnsScanned = 0
145+
let scanErrors = 0
141146
const tablesWithPii = new Set<string>()
142147

143148
for (const schemaName of schemas) {
@@ -170,21 +175,23 @@ async function detectPiiLive(params: PiiDetectParams): Promise<PiiDetectResult>
170175
const piiData = JSON.parse(JSON.stringify(result))
171176

172177
// Engine PiiReport: { columns, … } with a row per column; "None" = not PII.
173-
const piiColumns = piiColumnsFromReport(piiData)
178+
const piiColumns = EngineCoerce.piiColumnsFromReport(piiData)
174179
for (const piiCol of piiColumns) {
175180
findings.push({
176181
warehouse: params.warehouse!,
177182
schema: schemaName,
178183
table: tableInfo.name,
179184
column: piiCol.column || "",
180185
data_type: columns.find((c) => c.name === piiCol.column)?.data_type,
181-
pii_category: classificationToString(piiCol.classification, "UNKNOWN"),
182-
confidence: bandConfidence(piiCol.confidence),
186+
pii_category: EngineCoerce.classificationToString(piiCol.classification, "UNKNOWN"),
187+
confidence: EngineCoerce.bandConfidence(piiCol.confidence),
183188
})
184189
tablesWithPii.add(`${params.warehouse}.${schemaName}.${tableInfo.name}`)
185190
}
186191
} catch {
187-
// ignore
192+
// classifyPii threw or returned a malformed report — record it so
193+
// the scan fails closed instead of silently reporting fewer findings.
194+
scanErrors++
188195
}
189196
}
190197
}
@@ -195,7 +202,7 @@ async function detectPiiLive(params: PiiDetectParams): Promise<PiiDetectResult>
195202
}
196203

197204
return {
198-
success: true,
205+
success: scanErrors === 0,
199206
findings,
200207
finding_count: findings.length,
201208
columns_scanned: columnsScanned,

‎packages/opencode/src/altimate/review/runner.ts‎

Lines changed: 3 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { Dispatcher } from "../native"
2-
import { classificationToString } from "../native/engine-coerce"
2+
import { EngineCoerce } from "../native/engine-coerce"
33
import { parseManifest } from "../native/dbt/manifest"
44
import type { CheckResult, EquivalenceResult, GradeResult, ImpactResult, ReviewRunner } from "./orchestrate"
55
import { buildReviewSchemaContext, type SchemaContext } from "./schema-context"
@@ -58,16 +58,6 @@ function piiColumnOf(issue: any): string | undefined {
5858
return issue?.column ?? issue?.target ?? issue?.name ?? undefined
5959
}
6060

61-
/** The engine returns a numeric confidence (0..1); map it to a band. */
62-
function bandConfidence(c: unknown): "high" | "medium" | "low" {
63-
if (typeof c === "string") {
64-
const s = c.toLowerCase()
65-
if (s === "high" || s === "medium" || s === "low") return s
66-
}
67-
const n = typeof c === "number" ? c : 0.5
68-
return n >= 0.8 ? "high" : n >= 0.5 ? "medium" : "low"
69-
}
70-
7161
/**
7262
* Map a core lint finding (by rule name / L0xx code) to a review category.
7363
* Core's `LintFinding` carries no category, so without this every AST finding
@@ -404,7 +394,7 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun
404394
.map((c) => ({
405395
column: String(c.column ?? ""),
406396
// classification can be { Custom: string } — String() would emit "[object Object]"
407-
classification: classificationToString(c.classification, ""),
397+
classification: EngineCoerce.classificationToString(c.classification, ""),
408398
confidence: typeof c.confidence === "number" ? c.confidence : 0,
409399
masking: c.suggested_masking ?? undefined,
410400
}))
@@ -512,7 +502,7 @@ export function createDispatcherRunner(opts: DispatcherRunnerOptions): ReviewRun
512502
decided: true,
513503
equivalent,
514504
differences: diffs.map((d) => d?.description ?? String(d)),
515-
confidence: bandConfidence(data.confidence),
505+
confidence: EngineCoerce.bandConfidence(data.confidence),
516506
}
517507
} catch {
518508
return { decided: false }

‎packages/opencode/src/altimate/tools/altimate-core-check.ts‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import z from "zod"
22
import { Tool } from "../../tool/tool"
33
import { Dispatcher } from "../native"
4-
import { classificationToString } from "../native/engine-coerce"
4+
import { EngineCoerce } from "../native/engine-coerce"
55
import type { Telemetry } from "../telemetry"
66

77
export const AltimateCoreCheckTool = Tool.define("altimate_core_check", {
@@ -63,7 +63,8 @@ export function formatCheckTitle(data: Record<string, any>): string {
6363
if (!data.validation?.valid) parts.push("validation errors")
6464
if (!data.lint?.clean) parts.push(`${data.lint?.findings?.length ?? 0} lint findings`)
6565
if (!data.safety?.safe) parts.push("safety threats")
66-
if (data.pii?.pii_columns?.length || data.pii?.findings?.length) parts.push("PII detected")
66+
if (data.pii?.parse_error) parts.push("PII check skipped")
67+
else if (data.pii?.pii_columns?.length || data.pii?.findings?.length) parts.push("PII detected")
6768
return parts.length ? parts.join(", ") : "PASS"
6869
}
6970

@@ -108,7 +109,7 @@ export function formatCheck(data: Record<string, any>): string {
108109
lines.push("No PII detected.")
109110
} else {
110111
for (const p of piiCols) {
111-
const cls = classificationToString(p.classification ?? p.category)
112+
const cls = EngineCoerce.classificationToString(p.classification ?? p.category)
112113
const where = [p.table, p.column ?? "unknown"].filter(Boolean).join(".")
113114
const via = Array.isArray(p.query_targets) && p.query_targets.length ? ` exposed via: ${p.query_targets.join(", ")}` : ""
114115
lines.push(` ${where}: ${cls}${via}${p.suggested_masking ? ` (masking: ${p.suggested_masking})` : ""}`)

‎packages/opencode/src/altimate/tools/altimate-core-classify-pii.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
import z from "zod"
22
import { Tool } from "../../tool/tool"
33
import { Dispatcher } from "../native"
4-
import { classificationToString, piiColumnsFromReport } from "../native/engine-coerce"
4+
import { EngineCoerce } from "../native/engine-coerce"
55

66
/** Engine PiiReport lists EVERY column; classification "None" means not PII. */
77
function realPiiColumns(data: Record<string, any>): any[] {
8-
if (Array.isArray(data.columns)) return piiColumnsFromReport(data)
8+
if (Array.isArray(data.columns)) return EngineCoerce.piiColumnsFromReport(data)
99
// Legacy fallback shape.
1010
return ((data.findings ?? []) as any[]).filter((c) => c.classification !== "None")
1111
}
@@ -54,7 +54,7 @@ function formatClassifyPii(data: Record<string, any>): string {
5454
lines.push("")
5555
lines.push("PII columns found:")
5656
for (const f of piiColumns) {
57-
const classification = classificationToString(f.classification ?? f.category)
57+
const classification = EngineCoerce.classificationToString(f.classification ?? f.category)
5858
const confidence = f.confidence ?? "high"
5959
const table = f.table ?? "unknown"
6060
const column = f.column ?? "unknown"

‎packages/opencode/src/altimate/tools/altimate-core-query-pii.ts‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
import z from "zod"
22
import { Tool } from "../../tool/tool"
33
import { Dispatcher } from "../native"
4-
import { classificationToString } from "../native/engine-coerce"
4+
import { EngineCoerce } from "../native/engine-coerce"
55

66
export const AltimateCoreQueryPiiTool = Tool.define("altimate_core_query_pii", {
77
description:
@@ -30,7 +30,9 @@ export const AltimateCoreQueryPiiTool = Tool.define("altimate_core_query_pii", {
3030
: `Query PII: ${exposureCount === 0 ? "CLEAN" : `${exposureCount} exposure(s)`}`
3131
return {
3232
title,
33-
metadata: { success: result.success, exposure_count: exposureCount, ...(error && { error }) },
33+
// An abstention (parse_error) is a soft failure — telemetry classifies
34+
// on metadata.success === false, so it must not report success.
35+
metadata: { success: result.success && !error, exposure_count: exposureCount, ...(error && { error }) },
3436
output: error ? `Error: ${error}` : formatQueryPii(data),
3537
}
3638
} catch (e) {
@@ -52,7 +54,7 @@ function formatQueryPii(data: Record<string, any>): string {
5254
if (data.risk_level) lines.push(`Risk level: ${data.risk_level}`)
5355
lines.push("PII exposure detected:\n")
5456
for (const e of piiCols) {
55-
const classification = classificationToString(e.classification ?? e.category)
57+
const classification = EngineCoerce.classificationToString(e.classification ?? e.category)
5658
const table = e.table ?? "unknown"
5759
const column = e.column ?? "unknown"
5860
lines.push(` ${table}.${column}: ${classification}`)

0 commit comments

Comments
 (0)