@@ -117,7 +117,7 @@ export interface ReviewRunner {
117117 impact ( model : string ) : Promise < ImpactResult >
118118 grade ( sql : string , dialect : string ) : Promise < GradeResult >
119119 check ( sql : string , dialect : string , baseSql ?: string ) : Promise < CheckResult >
120- equivalence ( oldSql : string , newSql : string , dialect : string ) : Promise < EquivalenceResult >
120+ equivalence ( oldSql : string , newSql : string , dialect ? : string ) : Promise < EquivalenceResult >
121121 detectPii ( sql : string , dialect : string ) : Promise < { columns : string [ ] } >
122122 /**
123123 * Lexical scan (reserved-word aliases + dialect operators) — the curated lists
@@ -816,6 +816,20 @@ const STRUCTURAL_CATEGORY: Record<string, ReviewCategory> = {
816816 join_key_regression : "join_risk" ,
817817}
818818
819+ // PII classification thresholds for the diff-scoped lane. Names/addresses are
820+ // common, lower-sensitivity PII that the broad name-pattern detector over-flags,
821+ // so the precise lane requires higher confidence before surfacing them. Kept a
822+ // named constant (not a magic number) and deliberately narrow: broadening the
823+ // low-risk set to variants would suppress MORE real columns, hurting recall.
824+ const PII_LOW_RISK_CLASS = / ^ ( n a m e | a d d r e s s ) $ / i
825+ const PII_LOW_RISK_MIN_CONFIDENCE = 0.9
826+ const PII_DEFAULT_MIN_CONFIDENCE = 0.7
827+ // Column-name tokens the regex PII twins (`pii_into_mart`) key off — used to
828+ // recover which column a coarse regex finding flagged so suppression can be
829+ // column-aware (suppress only when core actually classified THAT column).
830+ const PII_TOKEN_RE =
831+ / \b ( e m a i l | s s n | s o c i a l _ s e c u r i t y | p h o n e _ n u m b e r | f i r s t _ n a m e | l a s t _ n a m e | f u l l _ n a m e | s t r e e t _ a d d r e s s | d a t e _ o f _ b i r t h | d o b | p a s s p o r t | c r e d i t _ c a r d ) \b / i
832+
819833async function structuralChangeLane ( ctx : ModelContext , runner : ReviewRunner ) : Promise < Finding [ ] > {
820834 if ( ! runner . structuralDiff ) return [ ]
821835 // Use the ENGINE (compiled) SQL — structural_diff parses with a real SQL parser,
@@ -829,7 +843,11 @@ async function structuralChangeLane(ctx: ModelContext, runner: ReviewRunner): Pr
829843 const out : Finding [ ] = [ ]
830844 for ( const f of raw ) {
831845 const cat : ReviewCategory = STRUCTURAL_CATEGORY [ f . rule ] ?? "semantic_change"
832- const sev = clampSeverity ( cat , f . severity === "error" ? "critical" : "warning" , "high" )
846+ // Only `join_risk` (SC010 join-key regression) is allowed to block: a changed
847+ // join key silently corrupts row sets. Other structural rules core marks
848+ // `error` stay advisory (warning) so this lane never over-blocks on a rule
849+ // it wasn't designed to gate.
850+ const sev = clampSeverity ( cat , cat === "join_risk" && f . severity === "error" ? "critical" : "warning" , "high" )
833851 out . push (
834852 makeFinding ( {
835853 severity : sev ,
@@ -920,8 +938,8 @@ async function piiClassifyLane(
920938 ctx : ModelContext ,
921939 runner : ReviewRunner ,
922940 dialect : string ,
923- ) : Promise < { findings : Finding [ ] ; introducedColumns : Set < string > ; completed : boolean } > {
924- const empty = { findings : [ ] , introducedColumns : new Set < string > ( ) , completed : false }
941+ ) : Promise < { findings : Finding [ ] ; classifiedColumns : Set < string > ; completed : boolean } > {
942+ const empty = { findings : [ ] , classifiedColumns : new Set < string > ( ) , completed : false }
925943 if ( ! runner . classifyPii ) return empty
926944 const { file, engineNewSql } = ctx
927945 if ( ! engineNewSql || file . status === "deleted" ) return empty
@@ -940,20 +958,29 @@ async function piiClassifyLane(
940958 } catch {
941959 return empty
942960 }
943- const introducedColumns = new Set ( newCols . map ( ( c ) => c . toLowerCase ( ) ) )
944- if ( ! newCols . length ) return { findings : [ ] , introducedColumns, completed : true }
961+ if ( ! newCols . length ) return { findings : [ ] , classifiedColumns : new Set < string > ( ) , completed : true }
945962 let pii
946963 try {
947964 pii = await runner . classifyPii ( newCols )
948965 } catch {
949966 return empty
950967 }
968+ // Columns core's classifier actually CONSIDERED (returned a verdict for), even
969+ // if below threshold. The broad fallback and the regex twins dedup against THIS
970+ // set — not all introduced columns — so a column core simply MISSED (never
971+ // classified) stays eligible for the broad name-pattern detector instead of
972+ // being silently dropped.
973+ const classifiedColumns = new Set ( pii . map ( ( p ) => String ( p . column ?? "" ) . toLowerCase ( ) ) . filter ( Boolean ) )
951974 const out : Finding [ ] = [ ]
975+ // "Exposure" severity only escalates for published/broadly-read layers
976+ // (marts/reporting). This is intentionally narrower than where grain tests
977+ // apply (`missingGrainTestLane` also includes `intermediate`): an internal
978+ // intermediate model surfacing PII is a warning, not a hard block.
952979 const highExposureLayer = / ( ^ | \/ ) ( m a r t s | r e p o r t i n g ) \/ / . test ( file . path )
953980 for ( const p of pii ) {
954981 const classification = String ( p . classification ?? "" )
955- const lowRiskClass = / ^ ( n a m e | a d d r e s s ) $ / i . test ( classification )
956- const minConfidence = lowRiskClass ? 0.9 : 0.7
982+ const lowRiskClass = PII_LOW_RISK_CLASS . test ( classification )
983+ const minConfidence = lowRiskClass ? PII_LOW_RISK_MIN_CONFIDENCE : PII_DEFAULT_MIN_CONFIDENCE
957984 if ( p . confidence < minConfidence || ! p . column ) continue
958985 const col = p . column . toLowerCase ( )
959986 const highConfidence = p . confidence >= 0.9
@@ -975,7 +1002,7 @@ async function piiClassifyLane(
9751002 } ) ,
9761003 )
9771004 }
978- return { findings : out , introducedColumns , completed : true }
1005+ return { findings : out , classifiedColumns , completed : true }
9791006}
9801007
9811008function piiLane ( file : ChangedFile & { kind : string } , columns : string [ ] ) : Finding [ ] {
@@ -1068,7 +1095,13 @@ export async function runReview(input: OrchestrateInput): Promise<VerdictEnvelop
10681095 const lanes = new Set ( input . config . reviewers . length ? input . config . reviewers : TIER_LANES [ tier ] )
10691096
10701097 const all : Finding [ ] [ ] = [ ]
1071- const diffScopedPiiFiles = new Set < string > ( )
1098+ // Files where the diff-scoped PII classifier completed (looked at the change),
1099+ // those where it actually emitted a finding, and — per file — the lowercased
1100+ // columns it classified. These drive column-aware dedup of the coarse regex
1101+ // PII twins in the merge step below.
1102+ const diffScopedPiiCompletedFiles = new Set < string > ( )
1103+ const diffScopedPiiFindingFiles = new Set < string > ( )
1104+ const classifiedPiiColumnsByFile = new Map < string , Set < string > > ( )
10721105 for ( const ctx of ctxByPath . values ( ) ) {
10731106 const tasks : Promise < Finding [ ] > [ ] = [ ]
10741107 // Engine lanes consume COMPILED SQL (rendered by dbt) when available.
@@ -1098,15 +1131,31 @@ export async function runReview(input: OrchestrateInput): Promise<VerdictEnvelop
10981131 // newly-introduced columns) — exposing PII is a hard-floor concern that
10991132 // shouldn't depend on the cost tier enabling the pii_exposure lane. If core
11001133 // lineage/classification fails, keep the broad deterministic fallback.
1101- const diffScopedPii = await piiClassifyLane ( ctx , input . runner , dialect )
1102- if ( diffScopedPii . completed ) diffScopedPiiFiles . add ( ctx . file . path )
1103- if ( lanes . has ( "pii_exposure" ) ) {
1104- const fallbackPii = diffScopedPii . completed
1105- ? ctx . pii . filter ( ( col ) => ! diffScopedPii . introducedColumns . has ( col . toLowerCase ( ) ) )
1106- : ctx . pii
1107- all . push ( piiLane ( ctx . file , fallbackPii ) )
1108- }
1109- all . push ( diffScopedPii . findings )
1134+ // Run it as a concurrent task (not an inline await) so the heavy native
1135+ // lineage/classify calls overlap the other engine lanes for this file.
1136+ const currentCtx = ctx
1137+ tasks . push (
1138+ ( async ( ) => {
1139+ const diffScopedPii = await piiClassifyLane ( currentCtx , input . runner , dialect )
1140+ if ( diffScopedPii . completed ) {
1141+ diffScopedPiiCompletedFiles . add ( currentCtx . file . path )
1142+ classifiedPiiColumnsByFile . set ( currentCtx . file . path , diffScopedPii . classifiedColumns )
1143+ }
1144+ if ( diffScopedPii . findings . length ) diffScopedPiiFindingFiles . add ( currentCtx . file . path )
1145+ const findings = [ ...diffScopedPii . findings ]
1146+ if ( lanes . has ( "pii_exposure" ) ) {
1147+ // The broad fallback covers output PII columns the precise lane did NOT
1148+ // classify (pre-existing columns, or ones core's classifier missed) —
1149+ // dedup against the columns core actually CONSIDERED, never all
1150+ // introduced columns, so a missed column still surfaces here.
1151+ const fallbackPii = diffScopedPii . completed
1152+ ? currentCtx . pii . filter ( ( col ) => ! diffScopedPii . classifiedColumns . has ( col . toLowerCase ( ) ) )
1153+ : currentCtx . pii
1154+ findings . push ( ...piiLane ( currentCtx . file , fallbackPii ) )
1155+ }
1156+ return findings
1157+ } ) ( ) ,
1158+ )
11101159 // Deterministic dbt anti-pattern detectors run on RAW SQL + diff (need Jinja).
11111160 if ( lanes . has ( "dbt_patterns" ) ) {
11121161 all . push ( detectModelPatterns ( ctx . file , ctx . newSql , input . rubric ) )
@@ -1197,13 +1246,34 @@ export async function runReview(input: OrchestrateInput): Promise<VerdictEnvelop
11971246 const merged = flat . filter ( ( f ) => {
11981247 const tool = f . evidence ?. tool
11991248 const fallbackRule = String ( ( f . evidence ?. result as any ) ?. rule ?? "" ) . replace ( / _ / g, "-" )
1249+ // Dedup the coarse regex PII twins against the precise deterministic PII
1250+ // detectors — but only when a better detector genuinely COVERS the same
1251+ // column. Suppressing whenever the classifier merely "ran" hid real PII
1252+ // (a high-risk column core's classifier missed but the regex would catch,
1253+ // especially on tiers without the broad `pii_exposure` lane).
12001254 if (
12011255 ( tool === "dbt-patterns" || tool === "rule-catalog" ) &&
1202- ( diffScopedPiiFiles . has ( f . file ) || schemaPiiFiles . has ( f . file ) ) &&
12031256 f . category === "pii_exposure" &&
12041257 ( fallbackRule === "pii-into-mart" || fallbackRule === "select-pii-columns" )
12051258 ) {
1206- return false
1259+ // The broad `schema.detect_pii` lane already flagged this file → redundant.
1260+ if ( schemaPiiFiles . has ( f . file ) ) return false
1261+ if ( fallbackRule === "pii-into-mart" ) {
1262+ // `pii_into_mart` carries the matched column in `result.line`: suppress
1263+ // only when core's classifier actually CONSIDERED that column (flagged or
1264+ // deliberately muted). A column core never classified falls through.
1265+ const token = PII_TOKEN_RE . exec ( String ( ( f . evidence ?. result as any ) ?. line ?? "" ) ) ?. [ 1 ] ?. toLowerCase ( )
1266+ if ( token && classifiedPiiColumnsByFile . get ( f . file ) ?. has ( token ) ) return false
1267+ // No recoverable column: defer to the prior file-level behavior so the
1268+ // twin isn't double-reported when core ran for the file.
1269+ if ( ! token && diffScopedPiiCompletedFiles . has ( f . file ) ) return false
1270+ return true
1271+ }
1272+ // `select-pii-columns` doesn't preserve its column. Suppress only when a
1273+ // precise detector actually EMITTED a PII finding for the file (so the
1274+ // exposure is reported by the better source, not silently dropped).
1275+ if ( diffScopedPiiFindingFiles . has ( f . file ) ) return false
1276+ return true
12071277 }
12081278 if ( ! coreRanFiles . has ( f . file ) ) return true
12091279 if ( tool !== "dbt-patterns" && tool !== "rule-catalog" ) return true
0 commit comments