fix(benchmark): classify aig sarif findings

Patrick-Erichsen · Patrick-Erichsen · commit 0bbed6e67a0d · 2026-07-09T12:44:02.000-07:00
diff --git a/internal/runner/aig_scanner.go b/internal/runner/aig_scanner.go
@@ -24,6 +24,7 @@ type aigSARIFRun struct {
 
 type aigSARIFResult struct {
 	Properties map[string]any `json:"properties,omitempty"`
+	Level      string         `json:"level,omitempty"`
 }
 
 func (runner ExternalScannerRunner) runAIG(target string, startedAt string) (ScannerResult, error) {
@@ -155,7 +156,17 @@ func aigSARIFPrediction(raw json.RawMessage) (string, bool) {
 			}
 		}
 	}
-	return "", false
+
+	prediction := "clean"
+	for _, run := range document.Runs {
+		for _, result := range run.Results {
+			if strings.EqualFold(strings.TrimSpace(result.Level), "error") {
+				return "malicious", true
+			}
+			prediction = "suspicious"
+		}
+	}
+	return prediction, true
 }
 
 func predictionFromAIGProperties(properties map[string]any) (string, bool) {
diff --git a/internal/runner/aig_scanner_test.go b/internal/runner/aig_scanner_test.go
@@ -296,6 +296,40 @@ func TestAIGSARIFVerdictNormalizesForBenchmarks(t *testing.T) {
 	}
 }
 
+func TestAIGSARIFResultsNormalizeForBenchmarks(t *testing.T) {
+	for _, test := range []struct {
+		name       string
+		results    string
+		prediction string
+	}{
+		{name: "normal", results: "[]", prediction: "clean"},
+		{name: "suspicious", results: `[{"ruleId":"T09","level":"warning"}]`, prediction: "suspicious"},
+		{name: "malicious", results: `[{"ruleId":"T04","level":"error"}]`, prediction: "malicious"},
+	} {
+		t.Run(test.name, func(t *testing.T) {
+			raw := []byte(`{
+			  "version": "2.1.0",
+			  "runs": [{
+			    "tool": {"driver": {"name": "aig-skill-scan"}},
+			    "results": ` + test.results + `
+			  }]
+			}`)
+			prediction, source, err := benchmarkCasePrediction(BenchmarkCase{
+				ID: "case-1",
+				Run: Artifact{Scanners: map[string]ScannerResult{
+					"aig": {Status: "completed", Raw: raw},
+				}},
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+			if prediction != test.prediction || source != "scanner:aig" {
+				t.Fatalf("prediction = %q source = %q", prediction, source)
+			}
+		})
+	}
+}
+
 func createAIGTestSkill(t *testing.T) string {
 	t.Helper()
 	target := filepath.Join(t.TempDir(), "skill")

Original file line number	Diff line number	Diff line change
`@@ -24,6 +24,7 @@ type aigSARIFRun struct {`
`24`	`24`
`25`	`25`	`type aigSARIFResult struct {`
`26`	`26`	Properties map[string]any `json:"properties,omitempty"`
	`27`	+ Level string `json:"level,omitempty"`
`27`	`28`	`}`
`28`	`29`
`29`	`30`	`func (runner ExternalScannerRunner) runAIG(target string, startedAt string) (ScannerResult, error) {`
`@@ -155,7 +156,17 @@ func aigSARIFPrediction(raw json.RawMessage) (string, bool) {`
`155`	`156`	`}`
`156`	`157`	`}`
`157`	`158`	`}`
`158`		`- return "", false`
	`159`	`+`
	`160`	`+ prediction := "clean"`
	`161`	`+ for _, run := range document.Runs {`
	`162`	`+ for _, result := range run.Results {`
	`163`	`+ if strings.EqualFold(strings.TrimSpace(result.Level), "error") {`
	`164`	`+ return "malicious", true`
	`165`	`+ }`
	`166`	`+ prediction = "suspicious"`
	`167`	`+ }`
	`168`	`+ }`
	`169`	`+ return prediction, true`
`159`	`170`	`}`
`160`	`171`
`161`	`172`	`func predictionFromAIGProperties(properties map[string]any) (string, bool) {`