Skip to content

Commit 4d0f19a

Browse files
authored
test(policy): add config coverage report (#87081)
Merged via squash. Prepared head SHA: 6897345 Co-authored-by: giodl73-repo <[email protected]> Co-authored-by: giodl73-repo <[email protected]> Reviewed-by: @giodl73-repo
1 parent 072d3ed commit 4d0f19a

3 files changed

Lines changed: 994 additions & 0 deletions

File tree

package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1695,6 +1695,7 @@
16951695
"plugin-sdk:surface:check": "node --max-old-space-size=8192 scripts/plugin-sdk-surface-report.mjs --check",
16961696
"plugin-sdk:sync-exports": "node scripts/sync-plugin-sdk-exports.mjs",
16971697
"plugin-sdk:usage": "node --max-old-space-size=8192 --import tsx scripts/analyze-plugin-sdk-usage.ts",
1698+
"policy:config-coverage": "node --import tsx scripts/check-policy-config-coverage.ts",
16981699
"plugins:boundary-report": "node --import tsx scripts/plugin-boundary-report.ts",
16991700
"plugins:boundary-report:ci": "node --import tsx scripts/plugin-boundary-report.ts --summary --fail-on-cross-owner --fail-on-unclassified-unused-reserved --fail-on-eligible-compat",
17001701
"plugins:boundary-report:json": "node --import tsx scripts/plugin-boundary-report.ts --json",
Lines changed: 232 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,232 @@
1+
#!/usr/bin/env node
2+
import fs from "node:fs/promises";
3+
import path from "node:path";
4+
import { fileURLToPath } from "node:url";
5+
import JSON5 from "json5";
6+
import {
7+
renderConfigDocBaselineArtifacts,
8+
type ConfigDocBaselineEntry,
9+
} from "../src/config/doc-baseline.js";
10+
11+
type ClassificationStatus = "observed" | "ignored" | "out-of-scope" | "deferred";
12+
13+
type CoverageClassification = {
14+
readonly pattern: string;
15+
readonly status: ClassificationStatus;
16+
readonly area: string;
17+
readonly policy?: string;
18+
readonly reason: string;
19+
readonly allowNoSchemaPath?: boolean;
20+
};
21+
22+
type CoverageConfig = {
23+
readonly monitored: readonly string[];
24+
readonly classifications: readonly CoverageClassification[];
25+
};
26+
27+
type ConfigDocBaseline = {
28+
readonly coreEntries: readonly ConfigDocBaselineEntry[];
29+
readonly channelEntries: readonly ConfigDocBaselineEntry[];
30+
readonly pluginEntries: readonly ConfigDocBaselineEntry[];
31+
};
32+
33+
function flattenConfigDocBaselineEntries(
34+
baseline: ConfigDocBaseline,
35+
): readonly ConfigDocBaselineEntry[] {
36+
return [...baseline.coreEntries, ...baseline.channelEntries, ...baseline.pluginEntries];
37+
}
38+
39+
type ClassifiedEntry = {
40+
readonly path: string;
41+
readonly kind: ConfigDocBaselineEntry["kind"];
42+
readonly classification?: CoverageClassification;
43+
};
44+
45+
type UnmatchedMonitoredPattern = {
46+
readonly pattern: string;
47+
};
48+
49+
const args = new Set(process.argv.slice(2));
50+
const json = args.has("--json");
51+
const check = args.has("--check");
52+
const showCovered = args.has("--show-covered");
53+
54+
if (args.has("--help")) {
55+
console.log(`Usage: pnpm policy:config-coverage [--check] [--json] [--show-covered]
56+
57+
Internal maintainer report for Policy config coverage.
58+
59+
Default mode is report-only and exits 0 even when paths are unclassified.
60+
Use --check when a policy maintainer intentionally wants unclassified or stale
61+
coverage entries to fail locally.`);
62+
process.exit(0);
63+
}
64+
65+
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
66+
const configPath = path.join(repoRoot, "scripts/lib/policy-config-coverage.jsonc");
67+
68+
const config = JSON5.parse(await fs.readFile(configPath, "utf8")) as CoverageConfig;
69+
const { baseline } = await renderConfigDocBaselineArtifacts();
70+
const monitoredEntries = flattenConfigDocBaselineEntries(baseline)
71+
.filter((entry) => !entry.hasChildren)
72+
.filter((entry) => matchesAny(config.monitored, entry.path))
73+
.toSorted((left, right) => left.path.localeCompare(right.path));
74+
const leafEntries = flattenConfigDocBaselineEntries(baseline).filter((entry) => !entry.hasChildren);
75+
const unmatchedMonitored = config.monitored
76+
.filter(
77+
(pattern) =>
78+
!leafEntries.some((entry) => pathMatchesPattern(pattern, entry.path)) &&
79+
!config.classifications.some(
80+
(item) => item.allowNoSchemaPath === true && pathMatchesPattern(item.pattern, pattern),
81+
),
82+
)
83+
.map((pattern) => ({ pattern }))
84+
.toSorted((left, right) => left.pattern.localeCompare(right.pattern));
85+
86+
const classified: ClassifiedEntry[] = monitoredEntries.map((entry) => ({
87+
path: entry.path,
88+
kind: entry.kind,
89+
classification: config.classifications.find((item) =>
90+
pathMatchesPattern(item.pattern, entry.path),
91+
),
92+
}));
93+
const unclassified = classified.filter((entry) => entry.classification === undefined);
94+
const stale = config.classifications.filter(
95+
(item) =>
96+
item.allowNoSchemaPath !== true &&
97+
!monitoredEntries.some((entry) => pathMatchesPattern(item.pattern, entry.path)),
98+
);
99+
const summaryCounts = summarize(classified);
100+
101+
if (json) {
102+
console.log(
103+
JSON.stringify(
104+
{
105+
ok: unclassified.length === 0 && stale.length === 0 && unmatchedMonitored.length === 0,
106+
monitoredPaths: monitoredEntries.length,
107+
counts: summaryCounts,
108+
unclassified,
109+
unmatchedMonitored,
110+
stale,
111+
},
112+
null,
113+
2,
114+
),
115+
);
116+
} else {
117+
printTextReport({
118+
monitoredPaths: monitoredEntries.length,
119+
counts: summaryCounts,
120+
unclassified,
121+
unmatchedMonitored,
122+
stale,
123+
classified,
124+
});
125+
}
126+
127+
if (check && (unclassified.length > 0 || stale.length > 0 || unmatchedMonitored.length > 0)) {
128+
process.exit(1);
129+
}
130+
131+
function printTextReport(input: {
132+
readonly monitoredPaths: number;
133+
readonly counts: Record<string, number>;
134+
readonly unclassified: readonly ClassifiedEntry[];
135+
readonly unmatchedMonitored: readonly UnmatchedMonitoredPattern[];
136+
readonly stale: readonly CoverageClassification[];
137+
readonly classified: readonly ClassifiedEntry[];
138+
}): void {
139+
console.log(`Policy config coverage: ${input.monitoredPaths} monitored config leaf paths`);
140+
for (const [key, count] of Object.entries(input.counts).toSorted(([a], [b]) =>
141+
a.localeCompare(b),
142+
)) {
143+
console.log(` ${key}: ${count}`);
144+
}
145+
146+
if (input.unclassified.length > 0) {
147+
console.log("\nUnclassified config paths:");
148+
for (const entry of input.unclassified) {
149+
console.log(` - ${entry.path} (${entry.kind})`);
150+
}
151+
console.log(
152+
"\nClassify each as observed, ignored, out-of-scope, or deferred in scripts/lib/policy-config-coverage.jsonc.",
153+
);
154+
} else {
155+
console.log("\nNo unclassified monitored config paths.");
156+
}
157+
158+
if (input.unmatchedMonitored.length > 0) {
159+
console.log("\nMonitored patterns with no matching config paths:");
160+
for (const entry of input.unmatchedMonitored) {
161+
console.log(` - ${entry.pattern}`);
162+
}
163+
} else {
164+
console.log("\nNo monitored patterns without matching config paths.");
165+
}
166+
167+
if (input.stale.length > 0) {
168+
console.log("\nStale coverage classifications:");
169+
for (const entry of input.stale) {
170+
console.log(` - ${entry.pattern} (${entry.area}, ${entry.status})`);
171+
}
172+
}
173+
174+
if (showCovered) {
175+
console.log("\nCovered paths:");
176+
for (const entry of input.classified) {
177+
const classification = entry.classification;
178+
console.log(
179+
` - ${entry.path}: ${classification?.area ?? "unclassified"} / ${
180+
classification?.status ?? "unclassified"
181+
}`,
182+
);
183+
}
184+
}
185+
}
186+
187+
function summarize(entries: readonly ClassifiedEntry[]): Record<string, number> {
188+
const counts: Record<string, number> = {};
189+
for (const entry of entries) {
190+
const key =
191+
entry.classification === undefined
192+
? "unclassified"
193+
: `${entry.classification.area}.${entry.classification.status}`;
194+
counts[key] = (counts[key] ?? 0) + 1;
195+
}
196+
return counts;
197+
}
198+
199+
function matchesAny(patterns: readonly string[], value: string): boolean {
200+
return patterns.some((pattern) => pathMatchesPattern(pattern, value));
201+
}
202+
203+
function pathMatchesPattern(pattern: string, value: string): boolean {
204+
const patternParts = pattern.split(".");
205+
const valueParts = value.split(".");
206+
return matchesParts(patternParts, valueParts);
207+
}
208+
209+
function matchesParts(patternParts: readonly string[], valueParts: readonly string[]): boolean {
210+
if (patternParts.length === 0) {
211+
return valueParts.length === 0;
212+
}
213+
const [head, ...tail] = patternParts;
214+
if (head === "**") {
215+
if (tail.length === 0) {
216+
return true;
217+
}
218+
for (let index = 0; index <= valueParts.length; index += 1) {
219+
if (matchesParts(tail, valueParts.slice(index))) {
220+
return true;
221+
}
222+
}
223+
return false;
224+
}
225+
if (valueParts.length === 0) {
226+
return false;
227+
}
228+
if (head !== "*" && head !== valueParts[0]) {
229+
return false;
230+
}
231+
return matchesParts(tail, valueParts.slice(1));
232+
}

0 commit comments

Comments
 (0)