Skip to content

Commit 86741f2

Browse files
committed
feat(scripts): measure placement coupling and reader-view legibility (#2677)
Three report-only module groups, gated by their model tests and nothing else: scripts/repo-history is the rename-resolved history model, scripts/coupling turns it into co-change affinity and per-family modularity, and scripts/legibility asks the typesafe-ai/jev evaluation model to file each module from a reader's evidence, next to majority, neighbour-vote, and k-NN over the same files. Legibility scores the reader view; names withheld is an --ablation reported as the gap to it, never beside the baselines, since the baselines read the physical layout. The own-family audit is a capped scrubber defect under the ablation and an uncapped name echo under the reader condition. ai moves to 7.0.106 in the lockfile: 7.0.68 does not export experimental_evaluate.
1 parent a556e11 commit 86741f2

61 files changed

Lines changed: 4816 additions & 20 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/workflows/ci.yml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,22 @@ jobs:
126126
uses: ./.github/actions/run-gate
127127
with: { gate: depgraph }
128128

129+
# Model tests for the two module-shape reports (#2677) and the rename-resolved history
130+
# model they share. All three run offline on a committed fixture mini-repo plus a live-tree
131+
# smoke that needs the full history this job already fetches; the reports themselves
132+
# (`pnpm coupling`, `pnpm legibility`) stay advisory and ungated.
133+
- name: Check the repo-history model
134+
uses: ./.github/actions/run-gate
135+
with: { gate: repo-history }
136+
137+
- name: Check the coupling report model
138+
uses: ./.github/actions/run-gate
139+
with: { gate: coupling }
140+
141+
- name: Check the legibility report model
142+
uses: ./.github/actions/run-gate
143+
with: { gate: legibility }
144+
129145
# Tests for the TMPDIR redirection itself.
130146
#
131147
# Deliberately NOT in Coverage next to `check:tmpdir-leaks`, where the subject matter

‎docs/agents/testing.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -138,6 +138,9 @@ pnpm depgraph affected src/daemon/ref-frame.ts --json --limit 25
138138
Use its dependent, command, and guarantee-cell report to scope inspection. `pnpm check:affected --run`
139139
selects gates independently from the diff; it does not consume the depgraph report.
140140

141+
`pnpm coupling` reports which families a module co-changes with. `pnpm legibility` scores placement
142+
from a reader's evidence; its `--ablation` pass measures how much of that is carried by names.
143+
141144
## Gate ownership
142145

143146
`CHECK_CATALOG` is the executable check registry. CI owns a check only through the shared `run-gate`

‎package.json‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,11 @@
148148
"check:di-seams": "node --experimental-strip-types scripts/node-test-tmpdir.ts --experimental-strip-types --test scripts/di-seams/*.test.ts && node --experimental-strip-types scripts/di-seams/check.ts",
149149
"depgraph": "node --experimental-strip-types scripts/depgraph/build.ts",
150150
"depgraph:test": "node --experimental-strip-types scripts/node-test-tmpdir.ts --experimental-strip-types --test scripts/depgraph/model.test.ts scripts/depgraph/affected.test.ts",
151+
"repo-history:test": "node --experimental-strip-types scripts/node-test-tmpdir.ts --experimental-strip-types --test scripts/repo-history/*.test.ts",
152+
"coupling": "node --experimental-strip-types scripts/coupling/build.ts",
153+
"coupling:test": "node --experimental-strip-types scripts/node-test-tmpdir.ts --experimental-strip-types --test scripts/coupling/*.test.ts",
154+
"legibility": "node --experimental-strip-types scripts/legibility/run.ts",
155+
"legibility:test": "node --experimental-strip-types scripts/node-test-tmpdir.ts --experimental-strip-types --test scripts/legibility/*.test.ts",
151156
"check:production-exports": "fallow dead-code --config fallow-production-exports.json --production --unused-exports --fail-on-issues",
152157
"check:bundle-owner-files": "node --experimental-strip-types scripts/check-bundle-owner-files.ts",
153158
"check:package": "node --experimental-strip-types scripts/check-package.ts",
@@ -165,7 +170,7 @@
165170
"sync:mcp-metadata": "node scripts/sync-mcp-metadata.mjs",
166171
"check:mcp-metadata": "node scripts/sync-mcp-metadata.mjs --check",
167172
"version": "pnpm sync:mcp-metadata && git add server.json",
168-
"check:tooling": "pnpm format:check && pnpm lint && pnpm typecheck && pnpm check:layering && pnpm depgraph:test && pnpm check:gate-manifest:test && pnpm check:gate-manifest && pnpm check:production-exports && pnpm check:tmpdir-leaks:test && pnpm check:xctest-selection && pnpm check:packaged-runner-swift && pnpm check:mcp-metadata && pnpm build && pnpm check:bundle-owner-files && pnpm check:package",
173+
"check:tooling": "pnpm format:check && pnpm lint && pnpm typecheck && pnpm check:layering && pnpm depgraph:test && pnpm repo-history:test && pnpm coupling:test && pnpm legibility:test && pnpm check:gate-manifest:test && pnpm check:gate-manifest && pnpm check:production-exports && pnpm check:tmpdir-leaks:test && pnpm check:xctest-selection && pnpm check:packaged-runner-swift && pnpm check:mcp-metadata && pnpm build && pnpm check:bundle-owner-files && pnpm check:package",
169174
"check:unit": "pnpm test:unit && pnpm check:tmpdir-leaks && pnpm test:smoke",
170175
"check": "pnpm check:tooling && pnpm check:fallow && pnpm check:unit",
171176
"prepack": "pnpm check:mcp-metadata && pnpm package:npm",

‎pnpm-lock.yaml‎

Lines changed: 19 additions & 19 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎scripts/check-affected/checks.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,9 @@ export const CHECK_CATALOG: readonly CheckSpec[] = [
101101
gate('gate-manifest', 'Gate manifest — every gate owned and wired', 'check:gate-manifest'),
102102
gate('gate-manifest-model', 'Gate manifest model', 'check:gate-manifest:test'),
103103
gate('depgraph', 'Dependency graph report model', 'depgraph:test'),
104+
gate('repo-history', 'Rename-resolved history model', 'repo-history:test'),
105+
gate('coupling', 'Change-coupling report model', 'coupling:test'),
106+
gate('legibility', 'Placement-legibility report model', 'legibility:test'),
104107
gate('tmpdir-leaks', 'Leaked test tmpdir detector', 'check:tmpdir-leaks'),
105108
gate('tmpdir-leaks-model', 'TMPDIR redirection model', 'check:tmpdir-leaks:test'),
106109
gate('coverage-model', 'Changed-line coverage model', 'check:coverage-changed:test'),

‎scripts/check-affected/model.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,9 @@ export type CheckId =
6161
| 'gate-manifest'
6262
| 'gate-manifest-model'
6363
| 'depgraph'
64+
| 'repo-history'
65+
| 'coupling'
66+
| 'legibility'
6467
| 'tmpdir-leaks'
6568
| 'tmpdir-leaks-model'
6669
| 'coverage-model'
@@ -127,6 +130,9 @@ export const ALL_CHECKS: readonly CheckId[] = [
127130
'gate-manifest',
128131
'gate-manifest-model',
129132
'depgraph',
133+
'repo-history',
134+
'coupling',
135+
'legibility',
130136
'tmpdir-leaks',
131137
'tmpdir-leaks-model',
132138
'coverage-model',

‎scripts/coupling/README.md‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
# Change-coupling report
2+
3+
```sh
4+
pnpm coupling # -> .tmp/coupling/report.json + a text summary
5+
pnpm coupling --since-days 30 --out /tmp/coupling.json
6+
pnpm coupling:test
7+
```
8+
9+
Logical coupling from git history alone: do the physical families (workspace packages and
10+
`src/` zones) match how the repository actually changes? There is no renderer; the JSON is the
11+
artifact and the text summary is the headline.
12+
13+
## What it answers
14+
15+
- **Affinity.** A commit touching `k >= 2` production files adds `1 / (k - 1)` to each unordered
16+
pair; `support` counts the commits containing the pair. Commits over 60 files are mass
17+
migrations and are skipped and listed, never folded in. Edges below support 3 are cut.
18+
- **Family modularity.** With `W` the kept weight, `in_f` the weight inside family `f`, `out_f`
19+
the weight leaving it, and `s_f = 2·in_f + out_f`: `intraShare = Σ in_f / W`,
20+
`expected = Σ (s_f / 2W)²`, `Q_f = in_f / W − (s_f / 2W)²`. `Σ Q_f = intraShare − expected`,
21+
asserted in `modularity.test.ts`. `expected` is what a size-matched random rewiring would keep
22+
inside families; the difference is what the families hold beyond their size.
23+
- **Family span.** Distinct families per commit: histogram, median over commits spanning at
24+
least two families, and the share spanning five or more.
25+
- Everything is computed twice, over all history and over the trailing window (`--since-days`,
26+
default 120), so drift is visible in one document.
27+
28+
## What the numbers do not mean
29+
30+
**This is not a removability or correctness claim.** Co-change says two files were edited in the
31+
same commits; it does not say either imports the other, that one could move, or that anything
32+
is wrong. A file with a high hub weight is a place to look, never a work list. In particular:
33+
34+
- `Q_f ≈ 0` with a large `out/in` flow says a family's changes land together with other
35+
families' changes. Whether that is a toolbox (`host-kit`), a vocabulary (`contracts`), or a
36+
wrongly-drawn boundary (`cli-schema`) is a reading the report cannot make for you.
37+
- A high `Q_f` says a family's changes stay inside it. `daemon-server` scores highest partly
38+
because it is the largest family; the `expected` term corrects for size only in aggregate.
39+
- The mass-commit threshold is a declared cut, not a detector. A 59-file refactor still counts.
40+
- The trailing window is only as good as the commits in it; a 30-day window on a quiet month is
41+
a coin toss.
42+
43+
## Reference values (2026-09-19, commit a556e114cd)
44+
45+
| | all-time | last 120 days |
46+
| --- | --- | --- |
47+
| commits used for pairs / skipped (> 60 files) | 1,044 / 34 | 779 / 31 |
48+
| kept edges at support ≥ 3 | 6,314 | 5,297 |
49+
| cross-family share of kept edges | 64.3% | 62.5% |
50+
| intraShare vs expected | 36.5% vs 11.5% | 39.7% vs 11.6% |
51+
| modularity | 0.250 | 0.282 |
52+
| median span of multi-family commits | 4 | 4 |
53+
| multi-family commits spanning ≥ 5 families | 37.6% | 39.6% |
54+
55+
Heaviest hubs by out-family weight: `src/cli-schema/command-schema.ts` (96.5, reaches 22
56+
families), `src/client/client-types.ts` (89.2, 26), `src/cli-schema/cli-help.ts` (85.7, 26),
57+
`src/cli.ts` (72.2, 21). Heaviest family pairs: `commands <-> daemon-server` (60.6),
58+
`cli-schema <-> daemon-server` (57.0), `daemon-server <-> platform-apple` (52.0).
59+
60+
A drift warning is printed, never a failure, when all-time modularity leaves 0.20–0.35, the
61+
cross-family share drops under 60%, or the ≥ 5-family share drops under 25%.
62+
63+
## What is authoritative
64+
65+
`scripts/layering/` is, for the file set and the family partition: families are
66+
`targetDagZone` from `scripts/layering/model.ts` and the file set is `listSourceFiles`, exactly as
67+
`pnpm check:layering` sees them. History comes from `scripts/repo-history/`, the same model the
68+
legibility report reads, so both reports describe one tree. Nothing here gates, ratchets, or
69+
allowlists; the `coupling:test` gate proves the formulas on the committed fixture mini-repo in
70+
`scripts/repo-history/__fixtures__/`, not the live numbers.
71+
72+
## What the JSON carries
73+
74+
- `generated` — commit, date, file and family counts.
75+
- `window.allTime` / `window.since` — commit counts (total, touching, used, skipped, threshold),
76+
the skipped commits, edge counts with the cross-family share, and the span histogram.
77+
- `assortativity` — `intraShare`, `expected`, `modularity` for both windows.
78+
- `perFamily[]` — `family, files, inWeight, outWeight, outInFlow, partners, Q` (all-time).
79+
- `hubs[]` — top files by kept weight with out-family weight and families reached.
80+
- `familyPairs[]` — cross-family weight per unordered family pair.
81+
- `edges[]` — every kept edge, `{ a, b, weight, support }`.

‎scripts/coupling/affinity.test.ts‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
import assert from 'node:assert/strict';
2+
import { test } from 'node:test';
3+
import {
4+
MASS_FILE_COUNT,
5+
MINI_REPO_COMMITS,
6+
MINI_REPO_FILES,
7+
MINI_REPO_PATHS as P,
8+
} from '../repo-history/__fixtures__/mini-repo.ts';
9+
import { buildRepoHistory } from '../repo-history/model.ts';
10+
import { buildAffinity, pairKey, splitPairKey } from './affinity.ts';
11+
12+
const history = buildRepoHistory(MINI_REPO_COMMITS, new Set(MINI_REPO_FILES));
13+
14+
function edge(edges: ReturnType<typeof buildAffinity>['edges'], a: string, b: string) {
15+
return edges.find((entry) => pairKey(entry.a, entry.b) === pairKey(a, b));
16+
}
17+
18+
test('a commit touching k files adds 1/(k-1) to each pair and support counts commits', () => {
19+
const { edges } = buildAffinity(history.commits, { maxFiles: 60, minSupport: 1 });
20+
// c01 (k=3) gives (a, b2) 0.5; c06, c07, c09 (k=2) give 1 each.
21+
assert.deepEqual(edge(edges, P.A, P.B2), { a: P.A, b: P.B2, weight: 3.5, support: 4 });
22+
// (b2, x) only ever co-changes in c01.
23+
assert.deepEqual(edge(edges, P.B2, P.X), { a: P.B2, b: P.X, weight: 0.5, support: 1 });
24+
// c11 (k=2) gives (x, z) 1 and c14 (k=3) gives 0.5.
25+
assert.deepEqual(edge(edges, P.X, P.Z), { a: P.X, b: P.Z, weight: 1.5, support: 2 });
26+
});
27+
28+
test('the support cut keeps exactly the pairs seen in at least minSupport commits', () => {
29+
const result = buildAffinity(history.commits);
30+
assert.deepEqual(
31+
result.edges.map((entry) => [entry.a, entry.b, entry.weight, entry.support]),
32+
[
33+
[P.A, P.B2, 3.5, 4],
34+
[P.A, P.X, 2.5, 3],
35+
[P.X, P.Y, 2.5, 3],
36+
],
37+
);
38+
assert.equal(result.rawEdges, 6);
39+
});
40+
41+
test('a commit over the file threshold is skipped and counted, never folded in', () => {
42+
const result = buildAffinity(history.commits);
43+
assert.deepEqual(result.skipped, [
44+
{ sha: 'c15', subject: 'chore: mass migration', files: MASS_FILE_COUNT },
45+
]);
46+
assert.equal(result.touchingCommits, 16);
47+
assert.equal(result.usedCommits, 11);
48+
assert.equal(
49+
result.edges.some((entry) => entry.a.startsWith('packages/mass/')),
50+
false,
51+
);
52+
const lenient = buildAffinity(history.commits, { maxFiles: 61, minSupport: 1 });
53+
assert.equal(lenient.skipped.length, 0);
54+
assert.equal(
55+
lenient.edges.filter((entry) => entry.a.startsWith('packages/mass/')).length,
56+
(MASS_FILE_COUNT * (MASS_FILE_COUNT - 1)) / 2,
57+
);
58+
});
59+
60+
test('pair keys are order-independent and split back', () => {
61+
assert.equal(pairKey('b', 'a'), pairKey('a', 'b'));
62+
assert.deepEqual(splitPairKey(pairKey('b', 'a')), ['a', 'b']);
63+
});

0 commit comments

Comments
 (0)