Skip to content

Commit dd700b6

Browse files
test(fuzz): envelope for malformed options; add scheduled-lane health consumer (#1414, #1430)
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
1 parent 0f4d99a commit dd700b6

12 files changed

Lines changed: 629 additions & 5 deletions

File tree

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
name: Scheduled Lane Health
2+
3+
# Watches the watchers (#1430). Every schedule-triggered workflow in .github/workflows/ is
4+
# classified from its recorded scheduled runs — healthy, failing (two consecutive failures), or
5+
# dark (no run within two cadences) — because a lane that silently stops running looks exactly like
6+
# a lane that never fails. Freshness fields land in lane-health.json for the repo-health snapshot,
7+
# and an unhealthy lane opens or pings a single tracking issue.
8+
9+
on:
10+
schedule:
11+
- cron: "0 6 * * *"
12+
workflow_dispatch:
13+
inputs:
14+
dry-run:
15+
description: "Report only; do not open or ping the alert issue"
16+
required: false
17+
default: "false"
18+
19+
permissions:
20+
contents: read
21+
issues: write
22+
23+
concurrency:
24+
group: ci-${{ github.workflow }}-${{ github.ref }}
25+
cancel-in-progress: true
26+
27+
jobs:
28+
lane-health:
29+
name: Scheduled Lane Health
30+
runs-on: ubuntu-latest
31+
timeout-minutes: 10
32+
steps:
33+
- name: Checkout
34+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
35+
36+
- name: Setup toolchain
37+
uses: ./.github/actions/setup-node-pnpm
38+
39+
- name: Classify scheduled lanes
40+
env:
41+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
42+
DRY_RUN: ${{ github.event.inputs.dry-run == 'true' && '--dry-run' || '' }}
43+
run: pnpm lanes:health --artifact-dir .tmp/lane-health $DRY_RUN
44+
45+
- name: Upload lane health snapshot
46+
if: always()
47+
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
48+
with:
49+
name: lane-health-${{ github.run_id }}-${{ github.run_attempt }}
50+
path: .tmp/lane-health
51+
if-no-files-found: warn

‎docs/agents/testing.md‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -128,6 +128,18 @@ to separate artifact subdirectories and both run unconditionally, so freshness/h
128128
(#1430) always finds an envelope; the step summary prints each one it finds and never fails on a
129129
missing file. Other scheduled lanes adopt the same writer rather than defining a second shape.
130130

131+
### Scheduled lane health
132+
133+
`pnpm lanes:health` is the consumer side of the envelope contract (#1430): it derives the lane list
134+
from `schedule:`-triggered workflows in `.github/workflows/` (never a hand-maintained list), reads
135+
each lane's recent scheduled runs from the GitHub API, and classifies it from recorded run fields —
136+
`healthy`, `failing` (two consecutive failed cadences), `dark` (no run within two cadences), or
137+
`pending` (no history to judge, e.g. a lane not on the default branch yet). Freshness fields land in
138+
`lane-health.json` for the repo-health snapshot; an unhealthy lane opens or pings one tracking issue.
139+
It runs as `Scheduled Lane Health` (`.github/workflows/scheduled-lane-health.yml`) and locally with
140+
`--dry-run`, which never touches issues. Classification lives in
141+
`scripts/scheduled-lane/health-model.ts` and is unit-tested per category.
142+
131143
A nightly discovery reaches the unit lane by promotion, not hand-editing: the printed
132144
`promote:` command re-runs the downloaded artifact and appends it to
133145
`scripts/fuzz/corpus/regressions.json`, which `scripts/fuzz/corpus-replay.test.ts` replays on every

‎package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,7 @@
112112
"format": "node ./node_modules/oxfmt/bin/oxfmt --write src test skills scripts/fuzz scripts/scheduled-lane scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/help-conformance-cases.mjs scripts/help-conformance-sample-outputs.mjs scripts/__tests__/help-conformance-bench.test.ts scripts/__tests__/help-conformance-sample-outputs.test.ts scripts/__tests__/help-conformance-topic-coverage.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'",
113113
"format:check": "node ./node_modules/oxfmt/bin/oxfmt --check src test skills scripts/fuzz scripts/scheduled-lane scripts/help-conformance-bench.mjs scripts/help-conformance-case-checks.mjs scripts/help-conformance-command-validator.ts scripts/help-conformance-expectations.mjs scripts/help-conformance-plan-validator.mjs scripts/help-conformance-runner-output.mjs scripts/help-conformance-summary.mjs scripts/help-conformance-cases.mjs scripts/help-conformance-sample-outputs.mjs scripts/__tests__/help-conformance-bench.test.ts scripts/__tests__/help-conformance-sample-outputs.test.ts scripts/__tests__/help-conformance-topic-coverage.test.ts package.json tsconfig.json tsconfig.lib.json tsdown.config.ts vitest.config.ts .github/actions/setup-node-pnpm/action.yml .oxlintrc.json .oxfmtrc.json '!test/skillgym/.skillgym-results/**'",
114114
"fuzz:parsers": "node --experimental-strip-types scripts/fuzz/run.ts",
115+
"lanes:health": "node --experimental-strip-types scripts/scheduled-lane/health.ts",
115116
"fallow": "fallow audit --base origin/main",
116117
"fallow:all": "fallow --summary",
117118
"fallow:baseline": "(fallow dead-code --save-baseline fallow-baselines/dead-code.json --summary || true) && (fallow health --save-baseline fallow-baselines/health.json --summary || true)",

‎scripts/fuzz/harness.test.ts‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,21 @@ describe('run envelope', () => {
123123
expect(envelope.details.reproCommands[0]).toContain('--input-file');
124124
});
125125

126+
// A malformed workflow-dispatch input used to throw out of option parsing before anything
127+
// could write an envelope, which reads to monitoring exactly like a lane that went dark.
128+
it('is written when the options themselves are malformed', () => {
129+
const { envelope, status } = envelopeFrom(['--iterations', 'lots']);
130+
expect(status).toBe(1);
131+
expect(envelope.result).toBe('error');
132+
expect(envelope.details.targetRuns).toEqual([]);
133+
});
134+
135+
it('is written for an unknown flag', () => {
136+
const { envelope, status } = envelopeFrom(['--not-a-flag']);
137+
expect(status).toBe(1);
138+
expect(envelope.result).toBe('error');
139+
});
140+
126141
it('is written for a self-check run', () => {
127142
const { envelope, status } = envelopeFrom(['--self-check', '--case-timeout-ms', '750']);
128143
expect(status).toBe(0);

‎scripts/fuzz/options.ts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,25 @@ function optional<K extends string>(key: K, value: string | undefined): Record<K
4747
return value === undefined ? {} : ({ [key]: value } as Record<K, string>);
4848
}
4949

50+
/**
51+
* Options to fall back on when argv itself is unusable, so a malformed dispatch input still
52+
* produces an envelope. `--artifact-dir` is recovered positionally: the value that decides *where*
53+
* monitoring looks must survive a rejected flag elsewhere in argv.
54+
*/
55+
export function fallbackFuzzOptions(argv: readonly string[]): FuzzOptions {
56+
const flag = argv.indexOf('--artifact-dir');
57+
const dir = flag === -1 ? undefined : argv[flag + 1];
58+
return {
59+
iterations: Number(DEFAULTS.iterations),
60+
seed: Number(DEFAULTS.seed),
61+
caseTimeoutMs: Number(DEFAULTS.caseTimeoutMs),
62+
artifactDir: dir !== undefined && !dir.startsWith('--') ? dir : DEFAULTS.artifactDir,
63+
appendCorpus: false,
64+
replayCorpus: false,
65+
selfCheck: argv.includes('--self-check'),
66+
};
67+
}
68+
5069
/** Parses argv into options; `null` means usage was requested and nothing should run. */
5170
export function readFuzzOptions(argv: readonly string[]): FuzzOptions | null {
5271
const { values } = parseArgs({

‎scripts/fuzz/run.ts‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ import {
1616
import { runTarget } from './execute.ts';
1717
import { describeFailure, type FuzzFailure } from './invariant.ts';
1818
import { generateCases } from './mutate.ts';
19-
import { FUZZ_USAGE, readFuzzOptions, type FuzzOptions } from './options.ts';
19+
import { fallbackFuzzOptions, FUZZ_USAGE, readFuzzOptions, type FuzzOptions } from './options.ts';
2020
import { getFuzzTarget } from './registry.ts';
2121
import { promoteFailures, reportFailures } from './report.ts';
2222
import { readSavedCase } from './saved-case.ts';
@@ -172,8 +172,7 @@ function reportCrash(options: FuzzOptions, startedAt: number, error: unknown): n
172172
return 1;
173173
}
174174

175-
async function run(options: FuzzOptions): Promise<number> {
176-
const startedAt = Date.now();
175+
async function run(options: FuzzOptions, startedAt: number): Promise<number> {
177176
try {
178177
const outcome = await runMode(options);
179178
const result = outcome.failed ? 'fail' : 'pass';
@@ -187,12 +186,21 @@ async function run(options: FuzzOptions): Promise<number> {
187186
}
188187

189188
async function main(): Promise<number> {
190-
const options = readFuzzOptions(process.argv.slice(2));
189+
const startedAt = Date.now();
190+
const argv = process.argv.slice(2);
191+
// Option parsing is inside the guarded path on purpose: a malformed workflow-dispatch input is
192+
// exactly the kind of terminal failure monitoring must still see an envelope for.
193+
let options: FuzzOptions | null;
194+
try {
195+
options = readFuzzOptions(argv);
196+
} catch (error) {
197+
return reportCrash(fallbackFuzzOptions(argv), startedAt, error);
198+
}
191199
if (!options) {
192200
process.stdout.write(FUZZ_USAGE);
193201
return 0;
194202
}
195-
return await run(options);
203+
return await run(options, startedAt);
196204
}
197205

198206
process.exitCode = await main();

‎scripts/scheduled-lane/discover.ts‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
// Which workflows are scheduled lanes (#1430).
2+
//
3+
// Derived from .github/workflows/ rather than a hand-maintained list: a lane added without being
4+
// registered anywhere is the failure mode the health job exists to prevent.
5+
6+
import fs from 'node:fs';
7+
import path from 'node:path';
8+
import { parse } from 'yaml';
9+
import { cadenceHours, type LaneCadence } from './health-model.ts';
10+
11+
type WorkflowDocument = {
12+
name?: string;
13+
on?: { schedule?: { cron?: string }[] } | string | string[];
14+
};
15+
16+
function cronExpressionsOf(document: WorkflowDocument): string[] {
17+
const on = document.on;
18+
if (typeof on !== 'object' || Array.isArray(on) || on === null) return [];
19+
return (on.schedule ?? []).flatMap((entry) => (entry.cron === undefined ? [] : [entry.cron]));
20+
}
21+
22+
export function discoverScheduledLanes(workflowDir: string): LaneCadence[] {
23+
const lanes: LaneCadence[] = [];
24+
for (const file of fs.readdirSync(workflowDir).sort()) {
25+
if (!file.endsWith('.yml') && !file.endsWith('.yaml')) continue;
26+
const document = parse(
27+
fs.readFileSync(path.join(workflowDir, file), 'utf8'),
28+
) as WorkflowDocument | null;
29+
const cronExpressions = cronExpressionsOf(document ?? {});
30+
if (cronExpressions.length === 0) continue;
31+
lanes.push({
32+
workflow: file,
33+
name: document?.name ?? file,
34+
cronExpressions,
35+
cadenceHours: cadenceHours(cronExpressions),
36+
});
37+
}
38+
return lanes;
39+
}
Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
// GitHub API access for the scheduled-lane health job (#1430).
2+
//
3+
// Only the two calls the job needs: recent `schedule`-triggered runs of a workflow, and upsert of
4+
// the single alert issue. Kept behind one module so the health logic stays pure and testable.
5+
6+
import type { LaneHistory, LaneRun } from './health-model.ts';
7+
8+
const API = process.env.GITHUB_API_URL ?? 'https://api.github.com';
9+
10+
type RunsResponse = {
11+
workflow_runs?: { conclusion: string | null; created_at: string; html_url: string }[];
12+
};
13+
14+
type IssuesResponse = { number: number; title: string }[];
15+
16+
function headers(token: string): Record<string, string> {
17+
return {
18+
accept: 'application/vnd.github+json',
19+
authorization: `Bearer ${token}`,
20+
'x-github-api-version': '2022-11-28',
21+
};
22+
}
23+
24+
/** Plain field, not a constructor parameter property: Node's type stripping rejects those. */
25+
class HttpError extends Error {
26+
status: number;
27+
28+
constructor(status: number, message: string) {
29+
super(message);
30+
this.status = status;
31+
}
32+
}
33+
34+
async function request<T>(token: string, url: string, init?: RequestInit): Promise<T> {
35+
const response = await fetch(url, { ...init, headers: headers(token) });
36+
if (!response.ok) {
37+
const method = init?.method ?? 'GET';
38+
throw new HttpError(response.status, `${method} ${url} failed: ${response.status}`);
39+
}
40+
return (await response.json()) as T;
41+
}
42+
43+
/** `null` for 404 only; every other status stays an error. */
44+
async function requestOrNullOn404<T>(token: string, url: string): Promise<T | null> {
45+
try {
46+
return await request<T>(token, url);
47+
} catch (error) {
48+
if (error instanceof HttpError && error.status === 404) return null;
49+
throw error;
50+
}
51+
}
52+
53+
function toLaneRun(run: { conclusion: string | null; created_at: string; html_url: string }) {
54+
return {
55+
conclusion: (run.conclusion ?? null) as LaneRun['conclusion'],
56+
createdAt: run.created_at,
57+
url: run.html_url,
58+
};
59+
}
60+
61+
/**
62+
* Newest-first scheduled runs. A 404 means the workflow is not on the default branch yet (a lane
63+
* being born, as in this PR), which is unknown history rather than a dark lane.
64+
*/
65+
export async function fetchScheduledRuns(input: {
66+
token: string;
67+
repo: string;
68+
workflow: string;
69+
perPage: number;
70+
}): Promise<LaneHistory> {
71+
const url = `${API}/repos/${input.repo}/actions/workflows/${input.workflow}/runs?event=schedule&per_page=${input.perPage}`;
72+
const body = await requestOrNullOn404<RunsResponse>(input.token, url);
73+
if (body === null) {
74+
return { known: false, reason: 'workflow not on the default branch yet', runs: [] };
75+
}
76+
return { known: true, runs: (body.workflow_runs ?? []).map(toLaneRun) };
77+
}
78+
79+
/** Creates the alert issue, or comments on it when it is already open. */
80+
export async function upsertAlertIssue(input: {
81+
token: string;
82+
repo: string;
83+
title: string;
84+
body: string;
85+
}): Promise<{ action: 'created' | 'commented'; number: number }> {
86+
const open = await request<IssuesResponse>(
87+
input.token,
88+
`${API}/repos/${input.repo}/issues?state=open&per_page=100`,
89+
);
90+
const existing = open.find((issue) => issue.title === input.title);
91+
if (!existing) {
92+
const created = await request<{ number: number }>(
93+
input.token,
94+
`${API}/repos/${input.repo}/issues`,
95+
{ method: 'POST', body: JSON.stringify({ title: input.title, body: input.body }) },
96+
);
97+
return { action: 'created', number: created.number };
98+
}
99+
await request(input.token, `${API}/repos/${input.repo}/issues/${existing.number}/comments`, {
100+
method: 'POST',
101+
body: JSON.stringify({ body: input.body }),
102+
});
103+
return { action: 'commented', number: existing.number };
104+
}

0 commit comments

Comments
 (0)