Skip to content

Commit 8c3bbdc

Browse files
obs(#1430): add scheduled-lane freshness/cadence health watcher
Discovers schedule: workflows from .github/workflows/, reads recent scheduled runs via the GitHub API, and opens/pings a tracking issue when a lane misses or fails two consecutive cadences. Pure model unit-tested and gated on PRs; API I/O + issue open/ping run nightly. Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
1 parent 9d0a266 commit 8c3bbdc

6 files changed

Lines changed: 610 additions & 3 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,12 @@ jobs:
120120
# rather than in its own job so the two can never be green independently.
121121
run: node --experimental-strip-types --test scripts/depgraph/model.test.ts
122122

123+
- name: Check scheduled-lane health watcher model
124+
# The nightly Scheduled Lane Health watcher (#1430) decides when a lane
125+
# has missed/failed two cadences; gate its pure model on PRs so the
126+
# freshness logic can't silently rot (the watcher itself runs nightly).
127+
run: node --experimental-strip-types --test scripts/scheduled-lane-health/model.test.ts
128+
123129
affected-selector:
124130
name: Affected-check Selector
125131
runs-on: ubuntu-latest
Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
name: Scheduled Lane Health
2+
3+
# Freshness/cadence watcher for scheduled lanes (#1430, umbrella #1412 Track E —
4+
# "the observatory must watch the watchers"). Nightly sweeps (torture, replays,
5+
# perf, conformance) can fail or stop running for weeks while PR CI stays green.
6+
# This job discovers every `schedule:`-triggered workflow from .github/workflows/
7+
# (the list is derived, not hand-maintained), reads each lane's recent scheduled
8+
# runs via the GitHub API, and opens/pings a single tracking issue when a lane
9+
# has missed or failed two consecutive cadences.
10+
11+
on:
12+
schedule:
13+
- cron: '0 8 * * *' # after the nightlies (03:00–06:00 UTC) have run
14+
workflow_dispatch: {}
15+
16+
permissions:
17+
contents: read
18+
actions: read
19+
issues: write
20+
21+
concurrency:
22+
group: ci-${{ github.workflow }}
23+
cancel-in-progress: false
24+
25+
jobs:
26+
health:
27+
name: Scheduled lane freshness watch
28+
runs-on: ubuntu-latest
29+
timeout-minutes: 10
30+
steps:
31+
- name: Checkout
32+
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
33+
34+
- name: Setup toolchain
35+
uses: ./.github/actions/setup-node-pnpm
36+
with:
37+
install-deps: 'true'
38+
39+
- name: Evaluate scheduled-lane freshness
40+
env:
41+
GITHUB_TOKEN: ${{ github.token }}
42+
run: node --experimental-strip-types scripts/scheduled-lane-health/run.ts

‎docs/agents/testing.md‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -246,11 +246,22 @@ on schedule and, per #1430, emits a machine-readable envelope (schema version, c
246246
hash, seed range, duration, result) via `TORTURE_ENVELOPE=<path>`, uploaded as the
247247
`concurrency-torture-envelope` artifact. The envelope is written once, after **all** lane tests
248248
settle, and reports `fail` if any of them (sweep, replay self-check, or forced-contention guardrail)
249-
failed — a later-failing guardrail can never be published as a passing envelope. The #1430 scheduled
250-
health job that watches lane freshness/last-success across workflows is that issue's own deliverable;
251-
this lane is born consumable by it (standard envelope + a `schedule:` trigger the job auto-discovers).
249+
failed — a later-failing guardrail can never be published as a passing envelope.
252250
Optional knobs: `TORTURE_CLIENTS`, `TORTURE_OPS`.
253251

252+
### Scheduled-lane health watcher (#1430)
253+
254+
Scheduled lanes can fail or stop running for weeks while PR CI stays green, so the `Scheduled Lane
255+
Health` workflow (`.github/workflows/scheduled-lane-health.yml`) watches the watchers. It discovers
256+
every `schedule:`-triggered workflow from `.github/workflows/` (the list is **derived**, not
257+
hand-maintained), reads each lane's recent scheduled runs via the GitHub API, and opens/pings a single
258+
tracking issue when a lane has not **succeeded** within two of its own cadences — which covers both a
259+
lane gone dark (no runs) and one failing every cadence. The cadence is estimated per workflow from its
260+
cron expression. All decision logic is pure and unit-tested in `scripts/scheduled-lane-health/model.ts`
261+
(gated on PRs via `scripts/scheduled-lane-health/model.test.ts`); the GitHub API I/O and issue
262+
open/ping live in `run.ts` and run nightly. New scheduled lanes need no wiring here — emitting the
263+
standard envelope and carrying a `schedule:` trigger is enough to be watched.
264+
254265
## Speed rules (experiment-backed, 2026-07-04)
255266

256267
Measured on the full unit suite (340 files, 3,210 tests, 48s wall at ~7x parallelism):
Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
import test from 'node:test';
2+
import assert from 'node:assert/strict';
3+
4+
import {
5+
ALERT_ISSUE_TITLE,
6+
buildAlertBody,
7+
cronCadenceMs,
8+
discoverScheduledLanes,
9+
evaluateLaneHealth,
10+
expandCronField,
11+
parseScheduledLane,
12+
type LaneRun,
13+
} from './model.ts';
14+
15+
const DAY_MS = 24 * 60 * 60 * 1000;
16+
const HOUR_MS = 60 * 60 * 1000;
17+
18+
function isoAgo(now: number, ms: number): string {
19+
return new Date(now - ms).toISOString();
20+
}
21+
22+
test('expandCronField expands wildcards, lists, ranges, and steps', () => {
23+
assert.deepEqual(expandCronField('*', 0, 6), [0, 1, 2, 3, 4, 5, 6]);
24+
assert.deepEqual(expandCronField('1,15', 0, 59), [1, 15]);
25+
assert.deepEqual(expandCronField('0-3', 0, 59), [0, 1, 2, 3]);
26+
assert.deepEqual(expandCronField('*/6', 0, 23), [0, 6, 12, 18]);
27+
// Unknown syntax fails open to the full range (never under-counts fires).
28+
assert.deepEqual(expandCronField('garbage', 0, 2), [0, 1, 2]);
29+
});
30+
31+
test('cronCadenceMs estimates common cadences', () => {
32+
assert.equal(cronCadenceMs('0 5 * * *'), DAY_MS); // daily
33+
assert.equal(cronCadenceMs('0 * * * *'), HOUR_MS); // hourly
34+
// Weekly (Sundays) is ~7 days.
35+
assert.equal(cronCadenceMs('0 5 * * 0'), 7 * DAY_MS);
36+
// Twice daily → ~12h.
37+
assert.equal(cronCadenceMs('0 0,12 * * *'), 12 * HOUR_MS);
38+
});
39+
40+
test('parseScheduledLane picks up schedule-triggered workflows and skips others', () => {
41+
const scheduled = parseScheduledLane(
42+
'nightly.yml',
43+
['name: Nightly Sweep', 'on:', ' schedule:', " - cron: '0 5 * * *'", 'jobs: {}'].join('\n'),
44+
);
45+
assert.ok(scheduled);
46+
assert.equal(scheduled.name, 'Nightly Sweep');
47+
assert.equal(scheduled.cadenceMs, DAY_MS);
48+
49+
const pushOnly = parseScheduledLane('ci.yml', ['name: CI', 'on:', ' push: {}'].join('\n'));
50+
assert.equal(pushOnly, undefined);
51+
});
52+
53+
test('discoverScheduledLanes excludes the watcher itself', () => {
54+
const files = [
55+
{ file: 'a.yml', content: ['on:', ' schedule:', " - cron: '0 5 * * *'"].join('\n') },
56+
{ file: 'scheduled-lane-health.yml', content: ['on:', ' schedule:', " - cron: '0 8 * * *'"].join('\n') },
57+
{ file: 'ci.yml', content: ['on:', ' push: {}'].join('\n') },
58+
];
59+
const lanes = discoverScheduledLanes(files, 'scheduled-lane-health.yml');
60+
assert.deepEqual(
61+
lanes.map((l) => l.file),
62+
['a.yml'],
63+
);
64+
});
65+
66+
test('evaluateLaneHealth: a fresh lane is healthy', () => {
67+
const now = Date.now();
68+
const lane = { file: 'nightly.yml', name: 'Nightly', cadenceMs: DAY_MS };
69+
const runs: LaneRun[] = [{ conclusion: 'success', createdAt: isoAgo(now, 3 * HOUR_MS) }];
70+
assert.equal(evaluateLaneHealth({ lane, runs, now }).healthy, true);
71+
});
72+
73+
test('evaluateLaneHealth: a lane gone dark (no runs for two cadences) is unhealthy', () => {
74+
const now = Date.now();
75+
const lane = { file: 'nightly.yml', name: 'Nightly', cadenceMs: DAY_MS };
76+
// Last success was 3 days ago and nothing since — two cadences missed.
77+
const runs: LaneRun[] = [{ conclusion: 'success', createdAt: isoAgo(now, 3 * DAY_MS) }];
78+
const health = evaluateLaneHealth({ lane, runs, now });
79+
assert.equal(health.healthy, false);
80+
assert.match(health.reason, /two/);
81+
});
82+
83+
test('evaluateLaneHealth: a lane with no successful run ever is unhealthy', () => {
84+
const now = Date.now();
85+
const lane = { file: 'nightly.yml', name: 'Nightly', cadenceMs: DAY_MS };
86+
const runs: LaneRun[] = [
87+
{ conclusion: 'failure', createdAt: isoAgo(now, HOUR_MS) },
88+
{ conclusion: 'failure', createdAt: isoAgo(now, DAY_MS) },
89+
];
90+
const health = evaluateLaneHealth({ lane, runs, now });
91+
assert.equal(health.healthy, false);
92+
assert.match(health.reason, /failing every cadence/);
93+
});
94+
95+
test('buildAlertBody summarizes only unhealthy lanes, or nothing when all healthy', () => {
96+
const now = Date.now();
97+
const healths = [
98+
{ file: 'a.yml', name: 'A', healthy: true, reason: 'fresh' },
99+
{ file: 'b.yml', name: 'B', healthy: false, reason: 'went dark' },
100+
];
101+
const body = buildAlertBody({ healths, now, runUrl: 'https://example/run/1' });
102+
assert.ok(body);
103+
assert.match(body, /b\.yml/);
104+
assert.doesNotMatch(body, /a\.yml/);
105+
assert.match(body, /https:\/\/example\/run\/1/);
106+
107+
const healthy = buildAlertBody({
108+
healths: [{ file: 'a.yml', name: 'A', healthy: true, reason: 'fresh' }],
109+
now,
110+
});
111+
assert.equal(healthy, undefined);
112+
});
113+
114+
test('ALERT_ISSUE_TITLE is stable so the watcher pings one issue', () => {
115+
assert.equal(typeof ALERT_ISSUE_TITLE, 'string');
116+
assert.ok(ALERT_ISSUE_TITLE.length > 0);
117+
});

0 commit comments

Comments
 (0)