Skip to content

Commit 76e54f7

Browse files
halindromeclaude
andcommitted
fix(watcher): back off re-forking a hard-failing index worker
A hard index failure (index_fn < 0) left the retry cadence untouched, so a persistently failing project re-forked an index worker every poll for as long as the daemon lived. Observed on 0.10.8/macOS: 2233 consecutive failed workers over 4h43m, ~8/min, every one dying at the same worker-side coordination seal. Read queries stayed healthy throughout, so nothing surfaced to the user while no project could be indexed at all. #937 deliberately leaves the baseline uncommitted on a failed reindex so the change is retried rather than lost. That guarantee is kept; what changes is the cadence. Each consecutive hard failure doubles the delay, capped at five minutes, taking a permanently failing project from ~480 attempts/hour to ~12 while still recovering on its own once the cause clears. Busy-skip (rc > 0) and success paths are untouched. The delay arithmetic is a pure exported helper so it is unit-testable without a clock. watcher.index.err now carries rc and the consecutive- failure count, and a distinct watcher.index.sustained_failure names a project whose failures are clearly not transient. Note: itoa_buf returns a single shared per-thread buffer, so the two-value log line uses local buffers rather than two itoa_buf calls. Refs #2015 Co-Authored-By: Claude Opus 5 (1M context) <[email protected]> Claude-Session: https://claude.ai/code/session_01FbrM52pmB1m2SR1vRRoNnf Signed-off-by: Shane McCarron <[email protected]>
1 parent 4d8ac9e commit 76e54f7

3 files changed

Lines changed: 115 additions & 2 deletions

File tree

‎src/watcher/watcher.c‎

Lines changed: 51 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,14 @@ typedef struct {
7575
uint64_t last_dirty_sig; /* committed dirty-state signature */
7676
uint64_t pending_dirty_sig; /* observed at check time */
7777
char pending_head[CBM_SZ_128]; /* HEAD observed at check time */
78+
/* Consecutive hard index failures (index_fn < 0). A hard error is
79+
* usually persistent — a poisoned coordination endpoint, an unreadable
80+
* DB — so retrying it at the plain poll interval re-forks a worker that
81+
* fails identically, for as long as the daemon lives. #937 deliberately
82+
* leaves the baseline uncommitted so the change is never lost; this
83+
* decays the retry cadence so "never lost" does not also mean "retried
84+
* forever". Reset to 0 by any successful reindex. */
85+
int index_failures;
7886
/* Hop from root_path up to the repository root ("" when they are the same),
7987
* from `rev-parse --show-cdup`. Porcelain paths are repository-relative, so
8088
* the signature needs this to stat them. Resolved once at baseline. */
@@ -114,6 +122,16 @@ struct cbm_watcher {
114122
#define POLL_FILE_STEP 500 /* add 1s per this many files */
115123
#define POLL_MAX_MS 60000
116124

125+
/* Hard index-failure backoff. Doubling per consecutive failure, capped, so a
126+
* persistently failing project costs ~12 attempts/hour instead of ~480 while
127+
* still recovering on its own within the ceiling once the cause clears. The
128+
* shift cap keeps the intermediate value well inside int64 for any interval. */
129+
#define INDEX_FAIL_SHIFT_MAX 6
130+
#define INDEX_FAIL_CEILING_MS 300000 /* 5 min */
131+
/* Log a distinct line once the failures are clearly not transient, so the
132+
* daemon log names the stuck project instead of only repeating the warning. */
133+
#define INDEX_FAIL_SUSTAINED 10
134+
117135
/* Stale-root pruning (#286): a watched project whose root directory stays
118136
* missing is pruned — its cached DB is deleted and the watch entry removed.
119137
* Deletion is destructive (the DB can hold user-authored data such as the
@@ -143,6 +161,19 @@ static int64_t now_ns(void) {
143161

144162
/* ── Adaptive interval ──────────────────────────────────────────── */
145163

164+
int cbm_watcher_index_backoff_ms(int interval_ms, int consecutive_failures) {
165+
if (interval_ms < 0) {
166+
interval_ms = 0;
167+
}
168+
if (consecutive_failures <= 0) {
169+
return interval_ms;
170+
}
171+
int shift =
172+
consecutive_failures < INDEX_FAIL_SHIFT_MAX ? consecutive_failures : INDEX_FAIL_SHIFT_MAX;
173+
int64_t delay = (int64_t)interval_ms << shift;
174+
return delay > INDEX_FAIL_CEILING_MS ? INDEX_FAIL_CEILING_MS : (int)delay;
175+
}
176+
146177
int cbm_watcher_poll_interval_ms(int file_count) {
147178
int ms = POLL_BASE_MS + ((file_count / POLL_FILE_STEP) * CBM_MSEC_PER_SEC);
148179
if (ms > POLL_MAX_MS) {
@@ -1429,6 +1460,7 @@ static void poll_project(const char *key, void *val, void *ud) {
14291460
int rc = ctx->w->index_fn(s->project_name, s->root_path, ctx->w->user_data);
14301461
if (rc == 0) {
14311462
ctx->reindexed++;
1463+
s->index_failures = 0;
14321464
/* Commit the baselines OBSERVED AT CHECK TIME — the state whose
14331465
* reindex just succeeded. A commit/edit landing during the
14341466
* reindex is deliberately not absorbed: the next poll sees it
@@ -1447,11 +1479,28 @@ static void poll_project(const char *key, void *val, void *ud) {
14471479
/* Busy-skip: baseline stays uncommitted, next poll retries. */
14481480
cbm_log_info("watcher.index.retry", "project", s->project_name);
14491481
} else {
1450-
cbm_log_warn("watcher.index.err", "project", s->project_name);
1482+
/* itoa_buf returns one shared per-thread buffer, so two of them
1483+
* in one call would print the same value twice. */
1484+
char rc_text[CBM_SZ_32];
1485+
char streak_text[CBM_SZ_32];
1486+
if (s->index_failures < INT_MAX) {
1487+
s->index_failures++;
1488+
}
1489+
snprintf(rc_text, sizeof(rc_text), "%d", rc);
1490+
snprintf(streak_text, sizeof(streak_text), "%d", s->index_failures);
1491+
cbm_log_warn("watcher.index.err", "project", s->project_name, "rc", rc_text,
1492+
"consecutive", streak_text);
1493+
if (s->index_failures == INDEX_FAIL_SUSTAINED) {
1494+
cbm_log_warn("watcher.index.sustained_failure", "project", s->project_name,
1495+
"consecutive", streak_text);
1496+
}
14511497
}
14521498
}
14531499

1454-
s->next_poll_ns = ctx->now + ((int64_t)s->interval_ms * US_PER_MS);
1500+
/* Failures back off; success and busy-skip keep the adaptive cadence. */
1501+
s->next_poll_ns =
1502+
ctx->now +
1503+
((int64_t)cbm_watcher_index_backoff_ms(s->interval_ms, s->index_failures) * US_PER_MS);
14551504
}
14561505

14571506
/* Callback to snapshot project state pointers into an array. */

‎src/watcher/watcher.h‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,6 +94,13 @@ int cbm_watcher_watch_count(cbm_watcher_t *w);
9494
/* Return the adaptive poll interval (ms) for a given file count. */
9595
int cbm_watcher_poll_interval_ms(int file_count);
9696

97+
/* Return the delay (ms) before the next index attempt for a project with
98+
* `consecutive_failures` consecutive hard index failures. Zero failures
99+
* yields `interval_ms` unchanged; each further failure doubles the delay up
100+
* to a fixed ceiling, so a permanently failing project stops re-forking a
101+
* worker at the poll cadence without ever being abandoned. */
102+
int cbm_watcher_index_backoff_ms(int interval_ms, int consecutive_failures);
103+
97104
/* Classify a stat() errno observed on a watched project root: returns true
98105
* only for values that mean the root itself is gone (ENOENT, ENOTDIR) and
99106
* may count toward stale-root pruning (#286). Any other failure (EACCES,

‎tests/test_watcher.c‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
#include <watcher/watcher.h>
1515
#include <store/store.h>
1616
#include <errno.h>
17+
#include <limits.h>
1718
#include <stdatomic.h>
1819
#include <signal.h>
1920
#include <string.h>
@@ -85,6 +86,57 @@ TEST(poll_interval_small) {
8586
PASS();
8687
}
8788

89+
/* ══════════════════════════════════════════════════════════════════
90+
* HARD INDEX-FAILURE BACKOFF
91+
* ══════════════════════════════════════════════════════════════════ */
92+
93+
TEST(index_backoff_no_failures_keeps_interval) {
94+
/* The success and busy-skip paths must be completely unaffected. */
95+
ASSERT_EQ(cbm_watcher_index_backoff_ms(5000, 0), 5000);
96+
ASSERT_EQ(cbm_watcher_index_backoff_ms(60000, 0), 60000);
97+
PASS();
98+
}
99+
100+
TEST(index_backoff_doubles_per_failure) {
101+
ASSERT_EQ(cbm_watcher_index_backoff_ms(5000, 1), 10000);
102+
ASSERT_EQ(cbm_watcher_index_backoff_ms(5000, 2), 20000);
103+
ASSERT_EQ(cbm_watcher_index_backoff_ms(5000, 3), 40000);
104+
ASSERT_EQ(cbm_watcher_index_backoff_ms(5000, 4), 80000);
105+
PASS();
106+
}
107+
108+
TEST(index_backoff_reaches_ceiling_and_stays) {
109+
/* 5000 << 6 = 320000, above the 5-minute ceiling. The shift is capped
110+
* too, so an arbitrarily long failure streak cannot overflow the
111+
* intermediate value or wrap back to a short delay. */
112+
ASSERT_EQ(cbm_watcher_index_backoff_ms(5000, 6), 300000);
113+
ASSERT_EQ(cbm_watcher_index_backoff_ms(5000, 100), 300000);
114+
ASSERT_EQ(cbm_watcher_index_backoff_ms(5000, INT_MAX), 300000);
115+
ASSERT_EQ(cbm_watcher_index_backoff_ms(60000, INT_MAX), 300000);
116+
PASS();
117+
}
118+
119+
TEST(index_backoff_is_monotonic_and_bounded) {
120+
/* The property that actually matters: the delay never decreases as the
121+
* streak grows, and never exceeds the ceiling. This is what bounds the
122+
* fork rate of a permanently failing project. */
123+
int previous = cbm_watcher_index_backoff_ms(5000, 0);
124+
for (int failures = 1; failures < 200; failures++) {
125+
int delay = cbm_watcher_index_backoff_ms(5000, failures);
126+
ASSERT_TRUE(delay >= previous);
127+
ASSERT_TRUE(delay <= 300000);
128+
previous = delay;
129+
}
130+
PASS();
131+
}
132+
133+
TEST(index_backoff_degenerate_inputs) {
134+
ASSERT_EQ(cbm_watcher_index_backoff_ms(0, 5), 0);
135+
ASSERT_EQ(cbm_watcher_index_backoff_ms(-1, 0), 0);
136+
ASSERT_EQ(cbm_watcher_index_backoff_ms(5000, -1), 5000);
137+
PASS();
138+
}
139+
88140
/* ══════════════════════════════════════════════════════════════════
89141
* LIFECYCLE
90142
* ══════════════════════════════════════════════════════════════════ */
@@ -3143,6 +3195,11 @@ SUITE(watcher) {
31433195
RUN_TEST(poll_interval_scaling);
31443196
RUN_TEST(poll_interval_cap);
31453197
RUN_TEST(poll_interval_small);
3198+
RUN_TEST(index_backoff_no_failures_keeps_interval);
3199+
RUN_TEST(index_backoff_doubles_per_failure);
3200+
RUN_TEST(index_backoff_reaches_ceiling_and_stays);
3201+
RUN_TEST(index_backoff_is_monotonic_and_bounded);
3202+
RUN_TEST(index_backoff_degenerate_inputs);
31463203

31473204
/* Lifecycle */
31483205
RUN_TEST(watcher_create_free);

0 commit comments

Comments
 (0)