Skip to content

Commit c1af82f

Browse files
authored
Merge pull request #2840 from halindrome/fix/watcher-reindex-cooldown
fix(watcher): pace re-indexes by the cost of the last successful run
2 parents fe55b86 + cbf3794 commit c1af82f

3 files changed

Lines changed: 208 additions & 4 deletions

File tree

‎src/watcher/watcher.c‎

Lines changed: 64 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,15 @@ struct cbm_watcher {
154154
* daemon log names the stuck project instead of only repeating the warning. */
155155
#define INDEX_FAIL_SUSTAINED 10
156156

157+
/* Success cooldown (#2835): after a successful re-index the next poll waits
158+
* INDEX_COOLDOWN_FACTOR x the index's own duration, counted from when the
159+
* index finished, so short rebuilds spend at most ~1/(K+1) of wall time
160+
* indexing one project. Sub-second indexes stay at the plain cadence. The
161+
* cooldown is capped at INDEX_FAIL_CEILING_MS (5 min) so that, under
162+
* continuous writes, a project never falls more than one rebuild plus five
163+
* minutes behind. */
164+
#define INDEX_COOLDOWN_FACTOR 3
165+
157166
/* Stale-root pruning (#286): a watched project whose root directory stays
158167
* missing is pruned — its cached DB is deleted and the watch entry removed.
159168
* Deletion is destructive (the DB can hold user-authored data such as the
@@ -175,7 +184,17 @@ struct cbm_watcher {
175184

176185
/* ── Time helper ────────────────────────────────────────────────── */
177186

187+
#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS
188+
/* Test seam: nonzero freezes the watcher's scheduling clock at this value. */
189+
static int64_t test_clock_ns = 0;
190+
#endif
191+
178192
static int64_t now_ns(void) {
193+
#if defined(CBM_ENABLE_TEST_SEAMS) && CBM_ENABLE_TEST_SEAMS
194+
if (test_clock_ns != 0) {
195+
return test_clock_ns;
196+
}
197+
#endif
179198
struct timespec ts;
180199
cbm_clock_gettime(CLOCK_MONOTONIC, &ts);
181200
return ((int64_t)ts.tv_sec * NS_PER_SEC) + ts.tv_nsec;
@@ -204,6 +223,20 @@ int cbm_watcher_index_backoff_ms(int interval_ms, int consecutive_failures) {
204223
return (int)(delay < interval_ms ? interval_ms : delay);
205224
}
206225

226+
int cbm_watcher_success_delay_ms(int interval_ms, int64_t index_elapsed_ms) {
227+
if (interval_ms < 0) {
228+
interval_ms = 0;
229+
}
230+
if (index_elapsed_ms <= 0) {
231+
return interval_ms;
232+
}
233+
/* Divide instead of multiply so a huge elapsed cannot overflow. */
234+
int64_t cooldown = index_elapsed_ms > INDEX_FAIL_CEILING_MS / INDEX_COOLDOWN_FACTOR
235+
? INDEX_FAIL_CEILING_MS
236+
: index_elapsed_ms * INDEX_COOLDOWN_FACTOR;
237+
return (int)(cooldown > interval_ms ? cooldown : interval_ms);
238+
}
239+
207240
int cbm_watcher_poll_interval_ms(int file_count) {
208241
int ms = POLL_BASE_MS + ((file_count / POLL_FILE_STEP) * CBM_MSEC_PER_SEC);
209242
if (ms > POLL_MAX_MS) {
@@ -1693,12 +1726,22 @@ static void poll_project(const char *key, void *val, void *ud) {
16931726
}
16941727
cbm_log_info("watcher.changed", "project", s->project_name, "strategy",
16951728
s->is_git ? "git" : "tree");
1729+
int delay_ms = -1; /* < 0: use the failure-backoff cadence below */
1730+
int64_t anchor_ns = ctx->now; /* success re-anchors to when the index finished */
16961731
if (ctx->w->index_fn) {
1732+
int64_t index_start = now_ns();
16971733
int rc = ctx->w->index_fn(s->project_name, s->root_path, ctx->w->user_data);
16981734
if (rc == 0) {
16991735
ctx->reindexed++;
17001736
s->index_failures = 0;
17011737
commit_baselines(ctx->w, s);
1738+
anchor_ns = now_ns();
1739+
delay_ms =
1740+
cbm_watcher_success_delay_ms(s->interval_ms, (anchor_ns - index_start) / US_PER_MS);
1741+
if (delay_ms > s->interval_ms) {
1742+
cbm_log_info("watcher.cooldown", "project", s->project_name, "delay_ms",
1743+
itoa_buf(delay_ms));
1744+
}
17021745
} else if (rc > 0) {
17031746
/* Busy-skip: baseline stays uncommitted, next poll retries. */
17041747
cbm_log_info("watcher.index.retry", "project", s->project_name);
@@ -1721,10 +1764,12 @@ static void poll_project(const char *key, void *val, void *ud) {
17211764
}
17221765
}
17231766

1724-
/* Failures back off; success and busy-skip keep the adaptive cadence. */
1725-
s->next_poll_ns =
1726-
ctx->now +
1727-
((int64_t)cbm_watcher_index_backoff_ms(s->interval_ms, s->index_failures) * US_PER_MS);
1767+
/* Success pays back the index's cost; failures back off; busy-skip keeps
1768+
* the adaptive cadence. */
1769+
if (delay_ms < 0) {
1770+
delay_ms = cbm_watcher_index_backoff_ms(s->interval_ms, s->index_failures);
1771+
}
1772+
s->next_poll_ns = anchor_ns + ((int64_t)delay_ms * US_PER_MS);
17281773
}
17291774

17301775
/* Callback to snapshot project state pointers into an array. */
@@ -1876,4 +1921,19 @@ int cbm_watcher_test_pending_free_count(cbm_watcher_t *w) {
18761921
cbm_mutex_unlock(&w->projects_lock);
18771922
return count;
18781923
}
1924+
1925+
void cbm_watcher_test_set_clock_ns(int64_t ns) {
1926+
test_clock_ns = ns;
1927+
}
1928+
1929+
int64_t cbm_watcher_test_next_poll_ns(cbm_watcher_t *w, const char *project) {
1930+
if (!w || !project) {
1931+
return -1;
1932+
}
1933+
cbm_mutex_lock(&w->projects_lock);
1934+
project_state_t *s = cbm_ht_get(w->projects, project);
1935+
int64_t next = s ? s->next_poll_ns : -1;
1936+
cbm_mutex_unlock(&w->projects_lock);
1937+
return next;
1938+
}
18791939
#endif

‎src/watcher/watcher.h‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,15 @@ int cbm_watcher_index_failure_count(cbm_watcher_t *w, const char *project_name);
145145
* otherwise unobservable — cbm_watcher_free frees the same list, so a poll
146146
* that stopped draining would neither crash nor leak. Test builds only. */
147147
int cbm_watcher_test_pending_free_count(cbm_watcher_t *w);
148+
149+
/* Test seam: freeze the watcher's monotonic scheduling clock at `ns`
150+
* (nonzero), or restore the real clock (0). Process-global; single-threaded
151+
* tests only. Lets a test advance time inside index_fn without sleeping. */
152+
void cbm_watcher_test_set_clock_ns(int64_t ns);
153+
154+
/* Test seam: a project's scheduled next poll (monotonic ns), read under
155+
* projects_lock; -1 when it is not watched. */
156+
int64_t cbm_watcher_test_next_poll_ns(cbm_watcher_t *w, const char *project);
148157
#endif
149158

150159
/* Per-project watch status, for index_status (#2167). */
@@ -195,6 +204,15 @@ int cbm_watcher_poll_interval_ms(int file_count);
195204
* Negative inputs are treated as zero. */
196205
int cbm_watcher_index_backoff_ms(int interval_ms, int consecutive_failures);
197206

207+
/* Return the delay (ms) before the next poll after a SUCCESSFUL re-index that
208+
* took `index_elapsed_ms`: max(interval_ms, INDEX_COOLDOWN_FACTOR (3) x
209+
* index_elapsed_ms), with the cooldown term capped at INDEX_FAIL_CEILING_MS
210+
* (5 min). The caller counts it from when the index finished, so short
211+
* rebuilds spend at most ~1/4 of wall time indexing a project; sub-second
212+
* indexes keep `interval_ms`.
213+
* Never below `interval_ms`; overflow-safe; negative inputs act as zero. */
214+
int cbm_watcher_success_delay_ms(int interval_ms, int64_t index_elapsed_ms);
215+
198216
/* Classify a stat() errno observed on a watched project root: returns true
199217
* only for values that mean the root itself is gone (ENOENT, ENOTDIR) and
200218
* may count toward stale-root pruning (#286). Any other failure (EACCES,

‎tests/test_watcher.c‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2260,6 +2260,127 @@ TEST(watcher_sustained_failure_logs_once_issue2015) {
22602260
PASS();
22612261
}
22622262

2263+
/* ── Success cooldown (#2835) ─────────────────────────────────────── */
2264+
2265+
TEST(success_delay_fast_index_keeps_interval) {
2266+
ASSERT_EQ(cbm_watcher_success_delay_ms(5000, 0), 5000);
2267+
ASSERT_EQ(cbm_watcher_success_delay_ms(5000, 1), 5000);
2268+
ASSERT_EQ(cbm_watcher_success_delay_ms(5000, 1666), 5000); /* 3x < interval */
2269+
ASSERT_EQ(cbm_watcher_success_delay_ms(60000, 20000), 60000);
2270+
PASS();
2271+
}
2272+
2273+
TEST(success_delay_slow_index_scales_by_factor) {
2274+
ASSERT_EQ(cbm_watcher_success_delay_ms(5000, 1667), 5001);
2275+
ASSERT_EQ(cbm_watcher_success_delay_ms(5000, 10000), 30000);
2276+
/* 90 s rebuild → next poll 4.5 min after it finishes. */
2277+
ASSERT_EQ(cbm_watcher_success_delay_ms(60000, 90000), 270000);
2278+
PASS();
2279+
}
2280+
2281+
TEST(success_delay_clamped_and_degenerate) {
2282+
/* Capped at 5 min (INDEX_FAIL_CEILING_MS) from 100 s of indexing up. */
2283+
ASSERT_EQ(cbm_watcher_success_delay_ms(5000, 100000), 300000);
2284+
ASSERT_EQ(cbm_watcher_success_delay_ms(5000, 100001), 300000);
2285+
ASSERT_EQ(cbm_watcher_success_delay_ms(60000, 300000), 300000);
2286+
ASSERT_EQ(cbm_watcher_success_delay_ms(5000, INT64_MAX), 300000);
2287+
ASSERT_EQ(cbm_watcher_success_delay_ms(5000, -1), 5000);
2288+
ASSERT_EQ(cbm_watcher_success_delay_ms(5000, INT64_MIN), 5000);
2289+
ASSERT_EQ(cbm_watcher_success_delay_ms(-1, 0), 0);
2290+
/* Never below the caller's own cadence, even above the ceiling. */
2291+
ASSERT_EQ(cbm_watcher_success_delay_ms(INT_MAX, INT64_MAX), INT_MAX);
2292+
PASS();
2293+
}
2294+
2295+
TEST(success_delay_is_monotonic) {
2296+
int previous = cbm_watcher_success_delay_ms(5000, 0);
2297+
for (int64_t e = 0; e <= 2000000; e += 997) {
2298+
int delay = cbm_watcher_success_delay_ms(5000, e);
2299+
ASSERT_TRUE(delay >= previous);
2300+
ASSERT_TRUE(delay >= 5000 && delay <= 300000);
2301+
previous = delay;
2302+
}
2303+
PASS();
2304+
}
2305+
2306+
/* Wire-up: the cooldown must be measured around the real index_fn call and
2307+
* counted from when the index finished. The watcher's clock is frozen via a
2308+
* test seam and the callback advances it, so an index "takes" a chosen time
2309+
* with no sleep and the scheduled poll time can be asserted exactly. */
2310+
#define COOL_NS_PER_MS 1000000LL
2311+
static int64_t cool_clock_ns = 0;
2312+
static int64_t cool_index_ms = 0;
2313+
static int cool_index_calls = 0;
2314+
static int cool_index_callback(const char *name, const char *path, void *ud) {
2315+
(void)name;
2316+
(void)path;
2317+
(void)ud;
2318+
cool_index_calls++;
2319+
cool_clock_ns += cool_index_ms * COOL_NS_PER_MS;
2320+
cbm_watcher_test_set_clock_ns(cool_clock_ns);
2321+
return 0;
2322+
}
2323+
2324+
/* Re-index once (touch + poll until the callback runs; a transient git probe
2325+
* failure makes a poll a no-op, see the sustained-failure test) and return
2326+
* the scheduled next poll relative to when the index finished, in ms. */
2327+
static int64_t cool_reindex_delay_ms(cbm_watcher_t *w, const char *file, const char *line,
2328+
int64_t index_ms) {
2329+
cool_index_ms = index_ms;
2330+
th_append_file(file, line);
2331+
int target = cool_index_calls + 1;
2332+
while (cool_index_calls < target) {
2333+
cbm_watcher_touch(w, "cool-repo");
2334+
cbm_watcher_poll_once(w);
2335+
}
2336+
return (cbm_watcher_test_next_poll_ns(w, "cool-repo") - cool_clock_ns) / COOL_NS_PER_MS;
2337+
}
2338+
2339+
TEST(watcher_success_cooldown_measures_index_issue2835) {
2340+
char tmpdir[256];
2341+
snprintf(tmpdir, sizeof(tmpdir), "/tmp/cbm_watcher_cool_XXXXXX");
2342+
if (!cbm_mkdtemp(tmpdir))
2343+
FAIL("cbm_mkdtemp failed");
2344+
if (wt_git(tmpdir, "init -q") != 0) {
2345+
th_rmtree(tmpdir);
2346+
FAIL("git init failed");
2347+
}
2348+
char p[300];
2349+
th_write_file(wt_path(p, sizeof(p), tmpdir, "file.txt"), "hello\n");
2350+
wt_git(tmpdir, "add file.txt");
2351+
wt_git(tmpdir, "commit -q -m init");
2352+
2353+
cool_clock_ns = 1000LL * 1000 * COOL_NS_PER_MS; /* any nonzero start */
2354+
cbm_watcher_test_set_clock_ns(cool_clock_ns);
2355+
cool_index_calls = 0;
2356+
2357+
cbm_store_t *store = cbm_store_open_memory();
2358+
cbm_watcher_t *w = cbm_watcher_new(store, cool_index_callback, NULL);
2359+
cbm_watcher_watch(w, "cool-repo", tmpdir);
2360+
cbm_watcher_poll_once(w); /* baseline */
2361+
2362+
wt_path(p, sizeof(p), tmpdir, "file.txt");
2363+
/* Instant index: plain cadence. */
2364+
int64_t fast = cool_reindex_delay_ms(w, p, "a\n", 0);
2365+
/* 60 s index → 3 x 60 s, counted from the END of the index (anchoring at
2366+
* the start would give 120 s here). */
2367+
int64_t slow = cool_reindex_delay_ms(w, p, "b\n", 60000);
2368+
/* 200 s index → 600 s, capped at 5 min. */
2369+
int64_t capped = cool_reindex_delay_ms(w, p, "c\n", 200000);
2370+
2371+
/* Restore the real clock before any assert can return early. */
2372+
cbm_watcher_test_set_clock_ns(0);
2373+
2374+
ASSERT_EQ(fast, cbm_watcher_poll_interval_ms(1));
2375+
ASSERT_EQ(slow, 180000);
2376+
ASSERT_EQ(capped, 300000);
2377+
2378+
cbm_watcher_free(w);
2379+
cbm_store_close(store);
2380+
th_rmtree(tmpdir);
2381+
PASS();
2382+
}
2383+
22632384
TEST(watcher_index_failure_count_unknown_project) {
22642385
cbm_store_t *store = cbm_store_open_memory();
22652386
cbm_watcher_t *w = cbm_watcher_new(store, index_callback, NULL);
@@ -4043,6 +4164,11 @@ SUITE(watcher) {
40434164
RUN_TEST(index_backoff_never_schedules_sooner_than_the_interval);
40444165
RUN_TEST(watcher_index_failure_streak_increments_and_resets_issue2015);
40454166
RUN_TEST(watcher_index_failure_backoff_gates_repolling_issue2015);
4167+
RUN_TEST(success_delay_fast_index_keeps_interval);
4168+
RUN_TEST(success_delay_slow_index_scales_by_factor);
4169+
RUN_TEST(success_delay_clamped_and_degenerate);
4170+
RUN_TEST(success_delay_is_monotonic);
4171+
RUN_TEST(watcher_success_cooldown_measures_index_issue2835);
40464172
RUN_TEST(watcher_sustained_failure_logs_once_issue2015);
40474173
RUN_TEST(watcher_index_failure_count_unknown_project);
40484174
RUN_TEST(watcher_project_info_publishes_scan_issue2167);

0 commit comments

Comments
 (0)