Skip to content

Commit 0c64de9

Browse files
authored
Merge pull request #1577 from DeusData/fix/spawn-retry-backoff
fix(subprocess): back off exponentially when the kernel says "try again"
2 parents e26b526 + 38b9f9e commit 0c64de9

3 files changed

Lines changed: 118 additions & 7 deletions

File tree

‎src/foundation/subprocess.c‎

Lines changed: 69 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -885,21 +885,75 @@ static cbm_proc_poll_t cbm_subprocess_poll_win(cbm_subprocess_t *process, cbm_pr
885885
/* Transient spawn-failure retry (see the EAGAIN note in cbm_posix_spawn_apple).
886886
* Three attempts over ~30ms total: long enough to ride out a burst of process
887887
* creation, short enough that a genuinely exhausted system still fails fast. */
888-
enum { CBM_SPAWN_RETRY = 2, CBM_SPAWN_RETRY_ATTEMPTS = 3 };
889-
static const struct timespec CBM_SPAWN_RETRY_DELAY = {0, 10L * 1000L * 1000L};
890-
#define CBM_SPAWN_RETRY_DELAY_NS (&CBM_SPAWN_RETRY_DELAY)
888+
enum { CBM_SPAWN_RETRY = 2, CBM_SPAWN_RETRY_ATTEMPTS = 6 };
889+
890+
/* Exponential backoff: 10, 20, 40, 80, 160, 320ms — ~630ms of total patience.
891+
*
892+
* The first version waited a flat 3 x 10ms, which was enough for a momentary
893+
* dip and NOT enough for the real thing: a CI runner building and testing in
894+
* parallel stays process-starved for hundreds of milliseconds at a stretch, and
895+
* `subprocess_run_spawn_failure` kept failing with the retry shipped (macos-15-
896+
* intel on a release matrix, macos-14 under ThreadSanitizer, which is itself
897+
* slow enough to create the pressure). A fixed short delay samples the same
898+
* congested instant repeatedly; doubling walks out of it.
899+
*
900+
* The ceiling is deliberate. ~0.6s is invisible next to spawning a process that
901+
* does real work, and a machine still refusing after that is genuinely out of
902+
* capacity — at which point failing IS the correct answer, and failing fast
903+
* beats hanging. */
904+
#ifdef CBM_ENABLE_TEST_SEAMS
905+
/* Deterministic EAGAIN injection: see the header. Counts DOWN, so a test asks
906+
* for N simulated refusals and the (N+1)th attempt proceeds for real. */
907+
static int g_force_spawn_eagain = 0;
908+
void cbm_subprocess_force_spawn_eagain_for_testing(int attempts) {
909+
g_force_spawn_eagain = attempts > 0 ? attempts : 0;
910+
}
911+
int cbm_subprocess_pending_spawn_eagain_for_testing(void) {
912+
return g_force_spawn_eagain;
913+
}
914+
static bool cbm_spawn_eagain_injected(void) {
915+
if (g_force_spawn_eagain > 0) {
916+
g_force_spawn_eagain--;
917+
return true;
918+
}
919+
return false;
920+
}
921+
#endif
922+
923+
static void cbm_spawn_backoff(int attempt) {
924+
long ms = 10L << (attempt < 6 ? attempt : 6);
925+
struct timespec delay = {ms / 1000L, (ms % 1000L) * 1000L * 1000L};
926+
(void)cbm_nanosleep(&delay, NULL);
927+
}
891928

892929
/* fork() fails with EAGAIN under the same pressure posix_spawn does, and the
893930
* fallback path must not be less robust than the primary one. */
894931
static pid_t cbm_fork_with_retry(void) {
895-
for (int attempt = 0; attempt < CBM_SPAWN_RETRY_ATTEMPTS; attempt++) {
932+
/* CBM_SPAWN_RETRY_ATTEMPTS backoffs means ATTEMPTS+1 tries. Every try goes
933+
* through the same branch — including the last — so the injection seam
934+
* models production exactly rather than leaving a final unguarded fork() the
935+
* tests could never reach. */
936+
for (int attempt = 0;; attempt++) {
937+
#ifdef CBM_ENABLE_TEST_SEAMS
938+
if (cbm_spawn_eagain_injected()) {
939+
if (attempt >= CBM_SPAWN_RETRY_ATTEMPTS) {
940+
errno = EAGAIN;
941+
return -1;
942+
}
943+
cbm_spawn_backoff(attempt);
944+
continue;
945+
}
946+
#endif
896947
pid_t pid = fork();
897948
if (pid >= 0 || (errno != EAGAIN && errno != ENOMEM)) {
898949
return pid;
899950
}
900-
(void)cbm_nanosleep(CBM_SPAWN_RETRY_DELAY_NS, NULL);
951+
if (attempt >= CBM_SPAWN_RETRY_ATTEMPTS) {
952+
errno = EAGAIN;
953+
return -1;
954+
}
955+
cbm_spawn_backoff(attempt);
901956
}
902-
return fork();
903957
}
904958

905959
/* Used by the fork+exec child. posix_spawn performs the same reset
@@ -989,6 +1043,14 @@ static int cbm_posix_fd_at_least_three(int fd) {
9891043
* because dup2 clears close-on-exec.
9901044
* posix_spawnp keeps execvp's PATH semantics for a bare tool name. */
9911045
static int cbm_posix_spawn_apple(cbm_subprocess_t *process, int input, int output, pid_t *pid_out) {
1046+
/* posix_spawn is the PRIMARY path on macOS; fork+exec is only the
1047+
* exec-class fallback. Injection therefore has to live here too, or a test
1048+
* on macOS exercises nothing. */
1049+
#ifdef CBM_ENABLE_TEST_SEAMS
1050+
if (cbm_spawn_eagain_injected()) {
1051+
return CBM_SPAWN_RETRY;
1052+
}
1053+
#endif
9921054
posix_spawn_file_actions_t actions;
9931055
posix_spawnattr_t attr;
9941056
if (posix_spawn_file_actions_init(&actions) != 0) {
@@ -1089,7 +1151,7 @@ static int cbm_subprocess_spawn_posix(cbm_subprocess_t *process) {
10891151
int spawn_rc = cbm_posix_spawn_apple(process, input, output, &pid);
10901152
for (int attempt = 0; spawn_rc == CBM_SPAWN_RETRY && attempt < CBM_SPAWN_RETRY_ATTEMPTS;
10911153
attempt++) {
1092-
(void)cbm_nanosleep(CBM_SPAWN_RETRY_DELAY_NS, NULL);
1154+
cbm_spawn_backoff(attempt);
10931155
spawn_rc = cbm_posix_spawn_apple(process, input, output, &pid);
10941156
}
10951157
if (spawn_rc == CBM_SPAWN_RETRY) {

‎src/foundation/subprocess.h‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -177,4 +177,12 @@ bool cbm_build_win_cmdline(char *buf, size_t cap, const char *const *argv);
177177
bool cbm_build_win_cmd_payload(char *buf, size_t cap, const char *cmd_executable,
178178
const char *payload);
179179

180+
#ifdef CBM_ENABLE_TEST_SEAMS
181+
/* Force the next N spawn attempts to behave as if the kernel returned EAGAIN
182+
* ("try again"), so the retry path can be exercised deterministically instead
183+
* of hoping a loaded machine reproduces it. Test builds only. */
184+
void cbm_subprocess_force_spawn_eagain_for_testing(int attempts);
185+
int cbm_subprocess_pending_spawn_eagain_for_testing(void);
186+
#endif
187+
180188
#endif /* CBM_SUBPROCESS_H */

‎tests/test_subprocess.c‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -169,6 +169,45 @@ TEST(subprocess_run_hang_is_hang) {
169169
}
170170

171171
/* A spawn of a non-existent binary fails cleanly (no child), not a crash. */
172+
/* The kernel refusing a spawn with EAGAIN means "not right now", not "never" —
173+
* a momentarily full process table on a busy machine. We used to treat it as a
174+
* permanent failure, so a git probe or LSP server refused to start for a reason
175+
* the user could neither see nor act on, and `subprocess_run_spawn_failure`
176+
* failed on loaded CI runners with the contract intact.
177+
*
178+
* These pin the retry itself rather than the constant. Injecting refusals is
179+
* deterministic, so this proves the loop retries on EVERY machine instead of
180+
* only on one that happens to be starved. */
181+
TEST(subprocess_retries_transient_spawn_refusal) {
182+
#ifdef _WIN32
183+
SKIP_PLATFORM("POSIX fork/EAGAIN path");
184+
#else
185+
/* Fewer refusals than the budget: the spawn must still succeed. */
186+
cbm_subprocess_force_spawn_eagain_for_testing(3);
187+
cbm_proc_result_t r = run_sh("exit 0", 0);
188+
ASSERT_EQ(cbm_subprocess_pending_spawn_eagain_for_testing(), 0);
189+
ASSERT_EQ(r.outcome, CBM_PROC_CLEAN);
190+
ASSERT_EQ(r.exit_code, 0);
191+
PASS();
192+
#endif
193+
}
194+
195+
TEST(subprocess_gives_up_after_the_retry_budget) {
196+
#ifdef _WIN32
197+
SKIP_PLATFORM("POSIX fork/EAGAIN path");
198+
#else
199+
/* More refusals than the budget: it must fail rather than retry forever.
200+
* A machine still refusing after ~0.6s is genuinely out of capacity, and
201+
* failing fast beats hanging. */
202+
cbm_subprocess_force_spawn_eagain_for_testing(50);
203+
cbm_proc_result_t r = run_sh("exit 0", 0);
204+
bool refused = r.outcome == CBM_PROC_SPAWN_FAILED;
205+
cbm_subprocess_force_spawn_eagain_for_testing(0); /* never leak into later tests */
206+
ASSERT_TRUE(refused);
207+
PASS();
208+
#endif
209+
}
210+
172211
TEST(subprocess_run_spawn_failure) {
173212
#ifdef _WIN32
174213
SKIP_PLATFORM("POSIX exec semantics");
@@ -993,6 +1032,8 @@ SUITE(subprocess) {
9931032
RUN_TEST(subprocess_run_resolves_literal_binary_name_from_path);
9941033
RUN_TEST(subprocess_run_crash_is_crash);
9951034
RUN_TEST(subprocess_run_hang_is_hang);
1035+
RUN_TEST(subprocess_retries_transient_spawn_refusal);
1036+
RUN_TEST(subprocess_gives_up_after_the_retry_budget);
9961037
RUN_TEST(subprocess_run_spawn_failure);
9971038
RUN_TEST(subprocess_run_null_bin_rejected);
9981039
RUN_TEST(subprocess_spawn_returns_while_child_is_running);

0 commit comments

Comments
 (0)