Skip to content

Commit 1d66538

Browse files
authored
Merge pull request #2112 from DeusData/fix/decision-a-memory-budget-fail-whole
fix(pipeline,mcp,daemon): fail an over-budget index attempt whole with a named reason (#1997, #832)
2 parents b5185d1 + 21591d5 commit 1d66538

9 files changed

Lines changed: 670 additions & 54 deletions

File tree

‎src/daemon/application.c‎

Lines changed: 54 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -181,7 +181,7 @@ struct cbm_daemon_application {
181181
cbm_daemon_application_update_ops_t update_ops;
182182
cbm_project_lock_manager_t *project_locks;
183183
size_t physical_job_limit;
184-
size_t worker_memory_budget_bytes;
184+
size_t aggregate_memory_budget_bytes;
185185
size_t active_mutations;
186186
size_t update_owners;
187187
cbm_daemon_application_update_worker_t update_worker;
@@ -693,6 +693,8 @@ static bool application_truncate_file(const char *path) {
693693
}
694694

695695
static bool application_job_cancel_requested(cbm_daemon_application_job_t *job);
696+
static size_t application_worker_memory_slice_locked(cbm_daemon_application_t *application,
697+
size_t *active_jobs_out);
696698

697699
static char **application_read_suspects(cbm_daemon_application_job_t *job, const char *path,
698700
int *count_out, bool *cancelled_out) {
@@ -1088,11 +1090,33 @@ static application_attempt_status_t application_job_run_attempt(cbm_daemon_appli
10881090
return APPLICATION_ATTEMPT_CANCELLED;
10891091
}
10901092

1093+
/* Decision 2 (#1997 #832): the slice is decided at spawn time from the jobs
1094+
* active right now (this job included), so a lone worker may use the whole
1095+
* aggregate and concurrent jobs split it. The divisor is logged so a
1096+
* fail-whole verdict can be read against the budget the worker really had. */
1097+
size_t active_jobs = 0;
1098+
cbm_mutex_lock(&application->mutex);
1099+
size_t memory_budget_bytes = application_worker_memory_slice_locked(application, &active_jobs);
1100+
size_t aggregate_memory_budget_bytes = application->aggregate_memory_budget_bytes;
1101+
cbm_mutex_unlock(&application->mutex);
1102+
if (memory_budget_bytes > 0) {
1103+
char active_text[32];
1104+
char aggregate_text[32];
1105+
char slice_text[32];
1106+
(void)snprintf(active_text, sizeof(active_text), "%zu", active_jobs);
1107+
(void)snprintf(aggregate_text, sizeof(aggregate_text), "%zu",
1108+
aggregate_memory_budget_bytes / (1024U * 1024U));
1109+
(void)snprintf(slice_text, sizeof(slice_text), "%zu",
1110+
memory_budget_bytes / (1024U * 1024U));
1111+
cbm_log_info("daemon.index.worker_budget", "project", job->project_key, "active_jobs",
1112+
active_text, "aggregate_mb", aggregate_text, "slice_mb", slice_text);
1113+
}
1114+
10911115
cbm_daemon_application_worker_t worker = NULL;
10921116
application_tmp_lock();
1093-
int start_result = application->worker_ops.start(
1094-
application->worker_ops.context, job->args_json, application->worker_memory_budget_bytes,
1095-
marker_path, quarantine_path, &worker);
1117+
int start_result =
1118+
application->worker_ops.start(application->worker_ops.context, job->args_json,
1119+
memory_budget_bytes, marker_path, quarantine_path, &worker);
10961120
application_tmp_unlock();
10971121
if (start_result != 0 || !worker) {
10981122
return application_job_cancel_requested(job) ? APPLICATION_ATTEMPT_CANCELLED
@@ -1564,6 +1588,22 @@ static size_t application_active_job_count_locked(cbm_daemon_application_t *appl
15641588
return count;
15651589
}
15661590

1591+
/* Decision 2 (#1997 #832): a worker's memory slice is the aggregate budget
1592+
* divided by the jobs active at spawn time. The job being spawned is already
1593+
* in the table, so the divisor never drops below one; a zero aggregate means
1594+
* "no cap" and the worker falls back to its own RAM-fraction budget. */
1595+
static size_t application_worker_memory_slice_locked(cbm_daemon_application_t *application,
1596+
size_t *active_jobs_out) {
1597+
size_t active = application_active_job_count_locked(application);
1598+
if (active == 0) {
1599+
active = 1;
1600+
}
1601+
if (active_jobs_out) {
1602+
*active_jobs_out = active;
1603+
}
1604+
return application->aggregate_memory_budget_bytes / active;
1605+
}
1606+
15671607
/* Compare the effective index request, not its JSON spelling. yyjson's deep
15681608
* equality treats object member order as insignificant, while the small
15691609
* normalization below removes values that the index handler interprets as
@@ -2948,18 +2988,19 @@ cbm_daemon_application_t *cbm_daemon_application_new(
29482988
application->ui_readiness_secret_set = true;
29492989
}
29502990
}
2951-
/* Equal fixed slices keep admission deterministic: starting fewer jobs does
2952-
* not let an early worker claim memory reserved for later concurrent jobs.
2953-
* The absurd sub-byte-per-slot case is made safe by reducing effective
2954-
* capacity before division; normal daemon budgets are many orders larger. */
2991+
/* The per-worker slice is decided at spawn time (see
2992+
* application_worker_memory_slice_locked): the aggregate divided by the
2993+
* jobs active then, so a lone worker may use the whole aggregate and
2994+
* concurrent jobs split it (decision 2, #1997 #832) — the former fixed
2995+
* aggregate/limit slice starved a lone job on hosts like #1864. The absurd
2996+
* sub-byte-per-slot case is still made safe by reducing effective capacity
2997+
* so every admitted job can receive at least one byte; normal daemon
2998+
* budgets are many orders larger. */
29552999
if (aggregate_memory_budget_bytes > 0 &&
29563000
application->physical_job_limit > aggregate_memory_budget_bytes) {
29573001
application->physical_job_limit = aggregate_memory_budget_bytes;
29583002
}
2959-
if (aggregate_memory_budget_bytes > 0 && application->physical_job_limit > 0) {
2960-
application->worker_memory_budget_bytes =
2961-
aggregate_memory_budget_bytes / application->physical_job_limit;
2962-
}
3003+
application->aggregate_memory_budget_bytes = aggregate_memory_budget_bytes;
29633004
if (!application->worker_ops.start) {
29643005
application->worker_ops = (cbm_daemon_application_worker_ops_t){
29653006
.context = NULL,
@@ -3558,7 +3599,7 @@ size_t cbm_daemon_application_worker_memory_budget_bytes(cbm_daemon_application_
35583599
return 0;
35593600
}
35603601
cbm_mutex_lock(&application->mutex);
3561-
size_t budget = application->worker_memory_budget_bytes;
3602+
size_t budget = application_worker_memory_slice_locked(application, NULL);
35623603
cbm_mutex_unlock(&application->mutex);
35633604
return budget;
35643605
}

‎src/mcp/mcp.c‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11209,6 +11209,49 @@ static char *handle_index_repository(cbm_mcp_server_t *srv, const char *args) {
1120911209
if (cbm_pipeline_had_format_migration(p)) {
1121011210
yyjson_mut_obj_add_bool(doc, root, "format_migration", true);
1121111211
}
11212+
} else if (rc == CBM_PIPELINE_ABORT_OVER_BUDGET) {
11213+
/* Decision A (#1997 #832): resident memory stayed above the budget
11214+
* after back-pressure and one confirmation cycle, so the run stopped
11215+
* before publication. Name the cause, the numbers and the fact that
11216+
* the previous index still serves — the generic "check repo_path"
11217+
* hint below sent people debugging a path that was fine. */
11218+
int budget_mb = (int)(cbm_mem_budget() / (1024 * 1024));
11219+
int peak_rss_mb = (int)(cbm_mem_peak_rss() / (1024 * 1024));
11220+
char budget_text[CBM_SZ_32];
11221+
char peak_text[CBM_SZ_32];
11222+
(void)snprintf(budget_text, sizeof(budget_text), "%d", budget_mb);
11223+
(void)snprintf(peak_text, sizeof(peak_text), "%d", peak_rss_mb);
11224+
cbm_log_error("index.abort", "reason", "over_memory_budget", "project", project_name,
11225+
"budget_mb", budget_text, "peak_rss_mb", peak_text);
11226+
yyjson_mut_obj_add_str(doc, root, "status", "error");
11227+
yyjson_mut_obj_add_str(doc, root, "reason", "over_memory_budget");
11228+
yyjson_mut_obj_add_str(doc, root, "previous_index", "preserved");
11229+
yyjson_mut_obj_add_int(doc, root, "budget_mb", budget_mb);
11230+
yyjson_mut_obj_add_int(doc, root, "peak_rss_mb", peak_rss_mb);
11231+
yyjson_mut_obj_add_str(doc, root, "hint",
11232+
"Indexing stopped: resident memory stayed above the budget after "
11233+
"backpressure; no partial graph was published and the previous "
11234+
"index still serves. Raise CBM_MEM_BUDGET_MB, lower CBM_WORKERS, "
11235+
"or exclude large subtrees.");
11236+
} else if (rc == CBM_PIPELINE_ABORT_PRESERVE_DB) {
11237+
/* The truthful abort message (#2020): the old generic "check repo_path"
11238+
* hint sent people debugging a path that was fine, when the run
11239+
* aborted pre-publication (semantic inputs changed mid-run, or a
11240+
* discovery/manifest phase failed transiently) and the previous index
11241+
* is intact. A 99-test cascade on the Windows leg traced back to
11242+
* exactly this — the response said error, but not which kind. */
11243+
yyjson_mut_obj_add_str(doc, root, "status", "aborted_previous_preserved");
11244+
yyjson_mut_obj_add_str(doc, root, "hint",
11245+
"Indexing aborted before publication; the previous index is "
11246+
"intact and still serving. Causes: files changed while the run "
11247+
"was in flight, or a discovery/manifest phase failed "
11248+
"transiently. Retry; if it repeats, check the run log.");
11249+
} else if (rc == CBM_PIPELINE_PERSIST_FAILED) {
11250+
yyjson_mut_obj_add_str(doc, root, "status", "persist_failed");
11251+
yyjson_mut_obj_add_str(doc, root, "hint",
11252+
"The validated staging database could not be published. Check "
11253+
"free disk space and permissions on the cache directory; the "
11254+
"previous index may have been rolled back.");
1121211255
} else {
1121311256
yyjson_mut_obj_add_str(doc, root, "status", "error");
1121411257
yyjson_mut_obj_add_str(doc, root, "hint",

‎src/pipeline/pass_parallel.c‎

Lines changed: 71 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,9 @@ enum {
3030
/* Extraction memory back-pressure: when the process is over its RSS budget,
3131
* a worker reclaims + naps before pulling another file so peers can finish
3232
* and return pages. Bounded spins avoid deadlock when the resident graph
33-
* itself is near budget (then proceed with a soft overshoot). */
33+
* itself is near budget: then ONE confirmation cycle decides — drained
34+
* means continue, still over means the attempt fails whole (decision A,
35+
* #1997 #832) instead of the former open-ended soft overshoot. */
3436
PP_BACKPRESSURE_MAX_SPINS = 40,
3537
PP_BACKPRESSURE_NAP_NS = 3000000, /* 3 ms */
3638
};
@@ -667,6 +669,11 @@ typedef struct {
667669
* While set, pulls skip the nap (the designed soft overshoot); the cheap
668670
* over-budget probe re-arms the gate once RSS drains under budget. */
669671
_Atomic int bp_futile;
672+
/* Decision A (#1997 #832): set once futility was CONFIRMED by a second
673+
* full cycle that still ended over budget. Every pull loop breaks on it
674+
* and extract returns CBM_PIPELINE_ABORT_OVER_BUDGET. Deliberately not
675+
* the shared cancel token — that one belongs to client cancellation. */
676+
_Atomic int over_budget_abort;
670677

671678
const CBMMacroTable *macro_table; /* ObjectScript $$$macros (NULL if none) */
672679
const CBMReturnTypeTable *return_type_table; /* ObjectScript return types (NULL if none) */
@@ -722,6 +729,39 @@ static void log_extract_done(int pos, uint64_t ms, int defs, const char *path) {
722729
}
723730
}
724731

732+
/* One back-pressure cycle: reclaim this thread's freed pages, then nap in
733+
* bounded 3 ms steps while the process stays over budget. Counted for test
734+
* observability. Returns true when the FULL cycle elapsed still over budget —
735+
* the resident floor, not in-flight transients, holds the memory. */
736+
static bool pp_backpressure_cycle_still_over(extract_ctx_t *ec) {
737+
cbm_mem_collect();
738+
atomic_fetch_add_explicit(&g_bp_nap_cycles, SKIP_ONE, memory_order_relaxed);
739+
int bp = 0;
740+
for (; bp < PP_BACKPRESSURE_MAX_SPINS && cbm_mem_over_budget() &&
741+
!atomic_load_explicit(ec->cancelled, memory_order_relaxed) &&
742+
!atomic_load_explicit(&ec->over_budget_abort, memory_order_relaxed);
743+
bp++) {
744+
struct timespec nap = {0, PP_BACKPRESSURE_NAP_NS};
745+
cbm_nanosleep(&nap, NULL);
746+
}
747+
return bp == PP_BACKPRESSURE_MAX_SPINS && cbm_mem_over_budget();
748+
}
749+
750+
/* Decision A (#1997 #832): back-pressure was futile and the confirmation
751+
* cycle still ended over budget. The budget stops being advisory here: flag
752+
* the attempt so every worker stops pulling and extract returns
753+
* CBM_PIPELINE_ABORT_OVER_BUDGET. Cooperative on purpose — the failure must
754+
* travel as a clean-exit worker response, never a kill, or the supervisor
755+
* would quarantine innocent files. */
756+
static void pp_fail_whole_over_budget(extract_ctx_t *ec) {
757+
if (atomic_exchange_explicit(&ec->over_budget_abort, 1, memory_order_relaxed) == 0) {
758+
cbm_log_error("mem.budget.exceeded", "rss_mb",
759+
itoa_log((int)(cbm_mem_rss() / (1024 * 1024))), "budget_mb",
760+
itoa_log((int)(cbm_mem_budget() / (1024 * 1024))), "phase",
761+
"parallel_extract", "action", "fail_whole");
762+
}
763+
}
764+
725765
static void extract_worker(int worker_id, void *ctx_ptr) {
726766
extract_ctx_t *ec = ctx_ptr;
727767
extract_worker_state_t *ws = &ec->workers[worker_id];
@@ -739,7 +779,8 @@ static void extract_worker(int worker_id, void *ctx_ptr) {
739779
break;
740780
}
741781
cbm_scale_tick(&ec->scale, sort_pos);
742-
if (atomic_load_explicit(ec->cancelled, memory_order_relaxed)) {
782+
if (atomic_load_explicit(ec->cancelled, memory_order_relaxed) ||
783+
atomic_load_explicit(&ec->over_budget_abort, memory_order_relaxed)) {
743784
break;
744785
}
745786

@@ -750,39 +791,41 @@ static void extract_worker(int worker_id, void *ctx_ptr) {
750791
* near the budget instead of letting all workers parse their biggest
751792
* files at once. Self-disabling when the budget is unset (tests) or RSS
752793
* is under budget; bounded spins avoid deadlock when the resident graph
753-
* is itself near budget (then proceed with a soft overshoot).
794+
* is itself near budget.
754795
*
755796
* Futility latch: when a FULL nap cycle ends still over budget, the
756797
* resident floor — not transients — holds the memory; napping again on
757798
* the next pull cannot reclaim it and only idles workers (linux kernel:
758-
* one full cycle per pull ≈ 390 s at 79% avg CPU). Latch bp_futile and
759-
* proceed with the soft overshoot; the over-budget probe below re-arms
760-
* the gate as soon as RSS drains under budget. */
799+
* one full cycle per pull ≈ 390 s at 79% avg CPU). The worker that
800+
* latches bp_futile (0→1) runs ONE confirmation cycle while its peers
801+
* skip the nap. A confirmation that ends under budget re-arms the gate
802+
* (RSS drained after all); one that still ends over budget fails the
803+
* attempt whole (decision A, #1997 #832): every worker stops pulling,
804+
* extract returns CBM_PIPELINE_ABORT_OVER_BUDGET, nothing is published
805+
* and the previously serving index keeps answering. */
761806
if (cbm_mem_budget() > 0) {
762807
bool over = cbm_mem_over_budget();
763808
bool futile = atomic_load_explicit(&ec->bp_futile, memory_order_relaxed) != 0;
764809
if (over && !futile) {
765-
cbm_mem_collect();
766-
atomic_fetch_add_explicit(&g_bp_nap_cycles, SKIP_ONE, memory_order_relaxed);
767-
int bp = 0;
768-
for (; bp < PP_BACKPRESSURE_MAX_SPINS && cbm_mem_over_budget() &&
769-
!atomic_load_explicit(ec->cancelled, memory_order_relaxed);
770-
bp++) {
771-
struct timespec nap = {0, PP_BACKPRESSURE_NAP_NS};
772-
cbm_nanosleep(&nap, NULL);
773-
}
774-
if (bp == PP_BACKPRESSURE_MAX_SPINS && cbm_mem_over_budget()) {
775-
/* Log only the 0→1 transition: all workers race into the
776-
* gate before anyone latches, so a plain store would WARN
777-
* once per worker (12 lines per latch event). */
778-
if (atomic_exchange_explicit(&ec->bp_futile, 1, memory_order_relaxed) == 0) {
779-
cbm_log_warn("mem.backpressure.futile", "action", "soft_overshoot");
810+
/* Act only on the 0→1 transition: all workers race into the
811+
* gate before anyone latches, so a plain store would confirm
812+
* (and WARN) once per worker. */
813+
if (pp_backpressure_cycle_still_over(ec) &&
814+
atomic_exchange_explicit(&ec->bp_futile, 1, memory_order_relaxed) == 0) {
815+
cbm_log_warn("mem.backpressure.futile", "action", "confirm");
816+
if (pp_backpressure_cycle_still_over(ec)) {
817+
pp_fail_whole_over_budget(ec);
818+
} else {
819+
atomic_store_explicit(&ec->bp_futile, 0, memory_order_relaxed);
780820
}
781821
}
782822
} else if (!over && futile) {
783823
atomic_store_explicit(&ec->bp_futile, 0, memory_order_relaxed);
784824
}
785825
}
826+
if (atomic_load_explicit(&ec->over_budget_abort, memory_order_relaxed)) {
827+
break;
828+
}
786829

787830
int file_idx = ec->sorted[sort_pos].idx;
788831
const cbm_file_info_t *fi = &ec->files[file_idx];
@@ -1129,6 +1172,7 @@ int cbm_parallel_extract_ex(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *file
11291172
atomic_init(&ec.retain_cap_warned, 0);
11301173
atomic_init(&ec.oversized_warned, 0);
11311174
atomic_init(&ec.bp_futile, 0);
1175+
atomic_init(&ec.over_budget_abort, 0);
11321176

11331177
/* Sub-phase: Dispatch workers (parse + extract per file, PARALLEL) */
11341178
CBM_PROF_START(t_dispatch);
@@ -1176,6 +1220,12 @@ int cbm_parallel_extract_ex(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t *file
11761220
free(sorted);
11771221
cbm_macro_table_free(pp_macro_table); /* ObjectScript macro table (NULL-safe) */
11781222

1223+
/* The over-budget verdict outranks the cancel sentinel: a caller must be
1224+
* able to name the cause, and the orchestrator discards the staging DB on
1225+
* every non-zero code alike (the live generation is never touched). */
1226+
if (atomic_load(&ec.over_budget_abort)) {
1227+
return CBM_PIPELINE_ABORT_OVER_BUDGET;
1228+
}
11791229
if (atomic_load(ctx->cancelled)) {
11801230
return CBM_NOT_FOUND;
11811231
}

‎src/pipeline/pipeline.h‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -63,9 +63,14 @@ void cbm_pipeline_free(cbm_pipeline_t *p);
6363
* need to know whether the PREVIOUS generation survived can distinguish the
6464
* failures by value: the run publishes by renaming a fully validated staging
6565
* database over the destination, so every abort before that rename leaves the
66-
* existing database in place. Those codes (CBM_PIPELINE_ABORT_PRESERVE_DB and
67-
* CBM_PIPELINE_PERSIST_FAILED) are defined in pipeline_internal.h alongside the
68-
* stages that raise them. */
66+
* existing database in place. */
67+
#define CBM_PIPELINE_ABORT_PRESERVE_DB (-2) /* aborted pre-publication; previous DB intact */
68+
#define CBM_PIPELINE_PERSIST_FAILED (-4) /* staging/rollback failure during publication */
69+
/* Resident memory stayed above the budget after back-pressure and one
70+
* confirmation cycle: the run stops before publication, no partial graph is
71+
* written, the previous DB is intact (#1997 #832). Distinct from the cancel
72+
* sentinel so callers can name the cause instead of "pipeline failed". */
73+
#define CBM_PIPELINE_ABORT_OVER_BUDGET (-5)
6974
int cbm_pipeline_run(cbm_pipeline_t *p);
7075

7176
/* Request cancellation of a running pipeline (thread-safe). */

0 commit comments

Comments
 (0)