Skip to content

Commit 0d2da15

Browse files
author
Martin Vogel
committed
Bind tree-sitter runtime to mimalloc in prod build to fix C++ indexing crash (#424)
Root cause (corrected from the report): not a tree-sitter null-subtree sentinel — 0xffffffffffffffff is a freed/heap-corrupted pointer. The vendored ts runtime allocates via its overridable ts_current_malloc/free; under the prod build's MI_OVERRIDE=1, the Windows static-MinGW link (--allow-multiple-definition) resolves ts_malloc to mimalloc but ts_free can resolve to the CRT free (or vice-versa), corrupting the heap freelist and crashing mid-parse on large templated C++ headers (scales with parse churn, not syntax). cbm_init now calls ts_set_allocator(mi_malloc, mi_calloc, mi_realloc, mi_free) so ts allocate+free go through one allocator on every platform. Guarded to CFLAGS_PROD (MI_OVERRIDE=1); the test build stays CRT+ASan to avoid an alloc/free mismatch there. Tests in test_stack_overflow.c: allocator-binding mechanism + large-templated-C++ extraction guard (the C++ gap noted in #424).
1 parent 0314a21 commit 0d2da15

3 files changed

Lines changed: 116 additions & 5 deletions

File tree

‎Makefile.cbm‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,10 @@ CXXFLAGS_COMMON = -std=c++14 -Wall -Wextra -Werror \
5858
-I$(CBM_DIR) -I$(TS_INCLUDE)
5959

6060
# Production flags (CFLAGS_EXTRA allows CI to inject -DCBM_VERSION)
61-
CFLAGS_PROD = $(CFLAGS_COMMON) -O2 $(CFLAGS_EXTRA)
61+
# CBM_BIND_TS_ALLOCATOR=1: bind the tree-sitter runtime to mimalloc (#424). Only
62+
# the prod build uses mimalloc (MI_OVERRIDE=1); the test build is CRT+ASan, where
63+
# binding would create an alloc/free mismatch, so the guard is prod-only.
64+
CFLAGS_PROD = $(CFLAGS_COMMON) -O2 -DCBM_BIND_TS_ALLOCATOR=1 $(CFLAGS_EXTRA)
6265
CXXFLAGS_PROD = $(CXXFLAGS_COMMON) -O2
6366

6467
# Test flags: debug + sanitizers (override SANITIZE= to disable on Windows)

‎internal/cbm/cbm.c‎

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,8 @@
1616
#include "foundation/compat.h"
1717
#include "tree_sitter/api.h" // TSParser, TSNode, TSTree, TSInput, TSLanguage, TSPoint, TSParseOptions, TSParseState
1818
#include "foundation/constants.h"
19-
#include <stdint.h> // uint32_t, uint64_t, int64_t
19+
#include "mimalloc.h" // mi_malloc/mi_calloc/mi_realloc/mi_free — bind ts runtime allocator (#424)
20+
#include <stdint.h> // uint32_t, uint64_t, int64_t
2021
#include <stdlib.h>
2122
#include <string.h>
2223
#include <ctype.h>
@@ -241,6 +242,22 @@ int cbm_init(void) {
241242
}
242243
enum { CBM_INIT_DONE = 1 };
243244
cbm_initialized = CBM_INIT_DONE;
245+
/* Bind the vendored tree-sitter runtime to the SAME allocator the rest of
246+
* cbm uses (mimalloc). The runtime otherwise allocates through its
247+
* overridable ts_current_malloc/free defaults (plain malloc / free). Under
248+
* the production build's MI_OVERRIDE=1 — particularly the Windows
249+
* static-MinGW link with --allow-multiple-definition — `malloc` and `free`
250+
* can resolve to DIFFERENT allocators (mimalloc vs the CRT), so a subtree
251+
* allocated by mimalloc gets freed by the CRT (or vice-versa), corrupting
252+
* the heap freelist and crashing mid-parse on a freed (0xffff…) pointer for
253+
* large templated C++ headers (#424 — scales with parse churn, not syntax).
254+
* Binding explicitly forces ts allocate+free through one allocator on every
255+
* platform, eliminating the mismatch class generically. Guarded to the
256+
* production build (MI_OVERRIDE=1): the test build is CRT + ASan, where
257+
* binding ts to mimalloc would mismatch ASan/CRT frees. */
258+
#if defined(CBM_BIND_TS_ALLOCATOR) && CBM_BIND_TS_ALLOCATOR
259+
ts_set_allocator(mi_malloc, mi_calloc, mi_realloc, mi_free);
260+
#endif
244261
return 0;
245262
}
246263

‎tests/test_stack_overflow.c‎

Lines changed: 94 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,17 @@
1616
#include <stdlib.h>
1717
#include <string.h>
1818

19+
/* tree-sitter runtime allocator hooks (ts_runtime/src/alloc.h, TS_PUBLIC) and
20+
* mimalloc (vendored) — for the #424 allocator-binding regression test. */
21+
extern void *(*ts_current_malloc)(size_t);
22+
extern void *(*ts_current_calloc)(size_t, size_t);
23+
extern void *(*ts_current_realloc)(void *, size_t);
24+
extern void (*ts_current_free)(void *);
25+
extern void *mi_malloc(size_t);
26+
extern void *mi_calloc(size_t, size_t);
27+
extern void *mi_realloc(void *, size_t);
28+
extern void mi_free(void *);
29+
1930
/* ── Helpers ───────────────────────────────────────────────────── */
2031

2132
static CBMFileResult *extract(const char *src, CBMLanguage lang, const char *proj,
@@ -24,7 +35,6 @@ static CBMFileResult *extract(const char *src, CBMLanguage lang, const char *pro
2435
return r;
2536
}
2637

27-
2838
/* ═══════════════════════════════════════════════════════════════════
2939
* Test: JavaScript calls exceeding 512 stack cap
3040
*
@@ -167,7 +177,8 @@ TEST(express_routes_exceed_512) {
167177
p += snprintf(p, (size_t)(buf_sz - (size_t)(p - src)), "const express = require('express');\n");
168178
p += snprintf(p, (size_t)(buf_sz - (size_t)(p - src)), "const app = express();\n\n");
169179
for (int i = 0; i < ROUTE_COUNT; i++) {
170-
p += snprintf(p, (size_t)(buf_sz - (size_t)(p - src)), "app.get('/route_%d', handler_%d);\n", i, i);
180+
p += snprintf(p, (size_t)(buf_sz - (size_t)(p - src)),
181+
"app.get('/route_%d', handler_%d);\n", i, i);
171182
}
172183
p += snprintf(p, (size_t)(buf_sz - (size_t)(p - src)), "\napp.listen(3000);\n");
173184

@@ -205,7 +216,8 @@ TEST(ts_imports_exceed_512) {
205216

206217
char *p = src;
207218
for (int i = 0; i < IMPORT_COUNT; i++) {
208-
p += snprintf(p, (size_t)(buf_sz - (size_t)(p - src)), "import { mod_%d } from './module_%d';\n", i, i);
219+
p += snprintf(p, (size_t)(buf_sz - (size_t)(p - src)),
220+
"import { mod_%d } from './module_%d';\n", i, i);
209221
}
210222
p += snprintf(p, (size_t)(buf_sz - (size_t)(p - src)), "console.log('done');\n");
211223

@@ -301,13 +313,92 @@ TEST(yaml_vars_exceed_256) {
301313
PASS();
302314
}
303315

316+
/* ═══════════════════════════════════════════════════════════════════
317+
* Test: tree-sitter runtime allocator is bound to cbm's allocator (#424)
318+
*
319+
* cbm_init() must bind the vendored tree-sitter runtime to mimalloc via
320+
* ts_set_allocator(). Otherwise the runtime allocates through its overridable
321+
* ts_current_malloc/free defaults (plain malloc/free); under the production
322+
* MI_OVERRIDE=1 build (esp. Windows static-MinGW + --allow-multiple-definition)
323+
* ts_malloc and ts_free can resolve to different allocators, corrupting the
324+
* heap and crashing mid-parse on large templated C++ headers. Asserting the
325+
* binding reproduces the mismatch CONDITION deterministically on every
326+
* platform: RED before the fix (ts_current_free == plain free), GREEN after.
327+
* ═══════════════════════════════════════════════════════════════════ */
328+
329+
TEST(ts_allocator_bound_to_mimalloc_issue424) {
330+
cbm_init();
331+
#if defined(CBM_BIND_TS_ALLOCATOR) && CBM_BIND_TS_ALLOCATOR
332+
/* Production build: cbm_init must have bound the ts runtime to mimalloc so
333+
* ts_malloc and ts_free can never resolve to different allocators (#424). */
334+
ASSERT_TRUE(ts_current_malloc == mi_malloc);
335+
ASSERT_TRUE(ts_current_free == mi_free);
336+
#else
337+
/* Test build (MI_OVERRIDE=0, CRT + ASan): binding is intentionally OFF (it
338+
* would mismatch ASan/CRT frees). The crash only affects the prod binary;
339+
* its reproduction runs in the prod smoke job. Independently verify the fix
340+
* MECHANISM — ts_set_allocator wires the runtime hooks — then restore the
341+
* CRT defaults so the rest of this build stays allocator-consistent. */
342+
void *(*save_m)(size_t) = ts_current_malloc;
343+
void *(*save_c)(size_t, size_t) = ts_current_calloc;
344+
void *(*save_r)(void *, size_t) = ts_current_realloc;
345+
void (*save_f)(void *) = ts_current_free;
346+
ts_set_allocator(mi_malloc, mi_calloc, mi_realloc, mi_free);
347+
int wired = (ts_current_malloc == mi_malloc) && (ts_current_free == mi_free);
348+
ts_set_allocator(save_m, save_c, save_r, save_f); /* restore BEFORE asserting */
349+
ASSERT_TRUE(wired);
350+
ASSERT_TRUE(ts_current_free == save_f);
351+
#endif
352+
PASS();
353+
}
354+
355+
/* ═══════════════════════════════════════════════════════════════════
356+
* Test: large templated C++ header extraction does not crash (#424)
357+
*
358+
* Generates a json.hpp-style header (hundreds of templated structs with
359+
* operator overloads) — the parse path that SEGV'd in #424. Asserts
360+
* extraction completes. (In the MI_OVERRIDE=0 test build the allocator is
361+
* consistent so this cannot fault here; it guards extraction on large
362+
* templated input and addresses the C++ gap in this file. The production-
363+
* binary crash reproduction runs in the smoke job.)
364+
* ═══════════════════════════════════════════════════════════════════ */
365+
366+
TEST(cpp_large_templated_header_no_crash_issue424) {
367+
const int N = 400; /* ~400 templated structs → ~2400 lines */
368+
size_t buf_sz = (size_t)N * 600;
369+
char *src = malloc(buf_sz);
370+
ASSERT_NOT_NULL(src);
371+
char *p = src;
372+
p += snprintf(p, buf_sz, "#include <cstddef>\nnamespace ns {\n");
373+
for (int i = 0; i < N; i++) {
374+
p += snprintf(p, (size_t)(buf_sz - (size_t)(p - src)),
375+
"template <typename T> struct Box%d {\n"
376+
" T value;\n"
377+
" bool operator<(const Box%d &o) const { return value < o.value; }\n"
378+
" bool operator==(const Box%d &o) const { return value == o.value; }\n"
379+
" T get() const { return value; }\n"
380+
"};\n",
381+
i, i, i);
382+
}
383+
snprintf(p, (size_t)(buf_sz - (size_t)(p - src)), "}\n");
384+
385+
CBMFileResult *r = extract(src, CBM_LANG_CPP, "test", "templated.hpp");
386+
ASSERT_NOT_NULL(r); /* no crash is the real assertion */
387+
cbm_free_result(r);
388+
free(src);
389+
PASS();
390+
}
391+
304392
/* ═══════════════════════════════════════════════════════════════════
305393
* Suite registration
306394
* ═══════════════════════════════════════════════════════════════════ */
307395

308396
SUITE(stack_overflow) {
309397
cbm_init();
310398

399+
RUN_TEST(ts_allocator_bound_to_mimalloc_issue424);
400+
RUN_TEST(cpp_large_templated_header_no_crash_issue424);
401+
311402
RUN_TEST(js_calls_exceed_512);
312403
RUN_TEST(python_calls_exceed_512);
313404
RUN_TEST(go_calls_exceed_1024);

0 commit comments

Comments
 (0)