Skip to content

Commit 8aeee86

Browse files
committed
Merge origin/main into pr/windows-setenv-utf8
Signed-off-by: Martin Vogel <[email protected]>
2 parents 21d3550 + 1f934b6 commit 8aeee86

6 files changed

Lines changed: 589 additions & 22 deletions

File tree

‎internal/cbm/extract_url_fold.c‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -401,10 +401,22 @@ static bool is_interpolated_string(TSNode node) {
401401
return false;
402402
}
403403

404+
/* A Python string whose opening token is more than one quote -- a prefix
405+
* (r"", b"", u"", an f"" with no field) or a triple quote -- cannot go down
406+
* the literal path: that strips exactly one quote from each end, so the
407+
* prefix and the extra quotes stayed in the URL and it never matched a route
408+
* (#523). Folding reads the content pieces instead. JS/TS strings have no
409+
* string_start, so this is Python only. */
410+
static bool is_wrapped_string(TSNode node) {
411+
TSNode start = ts_node_child(node, 0);
412+
return !ts_node_is_null(start) && strcmp(ts_node_type(start), "string_start") == 0 &&
413+
ts_node_end_byte(start) - ts_node_start_byte(start) > SKIP_ONE;
414+
}
415+
404416
static bool is_foldable_arg(TSNode arg) {
405417
const char *k = ts_node_type(arg);
406418
if (strcmp(k, "string") == 0) {
407-
return is_interpolated_string(arg);
419+
return is_interpolated_string(arg) || is_wrapped_string(arg);
408420
}
409421
return strcmp(k, "template_string") == 0 || strcmp(k, "binary_expression") == 0 ||
410422
strcmp(k, "binary_operator") == 0 || strcmp(k, "identifier") == 0 ||

‎src/mcp/mcp.c‎

Lines changed: 138 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -4012,6 +4012,7 @@ enum {
40124012
BM25_BIND_EXACT = 8,
40134013
BM25_BIND_KIND = 9,
40144014
BM25_BIND_NS = 10,
4015+
BM25_BIND_LABEL_WEIGHT = 11,
40154016
BM25_SQL_AUTO_LEN = -1,
40164017
/* Inner FTS5 candidate cap. SQLite can early-terminate a plain FTS5 query
40174018
* (no JOIN/WHERE on outer table) of the form:
@@ -4020,6 +4021,11 @@ enum {
40204021
* and then joining/filtering/re-ranking those, we bound all work to O(N) where
40214022
* N = BM25_INNER_LIMIT rather than the full match set size. */
40224023
BM25_INNER_LIMIT = 2000,
4024+
/* Longest label spliced into the MATCH pre-filter; real labels are short
4025+
* identifiers, so anything longer simply skips the pre-filter. */
4026+
BM25_LABEL_MAX = 64,
4027+
/* Room for "(<tokens>) AND label : \"<label>\"" around the token list. */
4028+
BM25_MATCH_BUF = BM25_QUERY_BUF + BM25_LABEL_MAX + 32,
40234029
};
40244030

40254031
/* Column weights for nodes_fts (name, qualified_name, label, file_path, body).
@@ -4038,8 +4044,28 @@ enum {
40384044
* Safe against a legacy four-column nodes_fts: FTS5's bm25() reads a weight
40394045
* only when an instance actually lands in that column (`nVal > ic`), so the
40404046
* fifth weight is simply never consulted on a table that has no fifth
4041-
* column. */
4042-
#define BM25_WEIGHTS "bm25(nodes_fts, 1.0, 1.0, 1.0, 1.0, 0.3)"
4047+
* column.
4048+
*
4049+
* The label weight is bound (?11): 1.0 normally, 0.0 when the MATCH is scoped
4050+
* to a label (#2386). Every candidate then carries that label, so a label hit
4051+
* cannot tell them apart, and a zero weight keeps the added `label : "..."`
4052+
* phrase from shifting BM25 order by row length. */
4053+
#define BM25_WEIGHTS "bm25(nodes_fts, 1.0, 1.0, ?11, 1.0, 0.3)"
4054+
static const double BM25_LABEL_WEIGHT = 1.0;
4055+
static const double BM25_LABEL_WEIGHT_SCOPED = 0.0;
4056+
4057+
/* Labels an UNLABELLED query never ranks: path-shaped File/Folder nodes, the
4058+
* Project node and Variables match almost any token through their qualified
4059+
* name and file path, and would crowd definitions out of the result. They are
4060+
* still in nodes_fts, so an explicit `label` that names one of them is served
4061+
* from its own label-scoped window like any other label (?7 is the caller's
4062+
* label, NULL when none was given). Excluding them there too returned 0 rows
4063+
* -- often with an exact "eq" total -- although such nodes match. Section and
4064+
* Module are NOT excluded (#518/#519): they carry the prose the body column
4065+
* indexes. Shared by the ranked and the count query, so the two cannot
4066+
* desynchronise. */
4067+
#define BM25_UNLABELLED_EXCLUSION \
4068+
"(?7 IS NOT NULL OR n.label NOT IN ('File','Folder','Variable','Project'))"
40434069

40444070
/* Module-local SQLITE_TRANSIENT wrapper to dodge performance-no-int-to-ptr.
40454071
* See the matching helper in src/store/store.c for the same pattern. */
@@ -4090,6 +4116,85 @@ static int bm25_build_match(const char *query, char *out, size_t out_size) {
40904116
return tokens;
40914117
}
40924118

4119+
/* Scope an FTS5 MATCH expression to one node label (#2386). The label filter
4120+
* must choose the BM25_INNER_LIMIT candidate window, not trim it afterwards:
4121+
* otherwise a label that is rare among a common token's matches falls outside
4122+
* the window and the search returns 0 rows although matching nodes exist.
4123+
* FTS5 matching is case-insensitive, so this is a superset pre-filter; the
4124+
* caller keeps the exact `n.label = ?` check. Only a plain identifier is
4125+
* spliced in, as a quoted FTS5 string; any other label leaves `tokens`
4126+
* unscoped, which is the pre-#2386 behaviour. Returns the label-column weight
4127+
* to bind for the resulting MATCH. */
4128+
static double bm25_scope_match_to_label(const char *tokens, const char *label, char *out,
4129+
size_t out_size) {
4130+
snprintf(out, out_size, "%s", tokens);
4131+
if (!label || !label[0]) {
4132+
return BM25_LABEL_WEIGHT;
4133+
}
4134+
size_t len = 0;
4135+
for (; label[len]; len++) {
4136+
char ch = label[len];
4137+
bool ident = (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') ||
4138+
(ch >= '0' && ch <= '9') || ch == '_';
4139+
if (!ident || len >= BM25_LABEL_MAX) {
4140+
return BM25_LABEL_WEIGHT;
4141+
}
4142+
}
4143+
char scoped[BM25_MATCH_BUF];
4144+
int n = snprintf(scoped, sizeof(scoped), "(%s) AND label : \"%s\"", tokens, label);
4145+
if (n <= 0 || (size_t)n >= sizeof(scoped) || (size_t)n >= out_size) {
4146+
return BM25_LABEL_WEIGHT;
4147+
}
4148+
memcpy(out, scoped, (size_t)n + 1);
4149+
return BM25_LABEL_WEIGHT_SCOPED;
4150+
}
4151+
4152+
/* A BM25 result is empty although nodes match when the label is spelled in a
4153+
* case no node carries: labels are case-sensitive, so `label=class` finds
4154+
* nothing while Class nodes match the query. Returns a hint (memory core,
4155+
* CBM_MEM_CLASS_OTHER) naming the label this project actually holds, or NULL
4156+
* when the label exists exactly (a true zero). The exact probe is an index
4157+
* seek on (project, label); the case-insensitive scan runs only when it
4158+
* misses. The caller's label is never echoed back, only the stored spelling. */
4159+
static char *bm25_label_zero_hint(sqlite3 *db, const char *project, const char *label) {
4160+
enum { HINT_BIND_PROJECT = 1, HINT_BIND_LABEL = 2, HINT_COL_LABEL = 0, HINT_PROBE_EXACT = 0 };
4161+
if (!db || !label || !label[0]) {
4162+
return NULL;
4163+
}
4164+
static const char *const probes[] = {
4165+
"SELECT label FROM nodes WHERE project = ?1 AND label = ?2 LIMIT 1",
4166+
"SELECT label FROM nodes WHERE project = ?1 AND label = ?2 COLLATE NOCASE LIMIT 1",
4167+
};
4168+
char found[CBM_SZ_256] = "";
4169+
for (size_t i = 0; i < sizeof(probes) / sizeof(probes[0]) && !found[0]; i++) {
4170+
sqlite3_stmt *st = NULL;
4171+
if (sqlite3_prepare_v2(db, probes[i], BM25_SQL_AUTO_LEN, &st, NULL) != SQLITE_OK) {
4172+
return NULL;
4173+
}
4174+
sqlite3_bind_text(st, HINT_BIND_PROJECT, project, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4175+
sqlite3_bind_text(st, HINT_BIND_LABEL, label, BM25_SQL_AUTO_LEN, MCP_SQLITE_TRANSIENT);
4176+
if (sqlite3_step(st) == SQLITE_ROW) {
4177+
const char *stored = (const char *)sqlite3_column_text(st, HINT_COL_LABEL);
4178+
snprintf(found, sizeof(found), "%s", stored ? stored : "");
4179+
}
4180+
sqlite3_finalize(st);
4181+
if (found[0] && i == HINT_PROBE_EXACT) {
4182+
return NULL;
4183+
}
4184+
}
4185+
if (!found[0]) {
4186+
return cbm_mem_strdup(CBM_MEM_CLASS_OTHER,
4187+
"no node in this project carries this label; get_graph_schema "
4188+
"lists the labels this index holds");
4189+
}
4190+
char hint[CBM_SZ_512];
4191+
snprintf(hint, sizeof(hint),
4192+
"no node carries this label as spelled; labels are case-sensitive -- retry with "
4193+
"label=\"%s\"",
4194+
found);
4195+
return cbm_mem_strdup(CBM_MEM_CLASS_OTHER, hint);
4196+
}
4197+
40934198
static char *bm25_file_pattern_like(const char *file_pattern) {
40944199
if (!file_pattern) {
40954200
return NULL;
@@ -4141,7 +4246,8 @@ static void bm25_lines_str(char *out, size_t size, int start, int end) {
41414246

41424247
static void bm25_emit_metadata_tree(cbm_sb_t *sb, int total, int offset, int returned,
41434248
bool candidate_window_saturated, bool budget_hit,
4144-
bool floor_exceeded, size_t max_output_bytes) {
4249+
bool floor_exceeded, size_t max_output_bytes,
4250+
const char *zero_hint) {
41454251
bool has_more = total > offset + returned;
41464252
cbm_tree_scalar_int(sb, "total", total);
41474253
cbm_tree_scalar_str(sb, "total_relation", candidate_window_saturated ? "gte" : "eq");
@@ -4172,12 +4278,15 @@ static void bm25_emit_metadata_tree(cbm_sb_t *sb, int total, int offset, int ret
41724278
cbm_tree_scalar_str(sb, "hint",
41734279
"first identity row exceeds max_output_bytes; raise "
41744280
"max_output_tokens");
4281+
} else if (zero_hint) {
4282+
cbm_tree_scalar_str(sb, "hint", zero_hint);
41754283
}
41764284
}
41774285

41784286
static void bm25_emit_metadata_json(yyjson_mut_doc *doc, yyjson_mut_val *root, int total,
41794287
int offset, int returned, bool candidate_window_saturated,
4180-
bool budget_hit, bool floor_exceeded, size_t max_output_bytes) {
4288+
bool budget_hit, bool floor_exceeded, size_t max_output_bytes,
4289+
const char *zero_hint) {
41814290
bool has_more = total > offset + returned;
41824291
yyjson_mut_obj_add_int(doc, root, "total", total);
41834292
yyjson_mut_obj_add_str(doc, root, "total_relation", candidate_window_saturated ? "gte" : "eq");
@@ -4209,12 +4318,14 @@ static void bm25_emit_metadata_json(yyjson_mut_doc *doc, yyjson_mut_val *root, i
42094318
yyjson_mut_obj_add_str(doc, root, "hint",
42104319
"first identity row exceeds max_output_bytes; raise "
42114320
"max_output_tokens");
4321+
} else if (zero_hint) {
4322+
yyjson_mut_obj_add_strcpy(doc, root, "hint", zero_hint);
42124323
}
42134324
}
42144325

42154326
static char *bm25_render(const bm25_output_row_t *rows, int returned, int total, int offset,
42164327
bool candidate_window_saturated, bool tree_format, bool budget_hit,
4217-
bool floor_exceeded, size_t max_output_bytes) {
4328+
bool floor_exceeded, size_t max_output_bytes, const char *zero_hint) {
42184329
if (tree_format) {
42194330
cbm_sb_t sb;
42204331
cbm_sb_init(&sb);
@@ -4232,7 +4343,7 @@ static char *bm25_render(const bm25_output_row_t *rows, int returned, int total,
42324343
cbm_tree_row_end(&sb);
42334344
}
42344345
bm25_emit_metadata_tree(&sb, total, offset, returned, candidate_window_saturated,
4235-
budget_hit, floor_exceeded, max_output_bytes);
4346+
budget_hit, floor_exceeded, max_output_bytes, zero_hint);
42364347
return cbm_sb_finish(&sb);
42374348
}
42384349

@@ -4259,7 +4370,7 @@ static char *bm25_render(const bm25_output_row_t *rows, int returned, int total,
42594370
}
42604371
yyjson_mut_obj_add_val(doc, root, "rows", result_rows);
42614372
bm25_emit_metadata_json(doc, root, total, offset, returned, candidate_window_saturated,
4262-
budget_hit, floor_exceeded, max_output_bytes);
4373+
budget_hit, floor_exceeded, max_output_bytes, zero_hint);
42634374
char *json = yy_doc_to_str(doc);
42644375
yyjson_mut_doc_free(doc);
42654376
return json;
@@ -4276,11 +4387,16 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu
42764387
if (!db) {
42774388
return NULL;
42784389
}
4279-
char fts_query[BM25_QUERY_BUF];
4280-
int tok_count = bm25_build_match(query, fts_query, sizeof(fts_query));
4390+
char fts_tokens[BM25_QUERY_BUF];
4391+
int tok_count = bm25_build_match(query, fts_tokens, sizeof(fts_tokens));
42814392
if (tok_count == 0) {
42824393
return NULL;
42834394
}
4395+
/* One MATCH string for the ranked query, the count and the saturation
4396+
* probe, so all three describe the same label-scoped window. */
4397+
char fts_query[BM25_MATCH_BUF];
4398+
double label_weight =
4399+
bm25_scope_match_to_label(fts_tokens, label, fts_query, sizeof(fts_query));
42844400
char *file_like = bm25_file_pattern_like(file_pattern);
42854401

42864402
/* BM25 ranked query using a two-step approach to enable FTS5 early termination.
@@ -4323,12 +4439,7 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu
43234439
") fts "
43244440
"JOIN nodes n ON n.id = fts.rowid "
43254441
"WHERE n.project = ?2 "
4326-
/* Section and Module are NO LONGER excluded (#518/#519): they are the
4327-
* labels that carry prose — a Markdown section's body, a config file's
4328-
* description — so excluding them made the body column unreachable.
4329-
* This exclusion list is MIRRORED in the count query below; the two
4330-
* must be changed together or results desynchronise from counts. */
4331-
" AND n.label NOT IN ('File','Folder','Variable','Project') "
4442+
" AND " BM25_UNLABELLED_EXCLUSION " "
43324443
" AND (?6 IS NULL OR n.file_path LIKE ?6) "
43334444
/* The caller's label filter applies in query mode exactly as it does
43344445
* in the structural mode (2026-09-16 probe: `label=Class` was ignored
@@ -4378,6 +4489,7 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu
43784489
} else {
43794490
sqlite3_bind_null(stmt, BM25_BIND_NS);
43804491
}
4492+
sqlite3_bind_double(stmt, BM25_BIND_LABEL_WEIGHT, label_weight);
43814493

43824494
/* Count hits within the same inner-limit window — capped at BM25_INNER_LIMIT.
43834495
* Uses the identical subquery structure so the FTS5 early-exit applies here too. */
@@ -4394,7 +4506,7 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu
43944506
/* MIRRORS the ranked query's filter verbatim — same weights, same
43954507
* label exclusions. Changing one alone reports a total that does
43964508
* not describe the rows returned. */
4397-
" AND n.label NOT IN ('File','Folder','Variable','Project')"
4509+
" AND " BM25_UNLABELLED_EXCLUSION " "
43984510
" AND (?6 IS NULL OR n.file_path LIKE ?6)"
43994511
" AND (?7 IS NULL OR n.label = ?7)"
44004512
" AND (?9 IS NULL OR (CASE WHEN json_valid(n.properties) "
@@ -4411,6 +4523,7 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu
44114523
sqlite3_bind_text(cs, BM25_BIND_PROJECT, project, BM25_SQL_AUTO_LEN,
44124524
MCP_SQLITE_TRANSIENT);
44134525
sqlite3_bind_int(cs, BM25_BIND_LIMIT, BM25_INNER_LIMIT);
4526+
sqlite3_bind_double(cs, BM25_BIND_LABEL_WEIGHT, label_weight);
44144527
if (file_like) {
44154528
sqlite3_bind_text(cs, BM25_BIND_FILE, file_like, BM25_SQL_AUTO_LEN,
44164529
MCP_SQLITE_TRANSIENT);
@@ -4443,9 +4556,10 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu
44434556

44444557
/* The top-candidate window is a performance ceiling, not an exact-total
44454558
* boundary. Probe one candidate beyond it so a broad query never presents
4446-
* a window-local count as the complete match count. This is global to the
4447-
* FTS table; saturation is therefore conservatively reported even when
4448-
* later project/path filters might discard the hidden candidates. */
4559+
* a window-local count as the complete match count. The probe shares the
4560+
* label-scoped MATCH but is otherwise global to the FTS table; saturation
4561+
* is therefore conservatively reported even when later project/path
4562+
* filters might discard the hidden candidates. */
44494563
bool candidate_window_saturated = true;
44504564
{
44514565
const char *probe_sql = "SELECT rowid FROM nodes_fts WHERE nodes_fts MATCH ?1 "
@@ -4483,13 +4597,15 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu
44834597
if (!rows) {
44844598
return heap_strdup("out of memory");
44854599
}
4600+
/* Never a silent zero for a label the index spells differently. */
4601+
char *zero_hint = total == 0 ? bm25_label_zero_hint(db, project, label) : NULL;
44864602

44874603
int returned = row_count;
44884604
bool budget_hit = false;
44894605
char *payload = NULL;
44904606
for (;;) {
44914607
payload = bm25_render(rows, returned, total, offset, candidate_window_saturated,
4492-
tree_format, budget_hit, false, max_output_bytes);
4608+
tree_format, budget_hit, false, max_output_bytes, zero_hint);
44934609
if (!payload || strlen(payload) <= max_output_bytes || returned == 0) {
44944610
break;
44954611
}
@@ -4502,8 +4618,9 @@ static char *bm25_search(cbm_store_t *store, const char *project, const char *qu
45024618
if (!payload || strlen(payload) > max_output_bytes || (row_count > 0 && returned == 0)) {
45034619
free(payload);
45044620
payload = bm25_render(rows, 0, total, offset, candidate_window_saturated, tree_format, true,
4505-
row_count > 0, max_output_bytes);
4621+
row_count > 0, max_output_bytes, zero_hint);
45064622
}
4623+
cbm_free(CBM_MEM_CLASS_OTHER, zero_hint);
45074624
bm25_output_rows_free(rows, row_count);
45084625
return payload;
45094626
}

‎src/pipeline/pipeline_incremental.c‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3719,6 +3719,17 @@ static int run_closure_delta(cbm_pipeline_t *p, const char *db_path, const char
37193719
cbm_store_count_edges(staging, project));
37203720
cbm_log_info("delta.committed_counts", "elapsed_ms", itoa_buf((int)elapsed_ms(t)));
37213721

3722+
/* The staging file started as a byte-for-byte clone of the previous
3723+
* generation (cbm_delta_stage_clone above), so its `projects` row still
3724+
* carries whichever root_path/indexed_at the LAST full rebuild wrote.
3725+
* Every other write in this function patches nodes/edges/hashes/surfaces
3726+
* for the CURRENT run's repo_path, so the project row must be refreshed
3727+
* to match, or a later root_path-keyed read (get_code_snippet and
3728+
* friends) resolves files against a directory this run never touched. */
3729+
if (cbm_store_upsert_project(staging, project, cbm_pipeline_repo_path(p)) != CBM_STORE_OK) {
3730+
goto out;
3731+
}
3732+
37223733
{
37233734
int index_mode = cbm_pipeline_get_mode(p);
37243735
cbm_pipeline_generation_t generation = {

0 commit comments

Comments
 (0)