Skip to content
This repository was archived by the owner on Jul 13, 2026. It is now read-only.

Commit fedfc3f

Browse files
Dashboard: fix jsonc UTF-8/string corruption (cortexkit#5) and guard memory_embeddings reads (cortexkit#3)
cortexkit#5 (jsonc.rs): two corruption bugs in strip_jsonc. - Byte iteration with `byte as char` mojibakes multi-byte UTF-8 (non-ASCII paths, emoji, CJK) by splitting code points into Latin-1 bytes. Iterate `char`s. - The trailing-comma removal ran as a SECOND pass over the comment-stripped output with no string-state awareness, so a `,}` / `,]` sequence inside a string value got stripped. Folded it into the single string-aware loop so it only fires outside strings. +tests for both (string with ',}' / ',]', and non-ASCII path/emoji/CJK). cortexkit#3 (db.rs): get_memories and get_memory_stats read memory_embeddings (EXISTS subquery / COUNT) with no table_exists guard, unlike the classify-column degradation. A pre-vN / never-initialized DB (the read-only dashboard never creates tables) would error the whole query. Guard both: report no embeddings when the table is absent. +regression test. Dashboard: cargo 178 tests pass, frontend build clean. Co-authored-by: Alfonso [Magic Context] <[email protected]>
1 parent ffcc52a commit fedfc3f

2 files changed

Lines changed: 87 additions & 30 deletions

File tree

‎packages/dashboard/src-tauri/src/db.rs‎

Lines changed: 42 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2350,6 +2350,16 @@ pub fn get_memories(
23502350
"50 AS importance, 'project' AS scope, 0 AS shareable,"
23512351
};
23522352

2353+
// Degrade gracefully on a pre-vN / never-initialized DB that has no
2354+
// `memory_embeddings` table: the EXISTS subquery would error the whole query
2355+
// (the dashboard opens DBs read-only and never creates tables). Mirror the
2356+
// classify-column degradation and report no embeddings instead of crashing.
2357+
let has_embedding_select = if table_exists(conn, "memory_embeddings") {
2358+
HAS_EMBEDDING_SELECT
2359+
} else {
2360+
"0 AS has_embedding"
2361+
};
2362+
23532363
let raw_search = search_query.unwrap_or("").trim().to_string();
23542364
let has_search = !raw_search.is_empty();
23552365

@@ -2450,7 +2460,7 @@ pub fn get_memories(
24502460
where_extra,
24512461
limit_idx,
24522462
offset_idx,
2453-
has_embedding = HAS_EMBEDDING_SELECT,
2463+
has_embedding = has_embedding_select,
24542464
)
24552465
} else if has_search {
24562466
// LIKE fallback for short queries or special-character-heavy input
@@ -2470,7 +2480,7 @@ pub fn get_memories(
24702480
where_extra,
24712481
limit_idx,
24722482
offset_idx,
2473-
has_embedding = HAS_EMBEDDING_SELECT,
2483+
has_embedding = has_embedding_select,
24742484
)
24752485
} else {
24762486
format!(
@@ -2489,7 +2499,7 @@ pub fn get_memories(
24892499
where_extra,
24902500
limit_idx,
24912501
offset_idx,
2492-
has_embedding = HAS_EMBEDDING_SELECT,
2502+
has_embedding = has_embedding_select,
24932503
)
24942504
};
24952505

@@ -2525,7 +2535,7 @@ pub fn get_memories(
25252535
where_extra,
25262536
limit_idx,
25272537
offset_idx,
2528-
has_embedding = HAS_EMBEDDING_SELECT,
2538+
has_embedding = has_embedding_select,
25292539
);
25302540
// Rebuild params with LIKE pattern instead of FTS query.
25312541
// Project filter uses the same resolved_paths set computed at the
@@ -2573,7 +2583,7 @@ pub fn get_memories(
25732583
where_extra,
25742584
limit_idx,
25752585
offset_idx,
2576-
has_embedding = HAS_EMBEDDING_SELECT,
2586+
has_embedding = has_embedding_select,
25772587
);
25782588
let mut like_params: Vec<Box<dyn rusqlite::types::ToSql>> = Vec::new();
25792589
like_params.push(Box::new(like_pattern));
@@ -2805,8 +2815,12 @@ pub fn get_memory_stats(
28052815
)?;
28062816

28072817
// Count distinct memories that have at least one embedding row. Multiple rows per
2808-
// memory (one per model_id) must not inflate this stat.
2809-
let with_embeddings: i64 = if has_filter {
2818+
// memory (one per model_id) must not inflate this stat. Guard against a
2819+
// pre-vN/never-initialized DB with no memory_embeddings table (read-only
2820+
// dashboard never creates it) — report 0 rather than crashing the stats query.
2821+
let with_embeddings: i64 = if !table_exists(conn, "memory_embeddings") {
2822+
0
2823+
} else if has_filter {
28102824
if resolved_paths.is_empty() {
28112825
// Empty filter resolves to zero rows — honor the same zero-stats
28122826
// contract the other counts get from the `0 = 1` sentinel. Rebuilding
@@ -6702,6 +6716,27 @@ mod memory_project_filter_tests {
67026716
assert!(!rows[0].shareable);
67036717
}
67046718

6719+
#[test]
6720+
fn get_memories_degrades_without_memory_embeddings_table() {
6721+
// A pre-vN / never-initialized DB may have `memories` but no
6722+
// `memory_embeddings` table (the read-only dashboard never creates it).
6723+
// get_memories and the stats path must report no embeddings rather than
6724+
// erroring on the missing table.
6725+
let conn = make_memory_db();
6726+
conn.execute_batch("DROP TABLE memory_embeddings;")
6727+
.expect("drop memory_embeddings");
6728+
assert!(!table_exists(&conn, "memory_embeddings"));
6729+
insert_memory(&conn, "/tmp/no-emb", "CONSTRAINTS", "active");
6730+
6731+
let rows = get_memories(&conn, None, None, None, None, None, 100, 0)
6732+
.expect("get_memories must not error when memory_embeddings is absent");
6733+
assert_eq!(rows.len(), 1);
6734+
assert!(!rows[0].has_embedding, "no embeddings table → has_embedding=false");
6735+
6736+
let stats = get_memory_stats(&conn, None, None).expect("stats must not error");
6737+
assert_eq!(stats.with_embeddings, 0);
6738+
}
6739+
67056740
#[test]
67066741
fn get_memories_reads_classify_columns_on_v44_schema() {
67076742
let conn = make_memory_db();

‎packages/dashboard/src-tauri/src/jsonc.rs‎

Lines changed: 45 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -10,14 +10,22 @@ use std::path::Path;
1010

1111
/// Strip JSONC line/block comments AND trailing commas, respecting string
1212
/// literals, so the result parses as plain JSON.
13+
///
14+
/// Single pass over `char`s (NOT bytes): iterating bytes and casting
15+
/// `byte as char` corrupts any multi-byte UTF-8 (non-ASCII paths, CJK, accents)
16+
/// by splitting each code point into Latin-1 bytes. The trailing-comma removal
17+
/// is folded INTO the same string-state-aware loop so a comma inside a string
18+
/// value (e.g. `"a,}b"`) is never mistaken for a trailing `,}` / `,]` and
19+
/// stripped — the previous second-pass operated on the comment-stripped output
20+
/// without string awareness and corrupted such values.
1321
pub fn strip_jsonc(input: &str) -> String {
14-
let bytes = input.as_bytes();
22+
let chars: Vec<char> = input.chars().collect();
1523
let mut out = String::with_capacity(input.len());
1624
let mut i = 0usize;
1725
let mut in_string = false;
1826
let mut escaped = false;
19-
while i < bytes.len() {
20-
let c = bytes[i] as char;
27+
while i < chars.len() {
28+
let c = chars[i];
2129
if in_string {
2230
out.push(c);
2331
if escaped {
@@ -36,51 +44,44 @@ pub fn strip_jsonc(input: &str) -> String {
3644
i += 1;
3745
continue;
3846
}
39-
let next = if i + 1 < bytes.len() {
40-
bytes[i + 1] as char
47+
let next = if i + 1 < chars.len() {
48+
chars[i + 1]
4149
} else {
4250
'\0'
4351
};
4452
if c == '/' && next == '/' {
45-
while i < bytes.len() && bytes[i] as char != '\n' {
53+
while i < chars.len() && chars[i] != '\n' {
4654
i += 1;
4755
}
4856
continue;
4957
}
5058
if c == '/' && next == '*' {
5159
i += 2;
52-
while i + 1 < bytes.len() {
53-
if bytes[i] as char == '*' && bytes[i + 1] as char == '/' {
60+
while i + 1 < chars.len() {
61+
if chars[i] == '*' && chars[i + 1] == '/' {
5462
break;
5563
}
5664
i += 1;
5765
}
5866
i += 2;
5967
continue;
6068
}
61-
out.push(c);
62-
i += 1;
63-
}
64-
65-
// Drop trailing commas: `,}` / `,]` (optional whitespace between).
66-
let chars: Vec<char> = out.chars().collect();
67-
let mut cleaned = String::with_capacity(chars.len());
68-
let mut j = 0usize;
69-
while j < chars.len() {
70-
if chars[j] == ',' {
71-
let mut k = j + 1;
69+
// Trailing comma (outside strings only): a `,` whose next non-whitespace
70+
// char is `}` or `]`. Skip emitting it.
71+
if c == ',' {
72+
let mut k = i + 1;
7273
while k < chars.len() && chars[k].is_whitespace() {
7374
k += 1;
7475
}
7576
if k < chars.len() && (chars[k] == '}' || chars[k] == ']') {
76-
j += 1;
77+
i += 1;
7778
continue;
7879
}
7980
}
80-
cleaned.push(chars[j]);
81-
j += 1;
81+
out.push(c);
82+
i += 1;
8283
}
83-
cleaned
84+
out
8485
}
8586

8687
/// True when the config file at `path` exists AND declares a non-empty `dreamer`
@@ -165,6 +166,27 @@ mod tests {
165166
assert_eq!(parsed["note"], "a // b");
166167
}
167168

169+
#[test]
170+
fn does_not_strip_commas_inside_string_values() {
171+
// A `,}` / `,]` sequence INSIDE a string value must survive — the
172+
// trailing-comma pass must be string-state-aware.
173+
let input = "{ \"a\": \"x,}y\", \"b\": \"p,]q\", \"c\": [1, 2,] }";
174+
let parsed: serde_json::Value = serde_json::from_str(&strip_jsonc(input)).unwrap();
175+
assert_eq!(parsed["a"], "x,}y");
176+
assert_eq!(parsed["b"], "p,]q");
177+
assert_eq!(parsed["c"], serde_json::json!([1, 2]));
178+
}
179+
180+
#[test]
181+
fn preserves_non_ascii_utf8() {
182+
// Byte-casting (`byte as char`) would mojibake multi-byte code points.
183+
let input = "{ \"path\": \"/Users/José/café\", \"emoji\": \"🌙\", \"cjk\": \"日本語\", }";
184+
let parsed: serde_json::Value = serde_json::from_str(&strip_jsonc(input)).unwrap();
185+
assert_eq!(parsed["path"], "/Users/José/café");
186+
assert_eq!(parsed["emoji"], "🌙");
187+
assert_eq!(parsed["cjk"], "日本語");
188+
}
189+
168190
#[test]
169191
fn detects_dreamer_block_presence() {
170192
let dir = tempfile::tempdir().unwrap();

0 commit comments

Comments
 (0)