Skip to content

Commit de1819e

Browse files
committed
fix(python): preserve ordered namespace metadata
Capture ordered Python namespace facts and retain them through compact, spill, and versioned LSP surfaces. Reject unavailable metadata before incremental reuse and rebuild on Python namespace changes or deletion. Signed-off-by: Martin Vogel <[email protected]>
1 parent 78aea85 commit de1819e

11 files changed

Lines changed: 2284 additions & 18 deletions

File tree

‎internal/cbm/cbm.c‎

Lines changed: 218 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,190 @@ void cbm_reset_profile(void) {
114114
atomic_store(&total_files, 0);
115115
}
116116

117+
/* New facts use checked allocation and atomic publication; legacy extraction
118+
* arrays retain their existing contracts. */
119+
static bool py_namespace_nonempty(const char *s) {
120+
return s && s[0];
121+
}
122+
123+
static bool py_namespace_fact_valid(const CBMPyNamespaceFact *f) {
124+
if (!f || f->name_count < 0 || (size_t)f->name_count > SIZE_MAX / sizeof(*f->names) ||
125+
(f->name_count == 0 ? f->names != NULL : f->names == NULL))
126+
return false;
127+
bool literal = f->kind == CBM_PY_NS_ALL_SET || f->kind == CBM_PY_NS_ALL_APPEND;
128+
if (!literal && (f->names || f->name_count || f->sequence_kind != CBM_PY_NS_SEQUENCE_NONE))
129+
return false;
130+
if (f->kind != CBM_PY_NS_OWN_DEF && f->def_kind != CBM_PY_NS_DEF_NONE)
131+
return false;
132+
if (f->kind != CBM_PY_NS_IMPORT_MODULE && f->flags)
133+
return false;
134+
if (f->kind != CBM_PY_NS_IMPORT_NAME && f->kind != CBM_PY_NS_IMPORT_STAR && f->relative_level)
135+
return false;
136+
bool unknown = f->kind == CBM_PY_NS_SHADOW || f->kind == CBM_PY_NS_UNKNOWN ||
137+
f->kind == CBM_PY_NS_ALL_UNKNOWN;
138+
if (!unknown && f->reason != CBM_PY_NS_REASON_NONE)
139+
return false;
140+
switch (f->kind) {
141+
case CBM_PY_NS_OWN_DEF:
142+
return py_namespace_nonempty(f->local_name) && !f->module_name && !f->member_name &&
143+
(f->def_kind == CBM_PY_NS_DEF_FUNCTION || f->def_kind == CBM_PY_NS_DEF_CLASS);
144+
case CBM_PY_NS_IMPORT_MODULE:
145+
return py_namespace_nonempty(f->local_name) && py_namespace_nonempty(f->module_name) &&
146+
!f->member_name && !(f->flags & ~CBM_PY_NS_IMPORT_BINDS_ROOT);
147+
case CBM_PY_NS_IMPORT_NAME:
148+
case CBM_PY_NS_IMPORT_STAR:
149+
if (!f->module_name || (!f->module_name[0] && !f->relative_level))
150+
return false;
151+
return f->kind == CBM_PY_NS_IMPORT_NAME
152+
? py_namespace_nonempty(f->local_name) && py_namespace_nonempty(f->member_name)
153+
: !f->local_name && !f->member_name;
154+
case CBM_PY_NS_SHADOW:
155+
return py_namespace_nonempty(f->local_name) && !f->module_name && !f->member_name &&
156+
(f->reason == CBM_PY_NS_REASON_VALUE || f->reason == CBM_PY_NS_REASON_DECORATED ||
157+
f->reason == CBM_PY_NS_REASON_UNPROVEN_CLASS ||
158+
f->reason == CBM_PY_NS_REASON_UNSUPPORTED);
159+
case CBM_PY_NS_DELETE:
160+
return py_namespace_nonempty(f->local_name) && strcmp(f->local_name, "__all__") != 0 &&
161+
!f->module_name && !f->member_name;
162+
case CBM_PY_NS_UNKNOWN:
163+
return !f->local_name && !f->module_name && !f->member_name &&
164+
(f->reason == CBM_PY_NS_REASON_COMPOUND || f->reason == CBM_PY_NS_REASON_EFFECT ||
165+
f->reason == CBM_PY_NS_REASON_UNSUPPORTED || f->reason == CBM_PY_NS_REASON_PARSE);
166+
case CBM_PY_NS_ALL_UNKNOWN:
167+
return !f->local_name && !f->module_name && !f->member_name &&
168+
(f->reason == CBM_PY_NS_REASON_VALUE || f->reason == CBM_PY_NS_REASON_EFFECT ||
169+
f->reason == CBM_PY_NS_REASON_UNSUPPORTED || f->reason == CBM_PY_NS_REASON_PARSE);
170+
case CBM_PY_NS_ALL_SET:
171+
case CBM_PY_NS_ALL_APPEND:
172+
if (f->local_name || f->module_name || f->member_name ||
173+
(f->sequence_kind != CBM_PY_NS_SEQUENCE_LIST &&
174+
f->sequence_kind != CBM_PY_NS_SEQUENCE_TUPLE))
175+
return false;
176+
for (int i = 0; i < f->name_count; i++) {
177+
if (!f->names[i])
178+
return false;
179+
}
180+
return true;
181+
case CBM_PY_NS_ALL_DELETE:
182+
return !f->local_name && !f->module_name && !f->member_name;
183+
default:
184+
return false;
185+
}
186+
}
187+
188+
bool cbm_py_namespace_facts_valid(const CBMPyNamespaceFacts *f) {
189+
if (!f)
190+
return false;
191+
bool empty = !f->items && f->count == 0 && f->cap == 0;
192+
if (f->version == 0)
193+
return f->status == CBM_PY_NS_NOT_CAPTURED && f->failure == CBM_PY_NS_FAILURE_NONE && empty;
194+
if (f->version != CBM_PY_NAMESPACE_FACTS_VERSION || f->language < 0 ||
195+
f->language >= CBM_LANG_COUNT)
196+
return false;
197+
switch (f->status) {
198+
case CBM_PY_NS_NOT_CAPTURED:
199+
return empty && f->failure == CBM_PY_NS_FAILURE_NONE;
200+
case CBM_PY_NS_NOT_APPLICABLE:
201+
return empty && f->language != CBM_LANG_PYTHON && f->failure == CBM_PY_NS_FAILURE_NONE;
202+
case CBM_PY_NS_INCOMPLETE:
203+
return empty && f->failure >= CBM_PY_NS_FAILURE_INVALID_INPUT &&
204+
f->failure <= CBM_PY_NS_FAILURE_LIMIT;
205+
case CBM_PY_NS_COMPLETE:
206+
if (f->language != CBM_LANG_PYTHON || f->failure != CBM_PY_NS_FAILURE_NONE ||
207+
f->count < 0 || f->cap < f->count || (size_t)f->cap > SIZE_MAX / sizeof(*f->items) ||
208+
(f->count == 0 ? !empty : f->items == NULL))
209+
return false;
210+
for (int i = 0; i < f->count; i++) {
211+
if (!py_namespace_fact_valid(&f->items[i]))
212+
return false;
213+
}
214+
return true;
215+
default:
216+
return false;
217+
}
218+
}
219+
220+
#ifdef CBM_ENABLE_TEST_SEAMS
221+
static CBM_TLS int py_namespace_copy_left = -1;
222+
void cbm_py_namespace_test_copy_fail_after(int successful_allocations) {
223+
py_namespace_copy_left = successful_allocations;
224+
}
225+
#endif
226+
227+
static void *py_namespace_copy_alloc(CBMArena *arena, size_t size) {
228+
#ifdef CBM_ENABLE_TEST_SEAMS
229+
if (py_namespace_copy_left == 0)
230+
return NULL;
231+
if (py_namespace_copy_left > 0)
232+
py_namespace_copy_left--;
233+
#endif
234+
return cbm_arena_alloc(arena, size);
235+
}
236+
237+
static bool py_namespace_copy_string(CBMArena *arena, const char *source, const char **out) {
238+
*out = NULL;
239+
if (!source)
240+
return true;
241+
size_t len = strlen(source);
242+
if (len == SIZE_MAX)
243+
return false;
244+
char *copy = py_namespace_copy_alloc(arena, len + 1);
245+
if (!copy)
246+
return false;
247+
memcpy(copy, source, len + 1);
248+
*out = copy;
249+
return true;
250+
}
251+
252+
bool cbm_py_namespace_facts_copy(CBMArena *arena, const CBMPyNamespaceFacts *source,
253+
CBMPyNamespaceFacts *out) {
254+
if (!out || source == out)
255+
return false;
256+
CBMPyNamespaceFacts failure = {.version = CBM_PY_NAMESPACE_FACTS_VERSION,
257+
.language = CBM_LANG_PYTHON,
258+
.status = CBM_PY_NS_INCOMPLETE,
259+
.failure = CBM_PY_NS_FAILURE_INVALID_INPUT};
260+
if (!arena || !cbm_py_namespace_facts_valid(source)) {
261+
*out = failure;
262+
return false;
263+
}
264+
failure.language = source->language;
265+
failure.failure = CBM_PY_NS_FAILURE_ALLOCATION;
266+
CBMPyNamespaceFacts copy = *source;
267+
copy.items = NULL;
268+
copy.cap = copy.count;
269+
if (copy.count) {
270+
copy.items = py_namespace_copy_alloc(arena, (size_t)copy.count * sizeof(*copy.items));
271+
if (!copy.items)
272+
goto failed;
273+
memcpy(copy.items, source->items, (size_t)copy.count * sizeof(*copy.items));
274+
}
275+
for (int i = 0; i < copy.count; i++) {
276+
const CBMPyNamespaceFact *src = &source->items[i];
277+
CBMPyNamespaceFact *dst = &copy.items[i];
278+
if (!py_namespace_copy_string(arena, src->local_name, &dst->local_name) ||
279+
!py_namespace_copy_string(arena, src->module_name, &dst->module_name) ||
280+
!py_namespace_copy_string(arena, src->member_name, &dst->member_name))
281+
goto failed;
282+
dst->names = NULL;
283+
if (src->name_count) {
284+
dst->names =
285+
py_namespace_copy_alloc(arena, (size_t)src->name_count * sizeof(*dst->names));
286+
if (!dst->names)
287+
goto failed;
288+
for (int j = 0; j < src->name_count; j++) {
289+
if (!py_namespace_copy_string(arena, src->names[j], &dst->names[j]))
290+
goto failed;
291+
}
292+
}
293+
}
294+
*out = copy;
295+
return true;
296+
failed:
297+
*out = failure;
298+
return false;
299+
}
300+
117301
// --- Growable array push functions ---
118302

119303
#define GROW_ARRAY(arr, arena) \
@@ -3007,6 +3191,10 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C
30073191
return NULL;
30083192
}
30093193

3194+
result->py_namespace.version = CBM_PY_NAMESPACE_FACTS_VERSION;
3195+
result->py_namespace.language = language;
3196+
result->py_namespace.status =
3197+
language == CBM_LANG_PYTHON ? CBM_PY_NS_NOT_CAPTURED : CBM_PY_NS_NOT_APPLICABLE;
30103198
cbm_work_arena_take(&result->arena);
30113199
CBMArena *a = &result->arena;
30123200
if (!cbm_test_declarations_validate(result, test_declarations)) {
@@ -3252,6 +3440,7 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C
32523440
cbm_extract_definitions(&ctx);
32533441
cbm_test_declarations_finish(&ctx);
32543442
cbm_extract_imports(&ctx);
3443+
cbm_extract_python_namespace_facts(&ctx);
32553444
cbm_extract_unified(&ctx);
32563445
if (cbm_lang_needs_cpp_branch_views(language)) {
32573446
extract_cpp_branch_views(&ctx, ts_lang, timeout_micros);
@@ -4002,6 +4191,35 @@ CBMFileResult *cbm_extract_file_ex_with_tests(
40024191
CBMFileResult *result = extract_file_ex_body(
40034192
source, source_len, language, project, rel_path, timeout_micros, extra_defines,
40044193
include_paths, macro_table, return_type_table, &scratch, test_declarations);
4194+
/* Even early parse/quarantine exits publish conservative Python metadata.
4195+
* No tree is reparsed, and existing best-effort extraction flags stay intact. */
4196+
if (result && language == CBM_LANG_PYTHON &&
4197+
result->py_namespace.status == CBM_PY_NS_NOT_CAPTURED) {
4198+
CBMPyNamespaceFailure failure = CBM_PY_NS_FAILURE_NONE;
4199+
if (!result->module_qn || !result->module_qn[0]) {
4200+
if (!project || !project[0] || !rel_path || !rel_path[0]) {
4201+
failure = CBM_PY_NS_FAILURE_INVALID_INPUT;
4202+
} else {
4203+
result->module_qn =
4204+
cbm_fqn_module_source_lang(&result->arena, project, rel_path, language);
4205+
if (!result->module_qn || !result->module_qn[0])
4206+
failure = CBM_PY_NS_FAILURE_ALLOCATION;
4207+
}
4208+
}
4209+
if (failure) {
4210+
result->py_namespace = (CBMPyNamespaceFacts){.version = CBM_PY_NAMESPACE_FACTS_VERSION,
4211+
.language = language,
4212+
.status = CBM_PY_NS_INCOMPLETE,
4213+
.failure = failure};
4214+
} else {
4215+
CBMExtractCtx facts_ctx = {.arena = &result->arena,
4216+
.result = result,
4217+
.source = source,
4218+
.source_len = source_len,
4219+
.language = language};
4220+
cbm_extract_python_namespace_facts(&facts_ctx);
4221+
}
4222+
}
40054223
/* !tl_scratch_live: a nested extraction (an embedded language inside this
40064224
* file) may already have parked its own; never overwrite it. */
40074225
/* Kept up to CBM_EXTRACT_SCRATCH_KEEP_BYTES, grown blocks included: the

‎internal/cbm/cbm.h‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -389,6 +389,79 @@ typedef struct {
389389
CBMImportKind kind; // CBM_IMPORT_KIND_DEFAULT unless the syntax names a member kind
390390
} CBMImport;
391391

392+
/* Content-only ordered Python namespace facts; no resolved target identities. */
393+
#define CBM_PY_NAMESPACE_FACTS_VERSION 1u
394+
395+
typedef enum {
396+
CBM_PY_NS_NOT_CAPTURED = 0,
397+
CBM_PY_NS_NOT_APPLICABLE = 1,
398+
CBM_PY_NS_COMPLETE = 2,
399+
CBM_PY_NS_INCOMPLETE = 3
400+
} CBMPyNamespaceStatus;
401+
typedef enum {
402+
CBM_PY_NS_FAILURE_NONE = 0,
403+
CBM_PY_NS_FAILURE_INVALID_INPUT = 1,
404+
CBM_PY_NS_FAILURE_ALLOCATION = 2,
405+
CBM_PY_NS_FAILURE_LIMIT = 3
406+
} CBMPyNamespaceFailure;
407+
typedef enum {
408+
CBM_PY_NS_OWN_DEF = 1,
409+
CBM_PY_NS_IMPORT_MODULE = 2,
410+
CBM_PY_NS_IMPORT_NAME = 3,
411+
CBM_PY_NS_IMPORT_STAR = 4,
412+
CBM_PY_NS_SHADOW = 5,
413+
CBM_PY_NS_DELETE = 6,
414+
CBM_PY_NS_UNKNOWN = 7,
415+
CBM_PY_NS_ALL_SET = 8,
416+
CBM_PY_NS_ALL_APPEND = 9,
417+
CBM_PY_NS_ALL_UNKNOWN = 10,
418+
CBM_PY_NS_ALL_DELETE = 11
419+
} CBMPyNamespaceFactKind;
420+
typedef enum {
421+
CBM_PY_NS_DEF_NONE = 0,
422+
CBM_PY_NS_DEF_FUNCTION = 1,
423+
CBM_PY_NS_DEF_CLASS = 2
424+
} CBMPyNamespaceDefKind;
425+
typedef enum {
426+
CBM_PY_NS_SEQUENCE_NONE = 0,
427+
CBM_PY_NS_SEQUENCE_LIST = 1,
428+
CBM_PY_NS_SEQUENCE_TUPLE = 2
429+
} CBMPyNamespaceSequenceKind;
430+
typedef enum {
431+
CBM_PY_NS_REASON_NONE = 0,
432+
CBM_PY_NS_REASON_VALUE = 1,
433+
CBM_PY_NS_REASON_DECORATED = 2,
434+
CBM_PY_NS_REASON_UNPROVEN_CLASS = 3,
435+
CBM_PY_NS_REASON_COMPOUND = 4,
436+
CBM_PY_NS_REASON_EFFECT = 5,
437+
CBM_PY_NS_REASON_UNSUPPORTED = 6,
438+
CBM_PY_NS_REASON_PARSE = 7
439+
} CBMPyNamespaceUnknownReason;
440+
enum { CBM_PY_NS_IMPORT_BINDS_ROOT = 1u << 0 };
441+
442+
typedef struct {
443+
CBMPyNamespaceFactKind kind;
444+
CBMPyNamespaceDefKind def_kind;
445+
CBMPyNamespaceSequenceKind sequence_kind;
446+
CBMPyNamespaceUnknownReason reason;
447+
uint32_t flags;
448+
uint32_t relative_level;
449+
const char *local_name;
450+
const char *module_name;
451+
const char *member_name;
452+
const char **names;
453+
int name_count;
454+
} CBMPyNamespaceFact;
455+
typedef struct {
456+
uint32_t version;
457+
CBMLanguage language;
458+
CBMPyNamespaceStatus status;
459+
CBMPyNamespaceFailure failure;
460+
CBMPyNamespaceFact *items;
461+
int count;
462+
int cap;
463+
} CBMPyNamespaceFacts;
464+
392465
typedef enum {
393466
CBM_USAGE_VALUE = 0,
394467
CBM_USAGE_CALL_REFERENCE,
@@ -728,6 +801,7 @@ typedef struct CBMFileResult {
728801
CBMDefArray defs;
729802
CBMCallArray calls;
730803
CBMImportArray imports;
804+
CBMPyNamespaceFacts py_namespace;
731805
CBMUsageArray usages;
732806
CBMThrowArray throws;
733807
CBMRWArray rw;
@@ -1186,6 +1260,15 @@ int cbm_macro_extraction_enabled(void);
11861260

11871261
// --- Internal helpers used by extractors ---
11881262

1263+
CBMPyNamespaceStatus cbm_extract_python_namespace_facts(CBMExtractCtx *ctx);
1264+
bool cbm_py_namespace_facts_valid(const CBMPyNamespaceFacts *facts);
1265+
bool cbm_py_namespace_facts_copy(CBMArena *arena, const CBMPyNamespaceFacts *source,
1266+
CBMPyNamespaceFacts *out);
1267+
#ifdef CBM_ENABLE_TEST_SEAMS
1268+
void cbm_py_namespace_test_capture_fail_after(int successful_allocations);
1269+
void cbm_py_namespace_test_copy_fail_after(int successful_allocations);
1270+
#endif
1271+
11891272
// Growable array push functions (arena-allocated, no individual free needed).
11901273
void cbm_defs_push(CBMDefArray *arr, CBMArena *a, CBMDefinition def);
11911274
void cbm_calls_push(CBMCallArray *arr, CBMArena *a, CBMCall call);

0 commit comments

Comments
 (0)