Skip to content

Commit 1346ac4

Browse files
authored
Merge pull request #2686 from DeusData/fix/issue-1548-python-nested-defs-v2
fix(python): extract defs nested in a function body and resolve bare names lexically (#1548)
2 parents 08ee95a + ef90708 commit 1346ac4

17 files changed

Lines changed: 1836 additions & 65 deletions

‎internal/cbm/extract_defs.c‎

Lines changed: 65 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6624,11 +6624,16 @@ static void extract_func_def(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec
66246624
* scope. Note the call-scope side (compute_func_qn in extract_unified.c) is
66256625
* NOT language-gated — it qualifies whenever a scope is pushed — so this gate
66266626
* and the scope-push rule must move together, or a call QN names a def QN that
6627-
* was never minted and the edge is dropped at write. */
6627+
* was never minted and the edge is dropped at write.
6628+
*
6629+
* Python: the only enclosing scope that reaches here is the def a nested
6630+
* def sits in (py_push_def_body), so `def index()` inside create_app is
6631+
* proj.mod.create_app.index. */
66286632
if (ctx->enclosing_class_qn &&
66296633
(ctx->language == CBM_LANG_CPP || ctx->language == CBM_LANG_CUDA ||
66306634
ctx->language == CBM_LANG_TYPESCRIPT || ctx->language == CBM_LANG_TSX ||
6631-
ctx->language == CBM_LANG_ARKTS || ctx->language == CBM_LANG_NIX)) {
6635+
ctx->language == CBM_LANG_ARKTS || ctx->language == CBM_LANG_NIX ||
6636+
ctx->language == CBM_LANG_PYTHON)) {
66326637
def.qualified_name = cbm_arena_sprintf(a, "%s.%s", ctx->enclosing_class_qn, qn_name);
66336638
}
66346639
def.label = "Function";
@@ -11667,6 +11672,51 @@ static void push_class_body_children(TSNode node, const CBMLangSpec *spec, wd_st
1166711672
}
1166811673
}
1166911674

11675+
/* Python: a def's body is a scope of its own, and a `def` or `class` statement
11676+
* in it is a definition like any other -- closures, decorator wrappers, the
11677+
* route handlers of an app factory (`@app.route` inside create_app), local
11678+
* classes. Walk the body with the def's QN as the enclosing scope, so a nested
11679+
* def is qualified by the def it sits in: proj.mod.create_app.index. That is
11680+
* the QN the Python LSP already gives the calls inside it (py_process_function)
11681+
* and the one the call-scope walk computes (extract_unified.c
11682+
* py_nearest_scope_qn). Only the body is walked: parameters,
11683+
* annotations and decorators hold expressions, never statements. A lambda or
11684+
* a comprehension holds no statements either, so neither becomes a def. */
11685+
static void py_push_def_body(wd_stack_t *s, TSNode def_node, const char *def_qn) {
11686+
TSNode body = ts_node_child_by_field_name(def_node, TS_FIELD("body"));
11687+
if (def_qn && !ts_node_is_null(body)) {
11688+
wd_push(s, body, def_qn);
11689+
}
11690+
}
11691+
11692+
/* Python methods are extracted by extract_class_methods, not by the walk, so
11693+
* their bodies are pushed here, under the QN push_method_def gave the method
11694+
* (class_qn.name), in source order. */
11695+
static void py_push_method_bodies(CBMExtractCtx *ctx, TSNode class_node, const char *class_qn,
11696+
wd_stack_t *s) {
11697+
TSNode body = find_class_member_body(class_node, ctx->language);
11698+
if (ts_node_is_null(body) || !class_qn) {
11699+
return;
11700+
}
11701+
for (int i = (int)ts_node_named_child_count(body) - SKIP_CHAR; i >= 0; i--) {
11702+
TSNode method = ts_node_named_child(body, (uint32_t)i);
11703+
if (strcmp(ts_node_type(method), "decorated_definition") == 0) {
11704+
method = ts_node_child_by_field_name(method, TS_FIELD("definition"));
11705+
}
11706+
if (ts_node_is_null(method) || strcmp(ts_node_type(method), "function_definition") != 0) {
11707+
continue;
11708+
}
11709+
TSNode name_node = resolve_method_name(method, ctx->language);
11710+
char *name =
11711+
ts_node_is_null(name_node)
11712+
? NULL
11713+
: cbm_func_name_node_text(ctx->arena, name_node, ctx->source, ctx->language);
11714+
if (name && name[0]) {
11715+
py_push_def_body(s, method, cbm_arena_sprintf(ctx->arena, "%s.%s", class_qn, name));
11716+
}
11717+
}
11718+
}
11719+
1167011720
// ASCII case-insensitive equality vs a lowercase literal (CFML attribute names
1167111721
// are case-insensitive, e.g. NAME / Name / name).
1167211722
static bool ascii_ci_equals(const char *s, const char *lower_lit) {
@@ -12211,6 +12261,16 @@ static void walk_defs(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec,
1221112261
}
1221212262
if (cbm_kind_in_set(node, spec->function_node_types)) {
1221312263
if (!is_template_class_node(node, ctx->language)) {
12264+
if (ctx->language == CBM_LANG_PYTHON) {
12265+
int defs_before = ctx->result->defs.count;
12266+
extract_func_def(ctx, node, spec);
12267+
if (ctx->result->defs.count > defs_before) {
12268+
py_push_def_body(&s, node,
12269+
ctx->result->defs.items[ctx->result->defs.count - SKIP_ONE]
12270+
.qualified_name);
12271+
}
12272+
continue;
12273+
}
1221412274
extract_func_def(ctx, node, spec);
1221512275
// Most languages stop here. JS/TS (and Wolfram) descend into the
1221612276
// function body so NESTED named definitions are also captured —
@@ -12268,6 +12328,9 @@ static void walk_defs(CBMExtractCtx *ctx, TSNode root, const CBMLangSpec *spec,
1226812328
extract_class_def(ctx, node, spec);
1226912329
const char *new_enclosing = compute_class_qn(ctx, node, frame.enclosing_class_qn);
1227012330
push_class_body_children(node, spec, &s, new_enclosing, ctx);
12331+
if (ctx->language == CBM_LANG_PYTHON && new_enclosing != frame.enclosing_class_qn) {
12332+
py_push_method_bodies(ctx, node, new_enclosing, &s);
12333+
}
1227112334
continue;
1227212335
}
1227312336

‎internal/cbm/extract_unified.c‎

Lines changed: 28 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -916,6 +916,21 @@ static const char *objectscript_get_method_qn(CBMExtractCtx *ctx, TSNode node,
916916
return NULL;
917917
}
918918

919+
/* Python: the nearest named scope around the node -- a def or a class. A def
920+
* or class nested in a def is qualified by that def (proj.mod.create_app.index),
921+
* exactly as extract_defs.c mints it; enclosing_class_qn alone would skip the
922+
* def and hand a def nested in a method its method's QN. Lambdas push no named
923+
* frame, so they stay transparent. */
924+
static const char *py_nearest_scope_qn(const WalkState *state) {
925+
for (int i = state->scope_top - SKIP_ONE; i >= 0; i--) {
926+
const CBMWalkScope *f = &state->scopes[i];
927+
if ((f->kind == SCOPE_FUNC || f->kind == SCOPE_CLASS) && f->qn) {
928+
return f->qn;
929+
}
930+
}
931+
return NULL;
932+
}
933+
919934
// Compute function QN for scope tracking (mirrors cbm_enclosing_func_qn logic).
920935
static const char *compute_func_qn(CBMExtractCtx *ctx, TSNode node, const CBMLangSpec *spec,
921936
WalkState *state) {
@@ -1107,8 +1122,10 @@ static const char *compute_func_qn(CBMExtractCtx *ctx, TSNode node, const CBMLan
11071122
}
11081123
}
11091124

1110-
if (state->enclosing_class_qn) {
1111-
return cbm_arena_sprintf(ctx->arena, "%s.%s", state->enclosing_class_qn, qn_name);
1125+
const char *outer_qn =
1126+
ctx->language == CBM_LANG_PYTHON ? py_nearest_scope_qn(state) : state->enclosing_class_qn;
1127+
if (outer_qn) {
1128+
return cbm_arena_sprintf(ctx->arena, "%s.%s", outer_qn, qn_name);
11121129
}
11131130
/* Java/Go: directory-based module so this enclosing-func QN matches the def
11141131
* QN and the LSP caller_qn (the lsp_resolve join keys on exact equality). */
@@ -1162,9 +1179,15 @@ static const char *compute_class_qn(CBMExtractCtx *ctx, TSNode node, const WalkS
11621179

11631180
/* Nested class: prefix with the enclosing class QN (Outer.Inner) so this
11641181
* scope QN matches the def-side class QN (extract_defs.c compute_class_qn /
1165-
* extract_class_def), which the lsp_resolve join requires for nested types. */
1166-
if (state && state->enclosing_class_qn) {
1167-
return cbm_arena_sprintf(ctx->arena, "%s.%s", state->enclosing_class_qn, name);
1182+
* extract_class_def), which the lsp_resolve join requires for nested types.
1183+
* Python: a class inside a def is qualified by the def (py_nearest_scope_qn). */
1184+
const char *outer_qn = NULL;
1185+
if (state) {
1186+
outer_qn = ctx->language == CBM_LANG_PYTHON ? py_nearest_scope_qn(state)
1187+
: state->enclosing_class_qn;
1188+
}
1189+
if (outer_qn) {
1190+
return cbm_arena_sprintf(ctx->arena, "%s.%s", outer_qn, name);
11681191
}
11691192

11701193
/* Java/Go: directory-based module (see compute_func_qn). */

‎internal/cbm/lsp/py_lsp.c‎

Lines changed: 140 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -246,6 +246,11 @@ static void py_register_lambda(PyLSPContext *ctx, const char *name, TSNode lambd
246246
static void py_register_dict_literal(PyLSPContext *ctx, const char *name, TSNode dict_node);
247247
static TSNode py_lookup_lambda(PyLSPContext *ctx, const char *name);
248248
static const char *py_lookup_dict_dispatch(PyLSPContext *ctx, const char *var, const char *key);
249+
static void py_process_function(PyLSPContext *ctx, TSNode func_node, const char *container_qn);
250+
static void py_process_class(PyLSPContext *ctx, TSNode class_node);
251+
static void py_process_class_in(PyLSPContext *ctx, TSNode class_node, const char *container_qn);
252+
typedef enum { PY_LOCAL_DEFS_BIND, PY_LOCAL_DEFS_PROCESS } PyLocalDefsMode;
253+
static void py_visit_local_defs(PyLSPContext *ctx, TSNode block, PyLocalDefsMode mode, int depth);
249254

250255
/* ── Scope mutation wrappers ─────────────────────────────────────────────
251256
*
@@ -4362,6 +4367,101 @@ static void py_bind_parameters(PyLSPContext *ctx, TSNode params) {
43624367
}
43634368
}
43644369

4370+
/* Bind a def or class statement of the current function body as a local of
4371+
* that function (#1548): its graph QN extends the function's
4372+
* (enclosing_func_qn.name), the QN extraction gives it. An undecorated def
4373+
* registered with that QN becomes an exact callable alias; anything else
4374+
* (decorated, overloaded, not registered) is a hard local shadow, so no call
4375+
* falls through to a same-named module symbol. A class is bound by its type
4376+
* when registered, else shadowed. A parameter of the same name is already a
4377+
* local and keeps its own binding. */
4378+
static void py_bind_local_def(PyLSPContext *ctx, TSNode def, bool is_class) {
4379+
TSNode name_node = ts_node_child_by_field_name(def, "name", 4);
4380+
char *name = ts_node_is_null(name_node) ? NULL : py_node_text(ctx, name_node);
4381+
if (!name || !name[0] || !ctx->enclosing_func_qn) {
4382+
py_disable_callable_value_proof(ctx);
4383+
return;
4384+
}
4385+
if (cbm_scope_lookup_local(ctx->current_scope, name))
4386+
return;
4387+
const char *qn = cbm_arena_sprintf(ctx->arena, "%s.%s", ctx->enclosing_func_qn, name);
4388+
if (!qn) {
4389+
py_disable_callable_value_proof(ctx);
4390+
py_scope_bind(ctx, name, cbm_type_unknown());
4391+
return;
4392+
}
4393+
if (is_class) {
4394+
py_scope_bind(ctx, name,
4395+
cbm_registry_lookup_type(ctx->registry, qn) ? cbm_type_named(ctx->arena, qn)
4396+
: cbm_type_unknown());
4397+
return;
4398+
}
4399+
const CBMRegisteredFunc *func = cbm_registry_lookup_func(ctx->registry, qn);
4400+
if (py_func_is_exact_callable_value(func)) {
4401+
py_scope_bind_callable(ctx, name, cbm_type_unknown(), func->qualified_name);
4402+
} else {
4403+
py_scope_bind(ctx, name, cbm_type_unknown());
4404+
}
4405+
}
4406+
4407+
/* Statements a function body nests further statements in. A def inside any of
4408+
* them is still a local of the function; a nested def, class or lambda body
4409+
* is a scope of its own and is never entered. */
4410+
static bool py_is_statement_block(const char *kind) {
4411+
static const char *const kinds[] = {"block",
4412+
"if_statement",
4413+
"elif_clause",
4414+
"else_clause",
4415+
"for_statement",
4416+
"while_statement",
4417+
"try_statement",
4418+
"except_clause",
4419+
"except_group_clause",
4420+
"finally_clause",
4421+
"with_statement",
4422+
"match_statement",
4423+
"case_clause",
4424+
NULL};
4425+
for (int i = 0; kinds[i]; i++) {
4426+
if (strcmp(kind, kinds[i]) == 0)
4427+
return true;
4428+
}
4429+
return false;
4430+
}
4431+
4432+
/* Visit the def/class statements of a function body, including those inside
4433+
* its compound statements: BIND binds each as a local (py_bind_local_def),
4434+
* PROCESS walks each nested def's own body (py_process_function). */
4435+
static void py_visit_local_defs(PyLSPContext *ctx, TSNode block, PyLocalDefsMode mode, int depth) {
4436+
if (!ctx || ctx->eval_failure || ts_node_is_null(block) || depth >= cbm_lsp_max_walk_depth())
4437+
return;
4438+
PyKids kids = py_kids(ctx->arena, block);
4439+
for (uint32_t i = 0; i < kids.n && !ctx->eval_failure; i++) {
4440+
TSNode c = py_kid(&kids, i);
4441+
const char *ck = ts_node_type(c);
4442+
TSNode def = c;
4443+
if (strcmp(ck, "decorated_definition") == 0) {
4444+
def = ts_node_child_by_field_name(c, "definition", 10);
4445+
if (ts_node_is_null(def))
4446+
continue;
4447+
ck = ts_node_type(def);
4448+
}
4449+
if (strcmp(ck, "function_definition") == 0) {
4450+
if (mode == PY_LOCAL_DEFS_BIND)
4451+
py_bind_local_def(ctx, def, false);
4452+
else
4453+
py_process_function(ctx, def, ctx->enclosing_func_qn);
4454+
} else if (strcmp(ck, "class_definition") == 0) {
4455+
if (mode == PY_LOCAL_DEFS_BIND)
4456+
py_bind_local_def(ctx, def, true);
4457+
else
4458+
py_process_class_in(ctx, def, ctx->enclosing_func_qn);
4459+
} else if (py_is_statement_block(ck)) {
4460+
py_visit_local_defs(ctx, c, mode, depth + 1);
4461+
}
4462+
}
4463+
}
4464+
43654465
static void py_process_function(PyLSPContext *ctx, TSNode func_node, const char *container_qn) {
43664466
if (!ctx || ctx->eval_failure)
43674467
return;
@@ -4391,22 +4491,13 @@ static void py_process_function(PyLSPContext *ctx, TSNode func_node, const char
43914491

43924492
TSNode body = ts_node_child_by_field_name(func_node, "body", 4);
43934493
if (!ts_node_is_null(body)) {
4494+
/* Python resolves a bare name in the function's own scope first: every
4495+
* def/class statement of the body binds a local for the whole body
4496+
* (LEGB), whatever a module-level symbol of the same spelling says. */
4497+
py_visit_local_defs(ctx, body, PY_LOCAL_DEFS_BIND, 0);
43944498
py_resolve_calls_in(ctx, body);
4395-
// Also descend into nested function/class definitions in the body.
4396-
PyKids bk = py_kids(ctx->arena, body);
4397-
for (uint32_t i = 0; i < bk.n; i++) {
4398-
TSNode c = py_kid(&bk, i);
4399-
const char *ck = ts_node_type(c);
4400-
if (strcmp(ck, "function_definition") == 0) {
4401-
py_process_function(ctx, c, ctx->enclosing_func_qn);
4402-
} else if (strcmp(ck, "decorated_definition") == 0) {
4403-
TSNode def = ts_node_child_by_field_name(c, "definition", 10);
4404-
if (!ts_node_is_null(def) &&
4405-
strcmp(ts_node_type(def), "function_definition") == 0) {
4406-
py_process_function(ctx, def, ctx->enclosing_func_qn);
4407-
}
4408-
}
4409-
}
4499+
// Also descend into nested function definitions in the body.
4500+
py_visit_local_defs(ctx, body, PY_LOCAL_DEFS_PROCESS, 0);
44104501
}
44114502

44124503
py_scope_restore(ctx, saved);
@@ -4536,7 +4627,10 @@ static void py_exec_class_definition(PyLSPContext *ctx, TSNode class_node) {
45364627
py_scope_restore(ctx, saved_scope);
45374628
}
45384629

4539-
static void py_process_class(PyLSPContext *ctx, TSNode class_node) {
4630+
/* Walk a class's methods with the class qualified under `container_qn`: the
4631+
* module for a module-level class, the enclosing def for a class declared in
4632+
* a function body (#1548: extraction gives it the def's QN as prefix). */
4633+
static void py_process_class_in(PyLSPContext *ctx, TSNode class_node, const char *container_qn) {
45404634
if (!ctx || ctx->eval_failure)
45414635
return;
45424636
TSNode name_node = ts_node_child_by_field_name(class_node, "name", 4);
@@ -4547,7 +4641,8 @@ static void py_process_class(PyLSPContext *ctx, TSNode class_node) {
45474641
return;
45484642

45494643
const char *prev_class = ctx->enclosing_class_qn;
4550-
ctx->enclosing_class_qn = cbm_arena_sprintf(ctx->arena, "%s.%s", ctx->module_qn, cname);
4644+
ctx->enclosing_class_qn =
4645+
cbm_arena_sprintf(ctx->arena, "%s.%s", container_qn ? container_qn : ctx->module_qn, cname);
45514646

45524647
TSNode body = ts_node_child_by_field_name(class_node, "body", 4);
45534648
if (!ts_node_is_null(body)) {
@@ -4621,6 +4716,10 @@ static void py_process_class(PyLSPContext *ctx, TSNode class_node) {
46214716
ctx->enclosing_class_qn = prev_class;
46224717
}
46234718

4719+
static void py_process_class(PyLSPContext *ctx, TSNode class_node) {
4720+
py_process_class_in(ctx, class_node, NULL);
4721+
}
4722+
46244723
/* Module bindings are replayed in source order before deferred function bodies
46254724
* are analyzed. Class definitions always replace an earlier callable identity;
46264725
* the registry decides whether the resulting class type is known precisely. */
@@ -4687,6 +4786,28 @@ static char *py_module_prefix(CBMArena *scratch, const char *module_qn, size_t l
46874786
return prefix;
46884787
}
46894788

4789+
/* A class declared inside a def -- django's per-test `class MyModelAdmin`, an
4790+
* app factory's local class, and anything nested in one -- is a local of that
4791+
* def, not a global of the module: bound here by its short name, every
4792+
* same-named local class of the file collapsed onto whichever registered last.
4793+
* Its QN extends a registered function's (proj.mod.Tests.test_x.MyModelAdmin),
4794+
* which tells it apart from a class nested in a class (proj.mod.Outer.Inner).
4795+
* Only prefixes longer than the module are probed. */
4796+
static bool py_type_is_function_local(const PyLSPContext *ctx, const char *qn, size_t module_len) {
4797+
char scope[512];
4798+
size_t len = strlen(qn);
4799+
if (len >= sizeof(scope))
4800+
return false;
4801+
memcpy(scope, qn, len + 1);
4802+
for (char *dot = strrchr(scope, '.'); dot && (size_t)(dot - scope) > module_len;
4803+
dot = strrchr(scope, '.')) {
4804+
*dot = '\0';
4805+
if (cbm_registry_lookup_func(ctx->registry, scope))
4806+
return true;
4807+
}
4808+
return false;
4809+
}
4810+
46904811
/* A project registry can contain another file's class in the same logical
46914812
* Python module. Keep those cross-file globals available, but do not prebind
46924813
* classes declared by this source: their binding epoch belongs in the ordered
@@ -4734,6 +4855,8 @@ static void py_bind_external_module_classes(PyLSPContext *ctx, const PyKids *rk)
47344855
continue;
47354856
if (package_scope && strchr(qn + prefix_len + 1, '.'))
47364857
continue; /* submodule/nested classes are not package globals */
4858+
if (py_type_is_function_local(ctx, qn, prefix_len))
4859+
continue; /* a def-local class is no module global (#1548) */
47374860
py_scope_bind(ctx, name, cbm_type_named(ctx->arena, qn));
47384861
}
47394862
if (prefix)

‎src/pipeline/fqn.c‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -355,6 +355,30 @@ char *cbm_pipeline_resolve_relative_import(const char *source_rel, const char *m
355355
return resolve_js_relative(buf, sizeof(buf), module_path);
356356
}
357357

358+
size_t cbm_pipeline_fqn_symbol_prefix_len(const char *project, const char *rel_path) {
359+
if (!project) {
360+
return 0;
361+
}
362+
char path[FQN_PATH_BUF];
363+
int n = snprintf(path, sizeof(path), "%s", rel_path ? rel_path : "");
364+
if (n < 0 || (size_t)n >= sizeof(path)) {
365+
return 0;
366+
}
367+
cbm_normalize_path_sep(path);
368+
strip_file_extension(path);
369+
const char *segments[CBM_SZ_256];
370+
int seg_count = 0;
371+
segments[seg_count++] = project;
372+
seg_count += tokenize_path(path, segments + seg_count, FQN_MAX_PATH_SEGS);
373+
/* Any symbol name: the prefix cbm_pipeline_fqn_compute puts before it. */
374+
strip_init_or_index(segments, &seg_count, "symbol");
375+
size_t total = 0;
376+
for (int i = 0; i < seg_count; i++) {
377+
total += strlen(segments[i]) + (i > 0 ? FQN_SEP_LEN : 0);
378+
}
379+
return total;
380+
}
381+
358382
char *cbm_pipeline_fqn_folder(const char *project, const char *rel_dir) {
359383
if (!project) {
360384
return strdup("");

0 commit comments

Comments
 (0)