Skip to content

Commit c06465b

Browse files
authored
Merge pull request #2680 from DeusData/fix/py-relative-package-import
fix(python): from . import x names the importing package, not its parent
2 parents 90d6d64 + 99dafc1 commit c06465b

3 files changed

Lines changed: 314 additions & 2 deletions

File tree

‎internal/cbm/extract_imports.c‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,19 @@ static const char *python_import_root(CBMArena *a, const char *path) {
123123
return dot ? cbm_arena_strndup(a, path, (size_t)(dot - path)) : path;
124124
}
125125

126+
/* Spell `from <module> import <name>` as one dotted path. A package-relative
127+
* module (`from . import x`, `from .. import x`) is nothing but dots, which
128+
* already end in the separator: joining with another '.' would add a level and
129+
* name the PARENT package ("..x"), so the import resolved one package too high
130+
* and fell to a same-named definition anywhere in the project. */
131+
static const char *python_join_from_import(CBMArena *a, const char *module, const char *name) {
132+
size_t len = strlen(module);
133+
if (len > 0 && module[len - SKIP_ONE] == '.') {
134+
return cbm_arena_sprintf(a, "%s%s", module, name);
135+
}
136+
return cbm_arena_sprintf(a, "%s.%s", module, name);
137+
}
138+
126139
// --- Go imports ---
127140
// import_declaration -> import_spec_list -> import_spec -> (name, path)
128141

@@ -209,7 +222,7 @@ static void emit_py_aliased_import(CBMExtractCtx *ctx, TSNode child, const char
209222
}
210223
const char *local = !ts_node_is_null(alias_node) ? cbm_node_text(a, alias_node, ctx->source)
211224
: path_last(a, name);
212-
const char *full = mod_prefix ? cbm_arena_sprintf(a, "%s.%s", mod_prefix, name) : name;
225+
const char *full = mod_prefix ? python_join_from_import(a, mod_prefix, name) : name;
213226
CBMImport imp = {.local_name = local, .module_path = full};
214227
cbm_imports_push(&ctx->result->imports, a, imp);
215228
}
@@ -267,7 +280,7 @@ static void emit_py_import_from_name(CBMExtractCtx *ctx, TSNode child, const cha
267280
CBMArena *a = ctx->arena;
268281
char *name = cbm_node_text(a, child, ctx->source);
269282
if (name && name[0]) {
270-
const char *full = mod_path ? cbm_arena_sprintf(a, "%s.%s", mod_path, name) : name;
283+
const char *full = mod_path ? python_join_from_import(a, mod_path, name) : name;
271284
CBMImport imp = {.local_name = name, .module_path = full};
272285
cbm_imports_push(&ctx->result->imports, a, imp);
273286
}

‎tests/test_extraction.c‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3790,6 +3790,48 @@ TEST(python_imports) {
37903790
PASS();
37913791
}
37923792

3793+
/* The module_path recorded for a local name, or NULL. */
3794+
static const char *import_path_of(CBMFileResult *r, const char *local) {
3795+
for (int i = 0; i < r->imports.count; i++) {
3796+
if (r->imports.items[i].local_name && strcmp(r->imports.items[i].local_name, local) == 0)
3797+
return r->imports.items[i].module_path;
3798+
}
3799+
return NULL;
3800+
}
3801+
3802+
/* `from . import x` names the importing file's OWN package: the path is ".x",
3803+
* as `from .m import f` is ".m.f". Joining the all-dots module with another '.'
3804+
* recorded "..x" (the parent package), so the import resolved one package too
3805+
* high and fell to an arbitrary same-named definition (home-assistant: 7,175
3806+
* calls bound to another integration's function). */
3807+
TEST(python_package_relative_import_paths) {
3808+
CBMFileResult *r = extract("from . import async_register\n"
3809+
"from .. import shared\n"
3810+
"from . import local_fn as lf\n"
3811+
"from .helpers import helper_fn\n"
3812+
"from pkg.db import get_row\n",
3813+
CBM_LANG_PYTHON, "t", "comp/web/sub/mod.py");
3814+
ASSERT_NOT_NULL(r);
3815+
ASSERT_FALSE(r->has_error);
3816+
const char *own = import_path_of(r, "async_register");
3817+
const char *parent = import_path_of(r, "shared");
3818+
const char *aliased = import_path_of(r, "lf");
3819+
const char *member = import_path_of(r, "helper_fn");
3820+
const char *absolute = import_path_of(r, "get_row");
3821+
ASSERT_NOT_NULL(own);
3822+
ASSERT_NOT_NULL(parent);
3823+
ASSERT_NOT_NULL(aliased);
3824+
ASSERT_NOT_NULL(member);
3825+
ASSERT_NOT_NULL(absolute);
3826+
ASSERT_STR_EQ(own, ".async_register");
3827+
ASSERT_STR_EQ(parent, "..shared");
3828+
ASSERT_STR_EQ(aliased, ".local_fn");
3829+
ASSERT_STR_EQ(member, ".helpers.helper_fn"); /* control: unchanged */
3830+
ASSERT_STR_EQ(absolute, "pkg.db.get_row"); /* control: unchanged */
3831+
cbm_free_result(r);
3832+
PASS();
3833+
}
3834+
37933835
TEST(js_imports) {
37943836
CBMFileResult *r = extract("import React from 'react';\nimport { useState } from "
37953837
"'react';\nconst fs = require('fs');\n",
@@ -12847,6 +12889,7 @@ SUITE(extraction) {
1284712889
RUN_TEST(python_iris_invokeClassMethod);
1284812890
RUN_TEST(go_calls);
1284912891
RUN_TEST(python_imports);
12892+
RUN_TEST(python_package_relative_import_paths);
1285012893
RUN_TEST(js_imports);
1285112894
RUN_TEST(go_imports);
1285212895
RUN_TEST(go_cgo_pseudo_import_dropped);

‎tests/test_pipeline.c‎

Lines changed: 256 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10618,6 +10618,258 @@ TEST(pipeline_python_bare_local_binding_parallel_suppresses_weak_edge) {
1061810618
PASS();
1061910619
}
1062010620

10621+
/* CALLS edges from a caller to a callee, both matched by QN suffix (start each
10622+
* suffix with '.' to anchor it at a segment boundary), any strategy. *guessed
10623+
* is set when a matching edge came from a name guess rather than an import
10624+
* binding or the LSP. */
10625+
static int py_calls_between(cbm_store_t *s, const char *project, const char *caller_suffix,
10626+
const char *callee_suffix, bool *guessed) {
10627+
cbm_edge_t *edges = NULL;
10628+
int edge_count = 0;
10629+
cbm_store_find_edges_by_type(s, project, "CALLS", &edges, &edge_count);
10630+
int hits = 0;
10631+
size_t cs = strlen(caller_suffix);
10632+
size_t ts = strlen(callee_suffix);
10633+
for (int i = 0; i < edge_count; i++) {
10634+
cbm_node_t src = {0};
10635+
cbm_node_t tgt = {0};
10636+
if (cbm_store_find_node_by_id(s, edges[i].source_id, &src) == CBM_STORE_OK &&
10637+
cbm_store_find_node_by_id(s, edges[i].target_id, &tgt) == CBM_STORE_OK) {
10638+
size_t sl = strlen(src.qualified_name);
10639+
size_t tl = strlen(tgt.qualified_name);
10640+
if (sl >= cs && tl >= ts && strcmp(src.qualified_name + sl - cs, caller_suffix) == 0 &&
10641+
strcmp(tgt.qualified_name + tl - ts, callee_suffix) == 0) {
10642+
hits++;
10643+
const char *pj = edges[i].properties_json ? edges[i].properties_json : "";
10644+
if (guessed && !strstr(pj, "\"import_map\"") && !strstr(pj, "\"lsp")) {
10645+
*guessed = true;
10646+
}
10647+
}
10648+
}
10649+
cbm_node_free_fields(&src);
10650+
cbm_node_free_fields(&tgt);
10651+
}
10652+
if (edges)
10653+
cbm_store_free_edges(edges, edge_count);
10654+
return hits;
10655+
}
10656+
10657+
/* Pin CBM_WORKERS=4 for a padded (> 50 files) parallel run; restore after. */
10658+
typedef struct {
10659+
char *saved;
10660+
bool had;
10661+
int rc;
10662+
} py_workers_pin_t;
10663+
10664+
static py_workers_pin_t py_pin_workers(bool parallel) {
10665+
py_workers_pin_t pin = {0};
10666+
if (!parallel) {
10667+
return pin;
10668+
}
10669+
const char *old = getenv("CBM_WORKERS");
10670+
pin.had = old != NULL;
10671+
pin.saved = old ? strdup(old) : NULL;
10672+
pin.rc = (old && !pin.saved) ? -1 : cbm_setenv("CBM_WORKERS", "4", 1);
10673+
return pin;
10674+
}
10675+
10676+
static int py_unpin_workers(bool parallel, py_workers_pin_t *pin) {
10677+
if (!parallel) {
10678+
return 0;
10679+
}
10680+
int rc = pin->had ? cbm_setenv("CBM_WORKERS", pin->saved, 1) : cbm_unsetenv("CBM_WORKERS");
10681+
free(pin->saved);
10682+
pin->saved = NULL;
10683+
return rc;
10684+
}
10685+
10686+
static void py_write_padding(const char *tmp, const char *stem, bool parallel) {
10687+
for (int i = 0; parallel && i < 52; i++) {
10688+
char name[64];
10689+
char body[96];
10690+
snprintf(name, sizeof(name), "%s%d.py", stem, i);
10691+
snprintf(body, sizeof(body), "def %s%d():\n return %d\n", stem, i, i);
10692+
write_temp_file(tmp, name, body);
10693+
}
10694+
}
10695+
10696+
/* `from . import x`, `from .. import x` and `from . import x as y` name the
10697+
* importing file's own (or parent) package. The extractor recorded them one
10698+
* package too high ("..x"), so the import fell to pass_pkgmap's symbol-name
10699+
* guess -- the lexicographically first same-named project definition -- and
10700+
* every call became a confident edge into comp.aaa, through the import map and
10701+
* the Python LSP's alias binding alike (home-assistant: 7,175 call pairs).
10702+
* Both resolvers: <= 50 files sequential, padded > 50 parallel. */
10703+
static int check_python_package_relative_imports(bool parallel) {
10704+
char tmp[256];
10705+
snprintf(tmp, sizeof(tmp), "/tmp/cbm_py_relpkg_XXXXXX");
10706+
if (!cbm_mkdtemp(tmp)) {
10707+
FAIL("tmpdir");
10708+
}
10709+
write_temp_file(tmp, "comp/__init__.py", "");
10710+
write_temp_file(tmp, "comp/aaa/__init__.py",
10711+
"def async_register(hass):\n"
10712+
" return 1\n\n"
10713+
"def local_fn():\n"
10714+
" return 2\n");
10715+
write_temp_file(tmp, "comp/web/__init__.py",
10716+
"def async_register(hass):\n"
10717+
" return 3\n");
10718+
write_temp_file(tmp, "comp/web/trigger.py",
10719+
"from . import async_register\n\n"
10720+
"def attach(hass):\n"
10721+
" return async_register(hass)\n");
10722+
write_temp_file(tmp, "comp/web/sub/__init__.py",
10723+
"def local_fn():\n"
10724+
" return 4\n");
10725+
write_temp_file(tmp, "comp/web/sub/mod.py",
10726+
"from .. import async_register\n"
10727+
"from . import local_fn as lf\n\n"
10728+
"def deep(hass):\n"
10729+
" async_register(hass)\n"
10730+
" return lf()\n");
10731+
py_write_padding(tmp, "relpad", parallel);
10732+
10733+
py_workers_pin_t pin = py_pin_workers(parallel);
10734+
char db_path[512];
10735+
snprintf(db_path, sizeof(db_path), "%s/relpkg.db", tmp);
10736+
cbm_pipeline_t *p = pin.rc == 0 ? cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL) : NULL;
10737+
int run_rc = p ? cbm_pipeline_run(p) : -1;
10738+
cbm_store_t *s = run_rc == 0 ? cbm_store_open_path(db_path) : NULL;
10739+
bool opened = s != NULL;
10740+
bool guessed = false;
10741+
int own = -1, own_decoy = -1, parent = -1, parent_decoy = -1, alias = -1, alias_decoy = -1;
10742+
if (s) {
10743+
const char *project = cbm_pipeline_project_name(p);
10744+
own = py_calls_between(s, project, ".comp.web.trigger.attach", ".comp.web.async_register",
10745+
&guessed);
10746+
own_decoy = py_calls_between(s, project, ".comp.web.trigger.attach",
10747+
".comp.aaa.async_register", NULL);
10748+
parent = py_calls_between(s, project, ".comp.web.sub.mod.deep", ".comp.web.async_register",
10749+
&guessed);
10750+
parent_decoy = py_calls_between(s, project, ".comp.web.sub.mod.deep",
10751+
".comp.aaa.async_register", NULL);
10752+
alias = py_calls_between(s, project, ".comp.web.sub.mod.deep", ".comp.web.sub.local_fn",
10753+
&guessed);
10754+
alias_decoy =
10755+
py_calls_between(s, project, ".comp.web.sub.mod.deep", ".comp.aaa.local_fn", NULL);
10756+
cbm_store_close(s);
10757+
}
10758+
cbm_pipeline_free(p);
10759+
int unpin_rc = py_unpin_workers(parallel, &pin);
10760+
th_rmtree(tmp);
10761+
10762+
ASSERT_EQ(pin.rc, 0);
10763+
ASSERT_EQ(unpin_rc, 0);
10764+
ASSERT_EQ(run_rc, 0);
10765+
ASSERT_TRUE(opened);
10766+
ASSERT_EQ(own, 1); /* from . import x: the file's own package */
10767+
ASSERT_EQ(own_decoy, 0); /* never the lexicographically first namesake */
10768+
ASSERT_EQ(parent, 1); /* from .. import x: the parent package */
10769+
ASSERT_EQ(parent_decoy, 0);
10770+
ASSERT_EQ(alias, 1); /* from . import x as y */
10771+
ASSERT_EQ(alias_decoy, 0);
10772+
ASSERT_FALSE(guessed); /* bound by the import, not by a name guess */
10773+
PASS();
10774+
}
10775+
10776+
TEST(pipeline_python_package_relative_imports) {
10777+
return check_python_package_relative_imports(false);
10778+
}
10779+
10780+
TEST(pipeline_python_package_relative_imports_parallel) {
10781+
return check_python_package_relative_imports(true);
10782+
}
10783+
10784+
/* #1237: a plain `from pkg.db import get_row_or_none` and bare calls resolve to
10785+
* the imported module-level def (function and method callers, plus the
10786+
* same-file caller), never to a same-named def elsewhere that a name guess
10787+
* would rank first (legacy.compat), and a re-index that only ADDS an unrelated
10788+
* file (the reporter's watcher re-index after writing benchmark scripts into the
10789+
* indexed tree) keeps exactly the same CALLS. Main already resolved this shape
10790+
* when the guard landed: it pins the shape and has no RED on revert. */
10791+
static int check_python_from_import_bare_call_issue1237(bool parallel) {
10792+
char tmp[256];
10793+
snprintf(tmp, sizeof(tmp), "/tmp/cbm_py_1237_XXXXXX");
10794+
if (!cbm_mkdtemp(tmp)) {
10795+
FAIL("tmpdir");
10796+
}
10797+
write_temp_file(tmp, "pkg/__init__.py", "");
10798+
write_temp_file(tmp, "pkg/db.py",
10799+
"def get_row_or_none(model, **filters):\n"
10800+
" return None\n\n"
10801+
"def other_helper():\n"
10802+
" return get_row_or_none(1, id=2)\n");
10803+
write_temp_file(tmp, "pkg/views.py",
10804+
"from pkg.db import get_row_or_none\n\n"
10805+
"def show(pk):\n"
10806+
" return get_row_or_none(object, id=pk)\n\n"
10807+
"class Detail:\n"
10808+
" def get(self, pk):\n"
10809+
" return get_row_or_none(object, pk=pk)\n");
10810+
write_temp_file(tmp, "legacy/__init__.py", "");
10811+
write_temp_file(tmp, "legacy/compat.py",
10812+
"def get_row_or_none(model):\n"
10813+
" return model\n");
10814+
py_write_padding(tmp, "rowpad", parallel);
10815+
10816+
py_workers_pin_t pin = py_pin_workers(parallel);
10817+
char db_path[512];
10818+
snprintf(db_path, sizeof(db_path), "%s/rows.db", tmp);
10819+
int run_rc[2] = {-1, -1};
10820+
int show[2] = {-1, -1};
10821+
int method[2] = {-1, -1};
10822+
int same[2] = {-1, -1};
10823+
int decoy[2] = {-1, -1};
10824+
int calls[2] = {-1, -1};
10825+
bool guessed = false;
10826+
for (int run = 0; run < 2 && pin.rc == 0; run++) {
10827+
if (run == 1) {
10828+
/* Second run on the same db: routes to the incremental re-index. */
10829+
write_temp_file(tmp, "bench_rows.py", "def bench():\n return 0\n");
10830+
}
10831+
cbm_pipeline_t *p = cbm_pipeline_new(tmp, db_path, CBM_MODE_FULL);
10832+
run_rc[run] = p ? cbm_pipeline_run(p) : -1;
10833+
cbm_store_t *s = run_rc[run] == 0 ? cbm_store_open_path(db_path) : NULL;
10834+
if (s) {
10835+
const char *project = cbm_pipeline_project_name(p);
10836+
const char *target = ".pkg.db.get_row_or_none";
10837+
const char *namesake = ".legacy.compat.get_row_or_none";
10838+
show[run] = py_calls_between(s, project, ".pkg.views.show", target, &guessed);
10839+
method[run] = py_calls_between(s, project, ".pkg.views.Detail.get", target, &guessed);
10840+
same[run] = py_calls_between(s, project, ".pkg.db.other_helper", target, &guessed);
10841+
decoy[run] = py_calls_between(s, project, ".pkg.views.show", namesake, NULL) +
10842+
py_calls_between(s, project, ".pkg.views.Detail.get", namesake, NULL);
10843+
calls[run] = cbm_store_count_edges_by_type(s, project, "CALLS");
10844+
cbm_store_close(s);
10845+
}
10846+
cbm_pipeline_free(p);
10847+
}
10848+
int unpin_rc = py_unpin_workers(parallel, &pin);
10849+
th_rmtree(tmp);
10850+
10851+
ASSERT_EQ(pin.rc, 0);
10852+
ASSERT_EQ(unpin_rc, 0);
10853+
for (int run = 0; run < 2; run++) {
10854+
ASSERT_EQ(run_rc[run], 0);
10855+
ASSERT_EQ(show[run], 1); /* cross-file bare call through a plain from-import */
10856+
ASSERT_EQ(method[run], 1); /* the same from a method */
10857+
ASSERT_EQ(same[run], 1); /* same-file bare call */
10858+
ASSERT_EQ(decoy[run], 0); /* never the namesake a name guess would pick */
10859+
}
10860+
ASSERT_EQ(calls[1], calls[0]); /* adding a call-free file moves no CALLS edge */
10861+
ASSERT_FALSE(guessed);
10862+
PASS();
10863+
}
10864+
10865+
TEST(pipeline_python_from_import_bare_call_issue1237) {
10866+
return check_python_from_import_bare_call_issue1237(false);
10867+
}
10868+
10869+
TEST(pipeline_python_from_import_bare_call_issue1237_parallel) {
10870+
return check_python_from_import_bare_call_issue1237(true);
10871+
}
10872+
1062110873
/* Reproduce-first: pass_parallel's fused cross-LSP eligibility currently counts
1062210874
* only parser-backed calls/call references. A Python binary operator has no
1062310875
* parser CBMCall; its __add__ semantic record and carrier are created together
@@ -25207,6 +25459,10 @@ SUITE(pipeline) {
2520725459
RUN_TEST(pipeline_python_receiver_parallel_suppresses_weak_method_edges);
2520825460
RUN_TEST(pipeline_python_bare_local_binding_suppresses_weak_edge);
2520925461
RUN_TEST(pipeline_python_bare_local_binding_parallel_suppresses_weak_edge);
25462+
RUN_TEST(pipeline_python_package_relative_imports);
25463+
RUN_TEST(pipeline_python_package_relative_imports_parallel);
25464+
RUN_TEST(pipeline_python_from_import_bare_call_issue1237);
25465+
RUN_TEST(pipeline_python_from_import_bare_call_issue1237_parallel);
2521025466
RUN_TEST(pipeline_parallel_python_cross_only_dunder_gets_synthetic_carrier);
2521125467
RUN_TEST(pipeline_parallel_rust_cross_only_macro_hidden_gets_synthetic_carrier);
2521225468
RUN_TEST(pipeline_arg_url_rejects_non_http_slash_arguments);

0 commit comments

Comments
 (0)