Skip to content

Commit 9bcfaab

Browse files
utilia-ai-woxMartin Vogel
authored andcommitted
fix(pkgmap+routes): resolve workspace imports + extract SvelteKit routes
Two structural fixes for TS monorepos (pnpm/yarn workspaces) and SvelteKit projects, measured to take a 885-file SvelteKit+Turborepo repo from 319 edges to 31802 (100x) with no fixture regressions. - pkgmap: package.json / composer.json live in IGNORED_JSON_FILES, so the manifest-driven pkgmap builders never saw them and every '@my/pkg' workspace import was silently dropped. Add a symlink-safe, skip-dir-aware filesystem walker (cbm_pkgmap_scan_repo / cbm_pkgmap_build_from_repo) that harvests manifests directly from the repo regardless of the discovery filter. Sequential and parallel paths both feed it now. - definitions: extract every file's defs (creating Module nodes) BEFORE resolving imports, so a workspace import in the first file can resolve against the complete in-memory graph (two-phase, cache-backed). - routes: synthesise Route nodes + HANDLES edges from SvelteKit's filesystem layout (+server / +page.server / +layout.server), which has no app.get(...) call site for pass_calls to pick up. Distilled from #369 onto current main. The PR's CBM_DISABLE_LSP_CROSS escape hatch was rebased onto the v0.7.0 fused cross-LSP architecture: it now gates run_cross_lsp (NULL all_defs makes the fused resolver no-op cross-file resolution) instead of wrapping the removed standalone pass_lsp_cross call. Relates to #271 / #56.
1 parent 2389d82 commit 9bcfaab

6 files changed

Lines changed: 489 additions & 19 deletions

File tree

‎src/pipeline/pass_definitions.c‎

Lines changed: 48 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -343,6 +343,21 @@ int cbm_pipeline_pass_definitions(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t
343343
int total_imports = 0;
344344
int errors = 0;
345345

346+
/* Sequential pass must extract all defs (which create Module/Function/...
347+
* nodes) BEFORE resolving imports — otherwise a workspace import in the
348+
* first file processed can't find the target Module node, because the
349+
* target file's defs haven't been extracted yet. Result cache is
350+
* required for this two-phase ordering. */
351+
CBMFileResult **local_cache = ctx->result_cache;
352+
bool owns_local_cache = false;
353+
if (!local_cache) {
354+
local_cache = (CBMFileResult **)calloc((size_t)file_count, sizeof(CBMFileResult *));
355+
owns_local_cache = (local_cache != NULL);
356+
}
357+
358+
/* Phase 1: extract every file and create def-derived nodes (Modules,
359+
* Functions, ...) so any file's IMPORTS can resolve against the
360+
* complete in-memory graph in Phase 2. */
346361
for (int i = 0; i < file_count; i++) {
347362
if (cbm_pipeline_check_cancel(ctx)) {
348363
return CBM_NOT_FOUND;
@@ -381,17 +396,45 @@ int cbm_pipeline_pass_definitions(cbm_pipeline_ctx_t *ctx, const cbm_file_info_t
381396
/* Store calls for pass_calls (we save them in the extraction results
382397
* for now — a future optimization would batch these) */
383398
total_calls += result->calls.count;
384-
total_imports += create_import_edges_for_file(ctx, result, rel);
385-
create_channel_edges_for_file(ctx, result, rel);
386399

387-
/* Cache or free the extraction result */
388-
if (ctx->result_cache) {
389-
ctx->result_cache[i] = result;
400+
if (local_cache) {
401+
local_cache[i] = result;
390402
} else {
403+
/* Cache unavailable: imports for this file can still only
404+
* resolve to defs already in the graph, but the file's
405+
* own defs are now persisted before the lookup. */
406+
total_imports += create_import_edges_for_file(ctx, result, rel);
407+
create_channel_edges_for_file(ctx, result, rel);
391408
cbm_free_result(result);
392409
}
393410
}
394411

412+
/* Phase 2: now that all extraction results are cached and Module
413+
* nodes for every file are in the graph, walk the cache again to
414+
* create IMPORTS / channel edges. Imports resolve against the full
415+
* project graph. */
416+
if (local_cache) {
417+
for (int i = 0; i < file_count; i++) {
418+
if (cbm_pipeline_check_cancel(ctx)) {
419+
break;
420+
}
421+
CBMFileResult *result = local_cache[i];
422+
if (!result) {
423+
continue;
424+
}
425+
total_imports += create_import_edges_for_file(ctx, result, files[i].rel_path);
426+
create_channel_edges_for_file(ctx, result, files[i].rel_path);
427+
}
428+
if (owns_local_cache) {
429+
for (int i = 0; i < file_count; i++) {
430+
if (local_cache[i]) {
431+
cbm_free_result(local_cache[i]);
432+
}
433+
}
434+
free(local_cache);
435+
}
436+
}
437+
395438
cbm_log_info("pass.done", "pass", "definitions", "defs", itoa_log(total_defs), "calls",
396439
itoa_log(total_calls), "imports", itoa_log(total_imports), "errors",
397440
itoa_log(errors));

‎src/pipeline/pass_parallel.c‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -594,6 +594,11 @@ static void merge_pkg_entries(cbm_pipeline_ctx_t *ctx, cbm_pkg_entries_t *pkg_en
594594
if (!pkg_entries) {
595595
return;
596596
}
597+
/* Supplement with a repo-wide filesystem walk so manifests filtered
598+
* by the main discoverer (package.json, composer.json — in
599+
* IGNORED_JSON_FILES) still feed pkgmap. Append into worker 0's
600+
* array so the existing merge below sees them. */
601+
cbm_pkgmap_scan_repo(ctx->repo_path, &pkg_entries[0]);
597602
cbm_pipeline_set_pkgmap(cbm_pkgmap_build(pkg_entries, worker_count, ctx->project_name));
598603
for (int i = 0; i < worker_count; i++) {
599604
cbm_pkg_entries_free(&pkg_entries[i]);

‎src/pipeline/pass_pkgmap.c‎

Lines changed: 136 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
*/
1313
#include "pipeline/pipeline.h"
1414
#include "pipeline/pipeline_internal.h"
15+
#include "discover/discover.h"
1516
#include "foundation/compat.h"
1617
#include "foundation/constants.h"
1718
#include "foundation/hash_table.h"
@@ -22,10 +23,12 @@
2223

2324
#include <yyjson/yyjson.h>
2425

26+
#include <dirent.h>
2527
#include <stdbool.h>
2628
#include <stdio.h>
2729
#include <stdlib.h>
2830
#include <string.h>
31+
#include <sys/stat.h>
2932

3033
/* Read an entire file into a malloc'd buffer. Returns NULL on failure. */
3134
static char *pkgmap_read_file(const char *path, int *out_len) {
@@ -747,6 +750,100 @@ CBMHashTable *cbm_pkgmap_build(cbm_pkg_entries_t *worker_entries, int worker_cou
747750
return map;
748751
}
749752

753+
/* Returns true if basename is a package manifest we know how to parse.
754+
* Used by the filesystem walker; cbm_pkgmap_try_parse is the source of
755+
* truth for which basenames produce entries. */
756+
static bool is_pkgmap_manifest_basename(const char *basename) {
757+
if (!basename) {
758+
return false;
759+
}
760+
if (strcmp(basename, "package.json") == 0 || strcmp(basename, "go.mod") == 0 ||
761+
strcmp(basename, "Cargo.toml") == 0 || strcmp(basename, "pyproject.toml") == 0 ||
762+
strcmp(basename, "composer.json") == 0 || strcmp(basename, "pubspec.yaml") == 0 ||
763+
strcmp(basename, "pom.xml") == 0 || strcmp(basename, "build.gradle") == 0 ||
764+
strcmp(basename, "build.gradle.kts") == 0 || strcmp(basename, "mix.exs") == 0) {
765+
return true;
766+
}
767+
return ends_with(basename, ".gemspec");
768+
}
769+
770+
/* Recursive filesystem walker that finds and parses package manifest
771+
* files independently of the main discovery filter. The main discovery
772+
* filter intentionally hides package.json / composer.json etc. from
773+
* code indexing (they're config, not source), but pass_pkgmap still
774+
* needs to read them to resolve workspace imports. Skips directories
775+
* matched by the shared cbm_should_skip_dir helper so we don't walk
776+
* node_modules, .git, build, etc. Returns the number of manifests
777+
* parsed, accumulated across the whole walk. */
778+
static int pkgmap_walk_dir(const char *abs_dir, const char *rel_dir,
779+
cbm_pkg_entries_t *entries) {
780+
DIR *dir = opendir(abs_dir);
781+
if (!dir) {
782+
return 0;
783+
}
784+
int parsed = 0;
785+
struct dirent *entry;
786+
while ((entry = readdir(dir)) != NULL) {
787+
const char *name = entry->d_name;
788+
if (name[0] == '.' && (name[1] == '\0' || (name[1] == '.' && name[2] == '\0'))) {
789+
continue;
790+
}
791+
char abs_path[PKGMAP_PATH_BUF];
792+
char rel_path[PKGMAP_PATH_BUF];
793+
snprintf(abs_path, sizeof(abs_path), "%s/%s", abs_dir, name);
794+
if (rel_dir && rel_dir[0]) {
795+
snprintf(rel_path, sizeof(rel_path), "%s/%s", rel_dir, name);
796+
} else {
797+
snprintf(rel_path, sizeof(rel_path), "%s", name);
798+
}
799+
struct stat st;
800+
if (lstat(abs_path, &st) != 0) {
801+
continue;
802+
}
803+
if (S_ISLNK(st.st_mode)) {
804+
continue;
805+
}
806+
if (S_ISDIR(st.st_mode)) {
807+
if (cbm_should_skip_dir(name, CBM_MODE_FULL)) {
808+
continue;
809+
}
810+
parsed += pkgmap_walk_dir(abs_path, rel_path, entries);
811+
continue;
812+
}
813+
if (!S_ISREG(st.st_mode)) {
814+
continue;
815+
}
816+
if (!is_pkgmap_manifest_basename(name)) {
817+
continue;
818+
}
819+
int source_len = 0;
820+
char *source = pkgmap_read_file(abs_path, &source_len);
821+
if (!source) {
822+
continue;
823+
}
824+
if (cbm_pkgmap_try_parse(name, rel_path, source, source_len, entries)) {
825+
parsed++;
826+
}
827+
free(source);
828+
}
829+
closedir(dir);
830+
return parsed;
831+
}
832+
833+
/* Scan a repository for package manifest files via the filesystem
834+
* walker above. Always-available companion to the parallel path's
835+
* per-worker manifest parsing, which is bound to whatever `files[]`
836+
* the discoverer produces and therefore misses ignored manifests like
837+
* package.json. NULL-safe; returns 0 entries when repo_path is unset. */
838+
int cbm_pkgmap_scan_repo(const char *repo_path, cbm_pkg_entries_t *entries) {
839+
if (!repo_path || !entries) {
840+
return 0;
841+
}
842+
int parsed = pkgmap_walk_dir(repo_path, "", entries);
843+
cbm_log_info("pkgmap.scan_repo", "manifests", pkgmap_itoa(parsed));
844+
return parsed;
845+
}
846+
750847
/* Build pkgmap for sequential path (reads manifest files directly) */
751848
CBMHashTable *cbm_pkgmap_build_from_files(const cbm_file_info_t *files, int file_count,
752849
const char *project_name) {
@@ -755,15 +852,7 @@ CBMHashTable *cbm_pkgmap_build_from_files(const cbm_file_info_t *files, int file
755852

756853
for (int i = 0; i < file_count; i++) {
757854
const char *basename = path_basename(files[i].rel_path);
758-
/* Quick check: is this a manifest file? */
759-
bool is_manifest =
760-
(strcmp(basename, "package.json") == 0 || strcmp(basename, "go.mod") == 0 ||
761-
strcmp(basename, "Cargo.toml") == 0 || strcmp(basename, "pyproject.toml") == 0 ||
762-
strcmp(basename, "composer.json") == 0 || strcmp(basename, "pubspec.yaml") == 0 ||
763-
strcmp(basename, "pom.xml") == 0 || strcmp(basename, "build.gradle") == 0 ||
764-
strcmp(basename, "build.gradle.kts") == 0 || strcmp(basename, "mix.exs") == 0 ||
765-
ends_with(basename, ".gemspec"));
766-
if (!is_manifest) {
855+
if (!is_pkgmap_manifest_basename(basename)) {
767856
continue;
768857
}
769858

@@ -782,6 +871,44 @@ CBMHashTable *cbm_pkgmap_build_from_files(const cbm_file_info_t *files, int file
782871
return map;
783872
}
784873

874+
/* Variant of cbm_pkgmap_build_from_files that ALSO walks the repo
875+
* filesystem to pick up manifests filtered out by the main discoverer
876+
* (the canonical case: package.json, which is in IGNORED_JSON_FILES).
877+
* Falls back to the files[]-only behaviour if repo_path is NULL. */
878+
CBMHashTable *cbm_pkgmap_build_from_repo(const char *repo_path, const cbm_file_info_t *files,
879+
int file_count, const char *project_name) {
880+
cbm_pkg_entries_t entries;
881+
cbm_pkg_entries_init(&entries);
882+
883+
/* Manifests already visible through discovery (Cargo.toml, go.mod,
884+
* pyproject.toml, ...). package.json typically isn't, but we still
885+
* harvest whatever the discovery filter exposed in case downstream
886+
* filters change. */
887+
int from_files = 0;
888+
for (int i = 0; i < file_count; i++) {
889+
const char *basename = path_basename(files[i].rel_path);
890+
if (!is_pkgmap_manifest_basename(basename)) {
891+
continue;
892+
}
893+
from_files++;
894+
int source_len = 0;
895+
char *source = pkgmap_read_file(files[i].path, &source_len);
896+
if (!source) {
897+
continue;
898+
}
899+
cbm_pkgmap_try_parse(basename, files[i].rel_path, source, source_len, &entries);
900+
free(source);
901+
}
902+
903+
int from_walk = cbm_pkgmap_scan_repo(repo_path, &entries);
904+
cbm_log_info("pkgmap.scan", "manifests_from_files", pkgmap_itoa(from_files),
905+
"manifests_from_walk", pkgmap_itoa(from_walk),
906+
"entries", pkgmap_itoa(entries.count));
907+
CBMHashTable *map = cbm_pkgmap_build(&entries, SKIP_ONE, project_name);
908+
cbm_pkg_entries_free(&entries);
909+
return map;
910+
}
911+
785912
static void pkgmap_free_entry(const char *key, void *value, void *userdata) {
786913
(void)userdata;
787914
free((void *)key);

0 commit comments

Comments
 (0)