Skip to content

Commit 879c11e

Browse files
authored
Merge pull request #2593 from DeusData/feat/doc-links-languages-diagrams
feat(docs): doc-comment links for eight more languages, diagrams and OpenAPI operations in the graph
2 parents 82b1ec9 + 65d365e commit 879c11e

63 files changed

Lines changed: 43517 additions & 254 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎Makefile.cbm‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -386,6 +386,16 @@ EXTRACTION_SRCS = \
386386
$(CBM_DIR)/pdf/pdf_text.c \
387387
$(CBM_DIR)/pdf/pdf_tables.c \
388388
$(CBM_DIR)/pdf/pdf_unicode.c \
389+
$(CBM_DIR)/doclink_go.c \
390+
$(CBM_DIR)/doclink_jvm.c \
391+
$(CBM_DIR)/doclink_rs.c \
392+
$(CBM_DIR)/doclink_perl.c \
393+
$(CBM_DIR)/doclink_ts.c \
394+
$(CBM_DIR)/doclink_php.c \
395+
$(CBM_DIR)/doclink_c.c \
396+
$(CBM_DIR)/diagram.c \
397+
$(CBM_DIR)/route_path.c \
398+
$(CBM_DIR)/openapi.c \
389399
$(CBM_DIR)/sql_values.c \
390400
$(CBM_DIR)/helpers.c \
391401
$(CBM_DIR)/cpp_branch_views.c \
@@ -512,6 +522,14 @@ PIPELINE_SRCS = \
512522
src/pipeline/doc_links_pdf.c \
513523
src/pipeline/doc_links_rst.c \
514524
src/pipeline/doc_links_adoc.c \
525+
src/pipeline/doc_links_go.c \
526+
src/pipeline/doc_links_jvm.c \
527+
src/pipeline/doc_links_rs.c \
528+
src/pipeline/doc_links_perl.c \
529+
src/pipeline/doc_links_ts.c \
530+
src/pipeline/doc_links_php.c \
531+
src/pipeline/doc_links_c.c \
532+
src/pipeline/pass_diagrams.c \
515533
src/pipeline/doc_links_msbuild.c \
516534
src/pipeline/pass_route_nodes.c \
517535
src/pipeline/pass_enrichment.c \
@@ -774,7 +792,7 @@ TEST_DISCOVER_SRCS = \
774792

775793
TEST_GRAPH_BUFFER_SRCS = tests/test_graph_buffer.c
776794

777-
TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_pipeline_conventions.c tests/test_pipeline_frozen.c tests/test_importance.c tests/test_cross_repo.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_doclinks.c tests/test_doc_mentions.c tests/test_doc_links_md.c tests/test_doc_links_pdf.c tests/test_doc_links_rst.c tests/test_doc_links_adoc.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c tests/test_index_resilience.c tests/test_index_format.c tests/test_call_reference_contract.c tests/repro/repro_call_scope_usages.c tests/repro/repro_call_argument_usages.c tests/repro/repro_reference_precision.c tests/repro/repro_lexical_binding_precision.c tests/repro/repro_call_argument_matrix_a.c tests/repro/repro_call_argument_matrix_b.c tests/repro/repro_call_node_behaviors.c tests/repro/repro_language_registry.c tests/repro/repro_call_node_manifest.c tests/repro/repro_lsp_ordered_signatures.c tests/repro/repro_lsp_ordered_local.c tests/repro/repro_ts_overload_return_chains.c tests/repro/repro_harness_cleanup.c tests/repro/repro_runner_filter.c
795+
TEST_PIPELINE_SRCS = tests/test_registry.c tests/test_pipeline.c tests/test_pipeline_conventions.c tests/test_pipeline_frozen.c tests/test_importance.c tests/test_cross_repo.c tests/test_fqn.c tests/test_route_canon.c tests/test_path_alias.c tests/test_configlink.c tests/test_doclinks.c tests/test_doc_mentions.c tests/test_doc_links_md.c tests/test_doc_links_pdf.c tests/test_doc_links_rst.c tests/test_doc_links_adoc.c tests/test_doc_mentions_go.c tests/test_doc_mentions_jvm.c tests/test_doc_mentions_rs.c tests/test_doc_mentions_perl.c tests/test_doc_mentions_ts.c tests/test_doc_mentions_php.c tests/test_doc_mentions_c.c tests/test_diagrams.c tests/test_openapi.c tests/test_infrascan.c tests/test_worker_pool.c tests/test_parallel.c tests/test_index_resilience.c tests/test_index_format.c tests/test_call_reference_contract.c tests/repro/repro_call_scope_usages.c tests/repro/repro_call_argument_usages.c tests/repro/repro_reference_precision.c tests/repro/repro_lexical_binding_precision.c tests/repro/repro_call_argument_matrix_a.c tests/repro/repro_call_argument_matrix_b.c tests/repro/repro_call_node_behaviors.c tests/repro/repro_language_registry.c tests/repro/repro_call_node_manifest.c tests/repro/repro_lsp_ordered_signatures.c tests/repro/repro_lsp_ordered_local.c tests/repro/repro_ts_overload_return_chains.c tests/repro/repro_harness_cleanup.c tests/repro/repro_runner_filter.c
778796

779797
TEST_WATCHER_SRCS = tests/test_watcher.c
780798

‎internal/cbm/cbm.c‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -277,6 +277,11 @@ void cbm_channels_push(CBMChannelArray *arr, CBMArena *a, CBMChannel ch) {
277277
arr->items[arr->count++] = ch;
278278
}
279279

280+
void cbm_diagram_arrows_push(CBMDiagramArrowArray *arr, CBMArena *a, CBMDiagramArrow da) {
281+
GROW_ARRAY(arr, a);
282+
arr->items[arr->count++] = da;
283+
}
284+
280285
// --- String input reader (for parse_with_options) ---
281286

282287
typedef struct {
@@ -2837,6 +2842,26 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C
28372842
.module_qn = result->module_qn,
28382843
};
28392844
cbm_adoc_extract_document(&adoc_ctx);
2845+
cbm_extract_diagrams(&adoc_ctx); /* after the Sections they hang under */
2846+
cbm_index_mark_done(rel_path);
2847+
return result;
2848+
}
2849+
/* Diagram files have no grammar either: each format is read from the text
2850+
* (diagram.c). */
2851+
if (language == CBM_LANG_DIAGRAM) {
2852+
result->module_qn = cbm_fqn_module_source_lang(a, project, rel_path, language);
2853+
CBMExtractCtx diagram_ctx = {
2854+
.arena = a,
2855+
.scratch = scratch,
2856+
.result = result,
2857+
.source = source,
2858+
.source_len = source_len,
2859+
.language = language,
2860+
.project = project,
2861+
.rel_path = rel_path,
2862+
.module_qn = result->module_qn,
2863+
};
2864+
cbm_extract_diagram_file(&diagram_ctx);
28402865
cbm_index_mark_done(rel_path);
28412866
return result;
28422867
}
@@ -3021,6 +3046,12 @@ static CBMFileResult *extract_file_ex_body(const char *source, int source_len, C
30213046
// Channel detection (Socket.IO / EventEmitter) — JS/TS only.
30223047
cbm_extract_channels(&ctx);
30233048

3049+
// Diagrams: blocks of documents and diagram files (self-gated by language).
3050+
cbm_extract_diagrams(&ctx);
3051+
3052+
// OpenAPI operations as Routes (self-gated: API description documents).
3053+
cbm_extract_openapi(&ctx);
3054+
30243055
// K8s / Kustomize semantic pass (additional structured extraction for YAML-based infra files).
30253056
if (ctx.language == CBM_LANG_KUSTOMIZE || ctx.language == CBM_LANG_K8S) {
30263057
cbm_extract_k8s(&ctx);

‎internal/cbm/cbm.h‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,10 @@ typedef enum {
211211
CBM_LANG_CHIALISP, // Chialisp (.clsp/.clib/.clinc — Chia smart-coin s-expression language)
212212
CBM_LANG_PDF, // PDF documents: the text layer, read without a grammar (doc_pdf.c)
213213
CBM_LANG_ASCIIDOC, // AsciiDoc documents, read without a grammar (doc_adoc.c)
214+
CBM_LANG_OPENAPI, // openapi.json / swagger.json: the JSON grammar, no node per key; their
215+
// operations are Routes (openapi.c)
216+
CBM_LANG_DIAGRAM, // diagram files (PlantUML, DOT, D2, draw.io, Excalidraw), read
217+
// without a grammar (diagram.c)
214218
CBM_LANG_COUNT
215219
} CBMLanguage;
216220

@@ -612,6 +616,23 @@ typedef struct {
612616
int cap;
613617
} CBMChannelArray;
614618

619+
/* One arrow of a diagram (diagram.c): from one DiagramElement definition of
620+
* the file to another, written as a DIAGRAM_ARROW edge. */
621+
typedef struct {
622+
const char *from_qn;
623+
const char *to_qn;
624+
const char *label; /* the arrow's text, NULL for none */
625+
const char *style; /* what the arrow means as its format writes it: "flow",
626+
* "message", "inherits", "composition", ... */
627+
uint32_t line;
628+
} CBMDiagramArrow;
629+
630+
typedef struct {
631+
CBMDiagramArrow *items;
632+
int count;
633+
int cap;
634+
} CBMDiagramArrowArray;
635+
615636
typedef struct {
616637
CBMFieldType *items;
617638
int count;
@@ -661,6 +682,7 @@ typedef struct CBMFileResult {
661682
CBMStringRefArray string_refs; // URL/config string literals from AST
662683
CBMInfraBindingArray infra_bindings; // topic→URL pairs from IaC configs
663684
CBMChannelArray channels; // Socket.IO / EventEmitter pub/sub participation
685+
CBMDiagramArrowArray diagram_arrows; // arrows between the file's DiagramElement defs
664686
CBMFieldTypeArray field_types; // Python: annotated instance fields (#1277)
665687

666688
const char *module_qn; // module qualified name
@@ -1122,6 +1144,7 @@ void cbm_infrabinding_push(CBMInfraBindingArray *arr, CBMArena *a, CBMInfraBindi
11221144
void cbm_impltrait_push(CBMImplTraitArray *arr, CBMArena *a, CBMImplTrait it);
11231145
void cbm_resolvedcall_push(CBMResolvedCallArray *arr, CBMArena *a, CBMResolvedCall rc);
11241146
void cbm_channels_push(CBMChannelArray *arr, CBMArena *a, CBMChannel ch);
1147+
void cbm_diagram_arrows_push(CBMDiagramArrowArray *arr, CBMArena *a, CBMDiagramArrow da);
11251148

11261149
// --- Sub-extractor entry points ---
11271150

@@ -1140,6 +1163,31 @@ void cbm_extract_type_refs(CBMExtractCtx *ctx);
11401163
void cbm_extract_env_accesses(CBMExtractCtx *ctx);
11411164
void cbm_extract_type_assigns(CBMExtractCtx *ctx);
11421165
void cbm_extract_channels(CBMExtractCtx *ctx);
1166+
/* Diagrams (diagram.c): a Diagram definition per diagram -- a Markdown, reST
1167+
* or AsciiDoc block, or a diagram file -- with a DiagramElement definition per
1168+
* element and the arrows between them. */
1169+
void cbm_extract_diagrams(CBMExtractCtx *ctx);
1170+
/* Is a Markdown block a diagram fence, and the labels its parser reads from
1171+
* it (space-joined; NULL for none): a Section's text keeps those labels in
1172+
* place of the fence (extract_markdown_section_body). */
1173+
bool cbm_diagram_fence_is_diagram(const char *source, TSNode block);
1174+
const char *cbm_diagram_fence_labels(CBMExtractCtx *ctx, TSNode block);
1175+
/* A diagram file (CBM_LANG_DIAGRAM): its format comes from its extension. */
1176+
void cbm_extract_diagram_file(CBMExtractCtx *ctx);
1177+
1178+
/* OpenAPI / Swagger documents (openapi.c): a Route definition per operation and
1179+
* a doc link per operationId. Self-gated: openapi.json / swagger.json, and YAML
1180+
* or JSON files whose root says `openapi` or `swagger`. */
1181+
void cbm_extract_openapi(CBMExtractCtx *ctx);
1182+
1183+
/* Canonicalize route-path parameter placeholders (":id", "{id}", "<id>",
1184+
* "${...}") to a single "{}" token so that client call sites, server handlers
1185+
* and OpenAPI operations rendezvous on the same Route QN regardless of
1186+
* framework syntax. Parameter names are intentionally discarded ("/u/{id}" and
1187+
* "/u/{slug}" both canonicalize to "/u/{}"). The result never exceeds the
1188+
* input length, so out_sz >= strlen(in) + 1 always suffices. Returns out.
1189+
* (route_path.c) */
1190+
const char *cbm_route_canon_path(const char *in, char *out, size_t out_sz);
11431191

11441192
// Single-pass unified extraction (replaces the 7 calls above except defs+imports).
11451193
void cbm_extract_unified(CBMExtractCtx *ctx);

0 commit comments

Comments
 (0)