Skip to content

Commit e3e268a

Browse files
authored
Merge pull request #623 from NVIDIA/codex/fix-missing-python-ast-columns
fix(analyzers): retain findings with unavailable Python AST columns
2 parents cb458ff + 7889241 commit e3e268a

4 files changed

Lines changed: 283 additions & 25 deletions

File tree

‎src/skillspector/nodes/analyzers/behavioral_ast.py‎

Lines changed: 30 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -437,6 +437,7 @@ def context_for(lineno: int, column: int) -> str:
437437
return context
438438

439439
def _emit(
440+
node_index: int,
440441
rule_id: str,
441442
ast_node: ast.Call,
442443
msg_override: str | None = None,
@@ -446,8 +447,8 @@ def _emit(
446447
complete_match = python_ast.source_segment(ast_node)
447448
if complete_match is None:
448449
complete_match = get_complete_source_segment(lines, lineno, end_lineno)
449-
start_byte_column = getattr(ast_node, "col_offset", 0)
450-
end_byte_column = getattr(ast_node, "end_col_offset", start_byte_column)
450+
start_byte_column = getattr(ast_node, "col_offset", None)
451+
end_byte_column = getattr(ast_node, "end_col_offset", None)
451452
start_column = python_ast.character_column(lineno, start_byte_column)
452453
end_column = python_ast.character_column(end_lineno or lineno, end_byte_column)
453454
finding = AnalyzerFinding(
@@ -466,13 +467,20 @@ def _emit(
466467
context=context_for(lineno, start_column if start_column is not None else 0),
467468
matched_text=complete_match[:200],
468469
complete_match=complete_match,
470+
# Evidence participates in report compaction. Keep distinct syntax
471+
# nodes distinguishable when their source spans are incomplete.
472+
evidence=(
473+
{"python_ast_node_index": node_index}
474+
if start_column is None or end_column is None
475+
else {}
476+
),
469477
)
470478
if budget is None:
471479
findings.append(finding)
472480
else:
473481
budget.emit(finding)
474482

475-
for ast_node in ast.walk(tree):
483+
for node_index, ast_node in enumerate(ast.walk(tree)):
476484
if budget is not None:
477485
budget.check_runtime()
478486
if isinstance(ast_node, ast.Subscript):
@@ -482,13 +490,15 @@ def _emit(
482490
if isinstance(key, ast.Constant):
483491
if isinstance(key.value, str) and key.value in _DANGEROUS_GETATTR_NAMES:
484492
_emit(
493+
node_index,
485494
"AST9",
486495
ast_node,
487496
f"Reflective dangerous access via {module}.__dict__ subscript "
488497
"with a literal sink name",
489498
)
490499
else:
491500
_emit(
501+
node_index,
492502
"AST7",
493503
ast_node,
494504
f"Dynamic attribute access via {module}.__dict__ subscript",
@@ -513,13 +523,15 @@ def _emit(
513523
if isinstance(key, ast.Constant):
514524
if isinstance(key.value, str) and key.value in _DANGEROUS_GETATTR_NAMES:
515525
_emit(
526+
node_index,
516527
"AST9",
517528
ast_node,
518529
f"Reflective dangerous access via {module}.__dict__.{method}() "
519530
"with a literal sink name",
520531
)
521532
else:
522533
_emit(
534+
node_index,
523535
"AST7",
524536
ast_node,
525537
f"Dynamic attribute access via {module}.__dict__.{method}()",
@@ -542,8 +554,10 @@ def _emit(
542554
budget.check_runtime if budget is not None else None,
543555
)
544556
if source:
545-
_emit("AST8", ast_node, f"Dangerous chain: exec() wrapping {source}")
546-
_emit("AST1", ast_node)
557+
_emit(
558+
node_index, "AST8", ast_node, f"Dangerous chain: exec() wrapping {source}"
559+
)
560+
_emit(node_index, "AST1", ast_node)
547561

548562
elif call_name == "eval":
549563
if _is_chain_sink(ast_node, aliases) and ast_node.args:
@@ -553,27 +567,29 @@ def _emit(
553567
budget.check_runtime if budget is not None else None,
554568
)
555569
if source:
556-
_emit("AST8", ast_node, f"Dangerous chain: eval() wrapping {source}")
557-
_emit("AST2", ast_node)
570+
_emit(
571+
node_index, "AST8", ast_node, f"Dangerous chain: eval() wrapping {source}"
572+
)
573+
_emit(node_index, "AST2", ast_node)
558574

559575
elif call_name == "__import__":
560-
_emit("AST3", ast_node)
576+
_emit(node_index, "AST3", ast_node)
561577

562578
elif call_name == "compile":
563-
_emit("AST6", ast_node)
579+
_emit(node_index, "AST6", ast_node)
564580

565581
elif call_name.startswith("subprocess."):
566582
attr = call_name.split(".", 1)[1]
567583
if attr in _SUBPROCESS_CALLS:
568-
_emit("AST4", ast_node)
584+
_emit(node_index, "AST4", ast_node)
569585

570586
elif call_name.startswith("os."):
571587
attr = call_name.split(".", 1)[1]
572588
if attr in _OS_EXEC_CALLS:
573-
_emit("AST5", ast_node)
589+
_emit(node_index, "AST5", ast_node)
574590

575591
elif (deser_msg := _deserialization_message(call_name, ast_node)) is not None:
576-
_emit("AST10", ast_node, deser_msg)
592+
_emit(node_index, "AST10", ast_node, deser_msg)
577593

578594
elif call_name == "getattr" and len(ast_node.args) >= 2:
579595
second_arg = ast_node.args[1]
@@ -584,9 +600,9 @@ def _emit(
584600
if isinstance(second_arg, ast.Constant) and not isinstance(second_arg.value, str):
585601
continue
586602
if resolved_name in _DANGEROUS_GETATTR_NAMES:
587-
_emit("AST9", ast_node)
603+
_emit(node_index, "AST9", ast_node)
588604
elif resolved_name is None or not isinstance(second_arg, ast.Constant):
589-
_emit("AST7", ast_node)
605+
_emit(node_index, "AST7", ast_node)
590606

591607
return findings if budget is None else list(budget.current_findings)
592608

‎src/skillspector/nodes/analyzers/behavioral_taint_tracking.py‎

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -465,7 +465,7 @@ def _analyze_python(
465465
lines = python_ast.lines
466466
findings: list[AnalyzerFinding] = []
467467
tainted: dict[str, _TaintedVar] = {}
468-
seen: set[tuple[str, int, int, int | None, int | None]] = set()
468+
seen: set[tuple[str, ast.Call]] = set()
469469
contexts: dict[int, str] = {}
470470

471471
def context_for(lineno: int) -> str:
@@ -476,27 +476,26 @@ def context_for(lineno: int) -> str:
476476
return context
477477

478478
def _emit(
479+
node_index: int,
479480
rule_id: str,
480481
ast_node: ast.Call,
481482
msg: str,
482483
) -> None:
483484
lineno = getattr(ast_node, "lineno", 1)
484485
end_lineno = getattr(ast_node, "end_lineno", None)
485-
start_byte_column = getattr(ast_node, "col_offset", 0)
486+
start_byte_column = getattr(ast_node, "col_offset", None)
486487
end_byte_column = getattr(ast_node, "end_col_offset", None)
487-
key = (rule_id, lineno, start_byte_column, end_lineno, end_byte_column)
488+
# Deduplicate flows into the same sink without merging distinct nodes
489+
# whose optional source columns are unavailable.
490+
key = (rule_id, ast_node)
488491
if key in seen:
489492
return
490493
seen.add(key)
491494
complete_match = python_ast.source_segment(ast_node)
492495
if complete_match is None:
493496
complete_match = get_complete_source_segment(lines, lineno, end_lineno)
494497
start_column = python_ast.character_column(lineno, start_byte_column)
495-
end_column = (
496-
python_ast.character_column(end_lineno or lineno, end_byte_column)
497-
if end_byte_column is not None
498-
else None
499-
)
498+
end_column = python_ast.character_column(end_lineno or lineno, end_byte_column)
500499
finding = AnalyzerFinding(
501500
rule_id=rule_id,
502501
message=msg,
@@ -513,13 +512,20 @@ def _emit(
513512
context=context_for(lineno),
514513
matched_text=complete_match[:200],
515514
complete_match=complete_match,
515+
# Evidence participates in report compaction. Keep distinct syntax
516+
# nodes distinguishable when their source spans are incomplete.
517+
evidence=(
518+
{"python_ast_node_index": node_index}
519+
if start_column is None or end_column is None
520+
else {}
521+
),
516522
)
517523
if budget is None:
518524
findings.append(finding)
519525
else:
520526
budget.emit(finding)
521527

522-
for ast_node in ast.walk(tree):
528+
for node_index, ast_node in enumerate(ast.walk(tree)):
523529
if budget is not None:
524530
budget.check_runtime()
525531
# Record tainted assignments.
@@ -577,6 +583,7 @@ def _emit(
577583
src_cat = _classify(src_name, _SOURCE_CATEGORIES, "data source")
578584
sink_cat = _classify(sink_name, _SINK_CATEGORIES, "data sink")
579585
_emit(
586+
node_index,
580587
rule,
581588
ast_node,
582589
f"Direct flow: {src_name} ({src_cat}) \u2192 {sink_name} ({sink_cat})",
@@ -591,6 +598,7 @@ def _emit(
591598
src_cat = _classify(tv.source_call, _SOURCE_CATEGORIES, "data source")
592599
sink_cat = _classify(sink_name, _SINK_CATEGORIES, "data sink")
593600
_emit(
601+
node_index,
594602
rule,
595603
ast_node,
596604
f"Tainted flow: '{tv.name}' from {tv.source_call} (line {tv.lineno}, "

‎src/skillspector/python_ast.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -125,8 +125,10 @@ def source_segment(self, node: ast.AST) -> str | None:
125125
except UnicodeDecodeError:
126126
return None
127127

128-
def character_column(self, lineno: int, byte_column: int) -> int | None:
129-
"""Convert one CPython UTF-8 byte column to a public character column."""
128+
def character_column(self, lineno: int, byte_column: int | None) -> int | None:
129+
"""Convert a UTF-8 byte column, retaining unavailable columns as None."""
130+
if byte_column is None:
131+
return None
130132
line_index = lineno - 1
131133
if line_index < 0 or line_index >= len(self.line_byte_starts):
132134
return None

0 commit comments

Comments
 (0)