Skip to content

Commit 10fa82f

Browse files
efegokdemirrng1995
andauthored
fix(analyzer): report malformed dependency manifests (#773)
* fix(analyzer): report malformed dependency manifests Signed-off-by: Efe <[email protected]> * fix(analyzer): report malformed manifests as incomplete Signed-off-by: Efe <[email protected]> * fix(analyzer): handle dependency manifest edge cases Avoid parsing documentation lookalikes as manifests, report empty JSON manifests, and keep oversized TOML values from aborting analysis. Signed-off-by: Efe Gökdemir <[email protected]> * fix: retain nested requirements manifest coverage Signed-off-by: Efe Gökdemir <[email protected]> --------- Signed-off-by: Efe <[email protected]> Signed-off-by: Efe Gökdemir <[email protected]> Signed-off-by: Narendran Raghavan <[email protected]> Co-authored-by: Narendran Raghavan <[email protected]>
1 parent 290fe93 commit 10fa82f

7 files changed

Lines changed: 392 additions & 45 deletions

File tree

‎docs/ANALYSIS_RESOURCE_BOUNDS.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,7 @@ malicious evasion. Keep required references and use the reason to choose the fix
123123

124124
| Reason | Next step |
125125
|---|---|
126+
| `dependency_parse_error` | Correct the malformed dependency manifest or lockfile, then rerun the scan. |
126127
| `static_parse_limit` | Inspect the expression and analyzer. If valid source is misinterpreted, correct or update the scanner and rerun. |
127128
| `read_error`, `stat_error`, `file_disappeared`, `missing_file_cache` | Ensure the resolved target remains readable throughout the scan. |
128129
| `size_limit`, `runtime_limit` | Review the reported bounds and input size; distinguish a scanner performance problem from a legitimate resource ceiling. |

‎src/skillspector/dependency_sources.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1335,7 +1335,7 @@ def _parse_poetry(
13351335
scan.check()
13361336
try:
13371337
parsed = tomllib.loads(content)
1338-
except tomllib.TOMLDecodeError:
1338+
except (ValueError, RecursionError):
13391339
return
13401340
tool = parsed.get("tool", {})
13411341
if not isinstance(tool, dict):
@@ -1427,7 +1427,7 @@ def _parse_cargo(
14271427
scan.check()
14281428
try:
14291429
parsed = tomllib.loads(content)
1430-
except tomllib.TOMLDecodeError:
1430+
except (ValueError, RecursionError):
14311431
return
14321432
sources = parsed.get("source", {})
14331433
if isinstance(sources, dict):

‎src/skillspector/inspection_ledger.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -111,8 +111,8 @@ class LedgerReason(StrEnum):
111111
REFERENCE_EXTRACTION_LIMIT = "reference_extraction_limit"
112112
REFERENCE_MISSING = "reference_missing"
113113
REFERENCE_UNRESOLVED = "reference_unresolved"
114-
MANIFEST_PARSE_ERROR = "manifest_parse_error"
115114
DEPENDENCY_PARSE_ERROR = "dependency_parse_error"
115+
MANIFEST_PARSE_ERROR = "manifest_parse_error"
116116
DEPENDENCY_LOCKFILE_MISSING = "dependency_lockfile_missing"
117117
DEPENDENCY_UNSUPPORTED_LOCKFILE = "dependency_unsupported_lockfile"
118118
DEPENDENCY_LOCKFILE_MISMATCH = "dependency_lockfile_mismatch"
@@ -273,6 +273,9 @@ def only_missing_reference_exceptions(exceptions: object) -> bool:
273273
"A local path-like reference matched more than one bundled artifact"
274274
" and could not be resolved to a single target."
275275
),
276+
LedgerReason.DEPENDENCY_PARSE_ERROR: (
277+
"Dependency metadata could not be parsed into supported registry coordinates."
278+
),
276279
LedgerReason.MANIFEST_PARSE_ERROR: (
277280
"Manifest frontmatter is malformed or uses an unsupported value shape."
278281
),

‎src/skillspector/nodes/analyzers/static_patterns_supply_chain.py‎

Lines changed: 97 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -1508,6 +1508,7 @@ def _extract_packages_from_package_json(
15081508
content: str,
15091509
*,
15101510
limit: int | None = None,
1511+
on_parse_error: Callable[[str], None] | None = None,
15111512
) -> list[tuple[str, str | None, int]]:
15121513
"""Extract (package_name, version_or_None, line_number) from package.json content.
15131514
@@ -1520,7 +1521,9 @@ def _extract_packages_from_package_json(
15201521
return []
15211522
try:
15221523
data = json.loads(content)
1523-
except (ValueError, TypeError):
1524+
except (ValueError, TypeError) as error:
1525+
if on_parse_error is not None:
1526+
on_parse_error(type(error).__name__)
15241527
return _extract_packages_from_package_json_scan(content, limit=limit)
15251528
if not isinstance(data, dict):
15261529
return []
@@ -1564,6 +1567,7 @@ def _extract_packages_from_pyproject(
15641567
content: str,
15651568
*,
15661569
limit: int | None = None,
1570+
on_parse_error: Callable[[str], None] | None = None,
15671571
) -> list[tuple[str, str | None, int]]:
15681572
"""Extract (package_name, version_or_None, line_number) from pyproject.toml.
15691573
@@ -1576,7 +1580,9 @@ def _extract_packages_from_pyproject(
15761580
return []
15771581
try:
15781582
data = tomllib.loads(content)
1579-
except tomllib.TOMLDecodeError:
1583+
except (ValueError, RecursionError) as error:
1584+
if on_parse_error is not None:
1585+
on_parse_error(type(error).__name__)
15801586
return []
15811587

15821588
specs: list[str] = []
@@ -1836,9 +1842,28 @@ def _normalize_package_name(name: str) -> str:
18361842
return name.lower().replace("_", "-")
18371843

18381844

1845+
def _dependency_filename(file_path: str) -> str:
1846+
"""Return a normalized basename so documentation is not mistaken for a manifest."""
1847+
return file_path.replace("\\", "/").rsplit("/", 1)[-1].lower()
1848+
1849+
1850+
def _is_requirements_file(file_path: str) -> bool:
1851+
normalized_path = file_path.replace("\\", "/").lower()
1852+
path_parts = normalized_path.split("/")
1853+
filename = path_parts[-1]
1854+
if filename in {"requirements", "requirements.txt", "requirements.in"} or re.fullmatch(
1855+
r"requirements[-_][a-z0-9_-]+\.(?:txt|in)", filename
1856+
):
1857+
return True
1858+
return "requirements" in path_parts[:-1] and bool(
1859+
re.fullmatch(r"[a-z0-9][a-z0-9_-]*\.(?:txt|in)", filename)
1860+
)
1861+
1862+
18391863
def _is_python_lockfile(file_path: str) -> bool:
1840-
lower_path = file_path.lower()
1841-
return "uv.lock" in lower_path or "poetry.lock" in lower_path or _is_pipenv_lockfile(file_path)
1864+
return _dependency_filename(file_path) in {"uv.lock", "poetry.lock"} or _is_pipenv_lockfile(
1865+
file_path
1866+
)
18421867

18431868

18441869
def _normalize_npm_package_name(name: str) -> str:
@@ -1853,8 +1878,18 @@ def _normalize_npm_package_name(name: str) -> str:
18531878

18541879

18551880
def _is_npm_lockfile(file_path: str) -> bool:
1856-
lower_path = file_path.lower()
1857-
return lower_path.endswith(("package-lock.json", "npm-shrinkwrap.json"))
1881+
return _dependency_filename(file_path) in {"package-lock.json", "npm-shrinkwrap.json"}
1882+
1883+
1884+
def _is_dependency_manifest(file_path: str) -> bool:
1885+
"""Recognize supported manifest names, not prose files that merely mention them."""
1886+
filename = _dependency_filename(file_path)
1887+
return (
1888+
_is_requirements_file(file_path)
1889+
or filename in {"package.json", "pyproject.toml", "setup.py", "pipfile", "pipfile.lock"}
1890+
or _is_python_lockfile(file_path)
1891+
or _is_npm_lockfile(file_path)
1892+
)
18581893

18591894

18601895
def _npm_lock_line_index(content: str) -> dict[str, int]:
@@ -1874,7 +1909,11 @@ def _npm_lock_line_index(content: str) -> dict[str, int]:
18741909
return index
18751910

18761911

1877-
def _npm_lock_entries(content: str) -> list[tuple[str, str, int, int]]:
1912+
def _npm_lock_entries(
1913+
content: str,
1914+
*,
1915+
on_parse_error: Callable[[str], None] | None = None,
1916+
) -> list[tuple[str, str, int, int]]:
18781917
"""Extract (name, version, line, depth) for every install recorded in an npm lockfile.
18791918
18801919
Covers both layouts: ``lockfileVersion`` 2/3 keep every install under ``packages`` keyed by
@@ -1891,7 +1930,11 @@ def _npm_lock_entries(content: str) -> list[tuple[str, str, int, int]]:
18911930
"""
18921931
try:
18931932
data = json.loads(content)
1894-
except (ValueError, TypeError):
1933+
except (ValueError, RecursionError) as error:
1934+
if on_parse_error is not None:
1935+
on_parse_error(type(error).__name__)
1936+
return []
1937+
except TypeError:
18951938
return []
18961939
if not isinstance(data, dict):
18971940
return []
@@ -1941,12 +1984,14 @@ def _extract_packages_from_npm_lock(
19411984
content: str,
19421985
*,
19431986
limit: int | None = None,
1987+
on_parse_error: Callable[[str], None] | None = None,
19441988
) -> list[tuple[str, str | None, int]]:
19451989
"""Extract exact package versions from an npm lockfile."""
19461990
if limit is not None and limit <= 0:
19471991
return []
19481992
found: list[tuple[str, str | None, int]] = [
1949-
(name, version, line) for name, version, line, _depth in _npm_lock_entries(content)
1993+
(name, version, line)
1994+
for name, version, line, _depth in _npm_lock_entries(content, on_parse_error=on_parse_error)
19501995
]
19511996
return found if limit is None else found[:limit]
19521997

@@ -1955,13 +2000,16 @@ def _extract_packages_from_toml_lock(
19552000
content: str,
19562001
*,
19572002
limit: int | None = None,
2003+
on_parse_error: Callable[[str], None] | None = None,
19582004
) -> list[tuple[str, str | None, int]]:
19592005
"""Extract exact package versions from TOML lockfiles such as uv.lock and poetry.lock."""
19602006
if limit is not None and limit <= 0:
19612007
return []
19622008
try:
19632009
data = tomllib.loads(content)
1964-
except tomllib.TOMLDecodeError:
2010+
except (ValueError, RecursionError) as error:
2011+
if on_parse_error is not None:
2012+
on_parse_error(type(error).__name__)
19652013
return []
19662014
packages = data.get("package")
19672015
if not isinstance(packages, list):
@@ -2164,7 +2212,7 @@ def _collect_npm_locked_versions(
21642212
if timeout_seconds is not None and time.monotonic() - started_at >= timeout_seconds:
21652213
break
21662214
content = file_cache.get(path)
2167-
if not content:
2215+
if content is None:
21682216
continue
21692217
files_read += 1
21702218
for name, version, _line_num, depth in _npm_lock_entries(content):
@@ -2393,10 +2441,7 @@ def line_number(start: int) -> int:
23932441

23942442
tag = [PatternCategory.SUPPLY_CHAIN.value]
23952443

2396-
is_dep_file = any(
2397-
n in file_path.lower()
2398-
for n in ["requirements", "package.json", "pyproject.toml", "setup.py", "pipfile"]
2399-
)
2444+
is_dep_file = _is_dependency_manifest(file_path)
24002445
if is_dep_file:
24012446
for pattern, confidence in SC1_PATTERNS:
24022447
matches = (
@@ -3148,14 +3193,15 @@ def _analyze_dependencies_detailed(
31483193
limitations: list[OsvQueryLimitation] = []
31493194
tag = [PatternCategory.SUPPLY_CHAIN.value]
31503195

3151-
lower_path = file_path.lower()
3152-
is_lockfile = _is_python_lockfile(lower_path)
3153-
is_npm_lock = _is_npm_lockfile(lower_path)
3196+
filename = _dependency_filename(file_path)
3197+
is_lockfile = _is_python_lockfile(file_path)
3198+
is_npm_lock = _is_npm_lockfile(file_path)
31543199
is_python_dep = (
3155-
any(n in lower_path for n in ["requirements", "pyproject.toml", "setup.py", "pipfile"])
3200+
_is_requirements_file(file_path)
3201+
or filename in {"pyproject.toml", "setup.py", "pipfile", "pipfile.lock"}
31563202
or is_lockfile
31573203
)
3158-
is_npm_dep = "package.json" in lower_path or is_npm_lock
3204+
is_npm_dep = filename == "package.json" or is_npm_lock
31593205

31603206
if not is_python_dep and not is_npm_dep:
31613207
return findings, limitations, 0
@@ -3169,18 +3215,34 @@ def _analyze_dependencies_detailed(
31693215
)
31703216
extraction_limit = package_limit + 1
31713217

3218+
def record_parse_error(error_class: str) -> None:
3219+
limitations.append(
3220+
OsvQueryLimitation(
3221+
reason=LedgerReason.DEPENDENCY_PARSE_ERROR,
3222+
error_class=error_class,
3223+
)
3224+
)
3225+
31723226
if is_python_dep:
3173-
if Path(file_path).name.lower() in {"pipfile", "pipfile.lock"}:
3227+
if filename in {"pipfile", "pipfile.lock"}:
31743228
packages, parse_limitations = _extract_packages_from_pipenv(
31753229
content,
31763230
is_lockfile=_is_pipenv_lockfile(file_path),
31773231
limit=extraction_limit,
31783232
)
31793233
limitations.extend(parse_limitations)
3180-
elif "pyproject.toml" in lower_path:
3181-
packages = _extract_packages_from_pyproject(content, limit=extraction_limit)
3234+
elif filename == "pyproject.toml":
3235+
packages = _extract_packages_from_pyproject(
3236+
content,
3237+
limit=extraction_limit,
3238+
on_parse_error=record_parse_error,
3239+
)
31823240
elif is_lockfile:
3183-
packages = _extract_packages_from_toml_lock(content, limit=extraction_limit)
3241+
packages = _extract_packages_from_toml_lock(
3242+
content,
3243+
limit=extraction_limit,
3244+
on_parse_error=record_parse_error,
3245+
)
31843246
else:
31853247
packages, oversized_spec = _extract_packages_from_requirements_detailed(
31863248
content,
@@ -3202,10 +3264,18 @@ def _analyze_dependencies_detailed(
32023264
known_legit = _KNOWN_LEGIT_PYPI
32033265
else:
32043266
if is_npm_lock:
3205-
packages = _extract_packages_from_npm_lock(content, limit=extraction_limit)
3267+
packages = _extract_packages_from_npm_lock(
3268+
content,
3269+
limit=extraction_limit,
3270+
on_parse_error=record_parse_error,
3271+
)
32063272
else:
32073273
packages = _apply_locked_versions(
3208-
_extract_packages_from_package_json(content, limit=extraction_limit),
3274+
_extract_packages_from_package_json(
3275+
content,
3276+
limit=extraction_limit,
3277+
on_parse_error=record_parse_error,
3278+
),
32093279
npm_locked_versions,
32103280
_normalize_npm_package_name,
32113281
)
@@ -4134,21 +4204,7 @@ def dependency_remaining_seconds() -> float:
41344204
dependency_findings_seen = 0
41354205
osv_budget = OsvQueryBudget.create(dependency_remaining_seconds())
41364206
for path in components:
4137-
lower_path = path.lower()
4138-
is_dep_file = any(
4139-
n in lower_path
4140-
for n in [
4141-
"requirements",
4142-
"package.json",
4143-
"package-lock.json",
4144-
"npm-shrinkwrap.json",
4145-
"pyproject.toml",
4146-
"setup.py",
4147-
"pipfile",
4148-
"uv.lock",
4149-
"poetry.lock",
4150-
]
4151-
)
4207+
is_dep_file = _is_dependency_manifest(path)
41524208
if not is_dep_file:
41534209
continue
41544210
dependency_files_seen += 1

‎tests/nodes/analyzers/test_dependency_source_review_regressions.py‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,39 @@ def test_non_mapping_toml_tool_preserves_earlier_supply_chain_results(monkeypatc
9898
)
9999

100100

101+
def test_oversized_toml_integer_does_not_drop_existing_supply_chain_findings(monkeypatch):
102+
earlier = Finding(
103+
rule_id="SC5", severity="HIGH", file="requirements.txt", message="Existing risk"
104+
)
105+
monkeypatch.setattr(
106+
supply_chain.static_runner,
107+
"run_static_patterns_with_ledger",
108+
lambda *_args: {
109+
"findings": [earlier],
110+
"inspection_ledger": [],
111+
"analyzer_status_events": [],
112+
},
113+
)
114+
files = {
115+
"requirements.txt": "risky-package==1.0.0\n",
116+
"pyproject.toml": "[project]\nname = " + "9" * 5_000,
117+
}
118+
119+
response = supply_chain.node({"components": list(files), "file_cache": files})
120+
121+
assert response["findings"] == [earlier]
122+
assert not any(
123+
event["outcome"] is LedgerOutcome.FAILED for event in response["inspection_ledger"]
124+
)
125+
126+
127+
@pytest.mark.parametrize("path", ["pyproject.toml", ".cargo/config.toml"])
128+
def test_oversized_toml_integer_does_not_abort_dependency_source_scan(path):
129+
content = "value = " + "9" * 5_000
130+
131+
assert analyze_dependency_sources([path], {path: content}) == []
132+
133+
101134
@pytest.mark.parametrize(
102135
("target", "canonical", "body"),
103136
[

0 commit comments

Comments
 (0)