Skip to content

Commit 92e8e65

Browse files
kevin9327claude
andauthored
fix(input): download the raw file for GitHub and GitLab /blob/ URLs (#566)
The input handler already routes a forge "/blob/" URL to the file download path instead of a clone, but it downloaded the URL as given. On github.com and gitlab.com that URL is the HTML file viewer, so the scan analysed the forge's page markup as the skill: scanning a clean SKILL.md through its GitHub /blob/ URL reported a 286 KB SKILL.md and DO_NOT_INSTALL. Rewrite those URLs to the raw file (raw.githubusercontent.com for GitHub, /-/raw/ for GitLab) before downloading. Hosts stay inside the existing download allowlist, and other URLs are unchanged. Signed-off-by: kevin9327 <[email protected]> Co-authored-by: Claude Opus 5 <[email protected]>
1 parent 20fdba7 commit 92e8e65

2 files changed

Lines changed: 70 additions & 0 deletions

File tree

‎src/skillspector/input_handler.py‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,28 @@ def _is_private_ip(host: str) -> bool:
211211
return False
212212

213213

214+
def _raw_file_url(url: str) -> str:
215+
"""Point a GitHub or GitLab ``/blob/`` file page at the file's raw bytes.
216+
217+
Those pages are HTML viewers, so downloading one scans the forge's page
218+
markup instead of the file. Every other URL is returned unchanged.
219+
"""
220+
parsed = urlparse(url)
221+
host = (parsed.hostname or "").lower()
222+
segments = parsed.path.split("/")
223+
# /<owner>/<repo>/blob/<ref>/<path> -> raw.githubusercontent.com/<owner>/<repo>/<ref>/<path>
224+
if host == "github.com" and len(segments) > 5 and segments[3] == "blob":
225+
raw_path = "/".join(segments[:3] + segments[4:])
226+
return parsed._replace(netloc="raw.githubusercontent.com", path=raw_path).geturl()
227+
# /<namespace>/<project>/-/blob/<ref>/<path> -> /<namespace>/<project>/-/raw/<ref>/<path>
228+
if host == "gitlab.com" and "-" in segments[3:]:
229+
marker = segments.index("-", 3)
230+
if marker + 3 < len(segments) and segments[marker + 1] == "blob":
231+
segments[marker + 1] = "raw"
232+
return parsed._replace(path="/".join(segments)).geturl()
233+
return url
234+
235+
214236
def _root_owned_root_alias(path: Path) -> Path | None:
215237
"""Return a root-owned symlink directly below ``/``, if *path* is one."""
216238
absolute_path = Path(os.path.abspath(path))
@@ -1123,6 +1145,7 @@ def _download_file(self, url: str) -> Path:
11231145
partial file produced by a mid-stream breach is removed before
11241146
the exception propagates.
11251147
"""
1148+
url = _raw_file_url(url)
11261149
if self._transitive_budget is not None:
11271150
return self._download_transitive_file(url)
11281151
self._validate_url_host(url, ALLOWED_DOWNLOAD_HOSTS)

‎tests/unit/test_input_handler.py‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
from types import SimpleNamespace
2424
from unittest.mock import MagicMock, patch
2525

26+
import httpx
2627
import pytest
2728

2829
from skillspector.input_handler import (
@@ -32,6 +33,7 @@
3233
_open_regular_file_from_windows_handle,
3334
_open_regular_file_no_follow,
3435
)
36+
from skillspector.state import WorkflowResourceBudget
3537

3638

3739
def _mock_windows_secure_open(
@@ -460,6 +462,51 @@ def test_http_urls_are_not_accepted_as_remote_inputs() -> None:
460462
assert handler._is_file_url("http://raw.githubusercontent.com/org/repo/SKILL.md") is False
461463

462464

465+
@pytest.mark.parametrize("budgeted", [False, True], ids=["direct", "workflow-budget"])
466+
@pytest.mark.parametrize(
467+
("page_url", "raw_url"),
468+
[
469+
(
470+
"https://github.com/org/repo/blob/main/skills/demo/SKILL.md",
471+
"https://raw.githubusercontent.com/org/repo/main/skills/demo/SKILL.md",
472+
),
473+
(
474+
"https://gitlab.com/group/repo/-/blob/main/skills/demo/SKILL.md",
475+
"https://gitlab.com/group/repo/-/raw/main/skills/demo/SKILL.md",
476+
),
477+
],
478+
ids=["github", "gitlab"],
479+
)
480+
def test_file_page_url_downloads_the_raw_file(
481+
monkeypatch: pytest.MonkeyPatch, page_url: str, raw_url: str, budgeted: bool
482+
) -> None:
483+
"""A forge's /blob/ file page resolves to the file itself, not its HTML viewer."""
484+
skill = b"---\nname: demo\ndescription: demo\n---\n# Demo\n"
485+
requested: list[str] = []
486+
487+
def serve(request: httpx.Request) -> httpx.Response:
488+
requested.append(str(request.url))
489+
if str(request.url) == raw_url:
490+
return httpx.Response(200, content=skill, headers={"content-type": "text/plain"})
491+
return httpx.Response(200, content=b"<!DOCTYPE html><html></html>")
492+
493+
real_client = httpx.Client
494+
monkeypatch.setattr(
495+
"skillspector.input_handler.httpx.Client",
496+
lambda *args, **kwargs: real_client(*args, transport=httpx.MockTransport(serve), **kwargs),
497+
)
498+
monkeypatch.setattr("skillspector.input_handler._is_private_ip", lambda _host: False)
499+
handler = InputHandler(transitive_budget=WorkflowResourceBudget() if budgeted else None)
500+
try:
501+
resolved, source_type = handler.resolve(page_url)
502+
503+
assert source_type == "url"
504+
assert requested == [raw_url]
505+
assert (resolved / "SKILL.md").read_bytes() == skill
506+
finally:
507+
handler.cleanup()
508+
509+
463510
def test_validate_url_host_scp_extracts_github() -> None:
464511
"""_validate_url_host extracts 'github.com' from an scp-style URL."""
465512
with patch("skillspector.input_handler._is_private_ip", return_value=False):

0 commit comments

Comments
 (0)