|
17 | 17 |
|
18 | 18 | from __future__ import annotations |
19 | 19 |
|
| 20 | +from pathlib import Path |
| 21 | +from unittest.mock import patch |
| 22 | + |
20 | 23 | import pytest |
21 | 24 | from langchain_anthropic import ChatAnthropic |
22 | 25 | from langchain_core.exceptions import OutputParserException |
23 | 26 | from langchain_core.messages import AIMessage |
24 | 27 | from langchain_core.outputs import ChatGeneration, ChatResult |
25 | 28 | from langchain_openai import ChatOpenAI |
26 | 29 | from pydantic import BaseModel |
| 30 | +from typer.testing import CliRunner |
27 | 31 |
|
| 32 | +from skillspector.cli import app |
| 33 | +from skillspector.constants import _MODEL_SLOTS |
28 | 34 | from skillspector.inference_usage import chat_model_controls, chat_model_requested_controls |
29 | | -from skillspector.llm_utils import StructuredOutputParseError, bind_structured_output |
| 35 | +from skillspector.llm_utils import ( |
| 36 | + StructuredOutputParseError, |
| 37 | + bind_structured_output, |
| 38 | + is_llm_available, |
| 39 | + unsupported_control_error, |
| 40 | +) |
30 | 41 | from skillspector.providers import registry |
31 | 42 | from skillspector.providers.anthropic import AnthropicProvider |
32 | 43 | from skillspector.providers.anthropic_proxy import AnthropicProxyProvider |
@@ -77,6 +88,7 @@ def _clean_env(monkeypatch: pytest.MonkeyPatch): |
77 | 88 | "AZURE_OPENAI_API_KEY", |
78 | 89 | "AZURE_OPENAI_ENDPOINT", |
79 | 90 | "AZURE_OPENAI_DEPLOYMENT", |
| 91 | + *(f"SKILLSPECTOR_MODEL_{slot.upper()}" for slot in _MODEL_SLOTS), |
80 | 92 | ): |
81 | 93 | monkeypatch.delenv(name, raising=False) |
82 | 94 | registry._load.cache_clear() |
@@ -531,3 +543,96 @@ def test_other_models_keep_temperature(self, monkeypatch: pytest.MonkeyPatch) -> |
531 | 543 | monkeypatch.setenv("SKILLSPECTOR_TEMPERATURE", "0.2") |
532 | 544 | llm = AzureOpenAIProvider().create_chat_model("gpt-4o", max_tokens=1_000) |
533 | 545 | assert chat_model_requested_controls(llm)["temperature"] == 0.2 |
| 546 | + |
| 547 | + |
| 548 | +_DEFAULT_PROVIDER_ENV = pytest.mark.parametrize( |
| 549 | + ("provider", "credential", "model"), |
| 550 | + [ |
| 551 | + ("openai", "OPENAI_API_KEY", "gpt-6.1-sol"), |
| 552 | + ("anthropic", "ANTHROPIC_API_KEY", "claude-opus-5-5"), |
| 553 | + ], |
| 554 | +) |
| 555 | + |
| 556 | + |
| 557 | +class TestScanPreflightControls: |
| 558 | + """A rejected control stops the scan before analysis and names the setting.""" |
| 559 | + |
| 560 | + @_DEFAULT_PROVIDER_ENV |
| 561 | + def test_scan_exits_before_analysis_with_the_reason( |
| 562 | + self, |
| 563 | + monkeypatch: pytest.MonkeyPatch, |
| 564 | + tmp_path: Path, |
| 565 | + provider: str, |
| 566 | + credential: str, |
| 567 | + model: str, |
| 568 | + ) -> None: |
| 569 | + (tmp_path / "SKILL.md").write_text("---\nname: demo\n---\nHello.\n", encoding="utf-8") |
| 570 | + monkeypatch.setenv("SKILLSPECTOR_PROVIDER", provider) |
| 571 | + monkeypatch.setenv(credential, "sk-test") |
| 572 | + monkeypatch.setenv("SKILLSPECTOR_TEMPERATURE", "0") |
| 573 | + |
| 574 | + with patch("skillspector.cli.graph.invoke") as invoke: |
| 575 | + result = CliRunner().invoke(app, ["scan", str(tmp_path), "--format", "json"]) |
| 576 | + |
| 577 | + assert result.exit_code == 2 |
| 578 | + assert f"SKILLSPECTOR_TEMPERATURE is not supported by {model}" in result.output |
| 579 | + invoke.assert_not_called() |
| 580 | + |
| 581 | + @_DEFAULT_PROVIDER_ENV |
| 582 | + def test_preflight_reports_the_reason_as_llm_error( |
| 583 | + self, monkeypatch: pytest.MonkeyPatch, provider: str, credential: str, model: str |
| 584 | + ) -> None: |
| 585 | + monkeypatch.setenv("SKILLSPECTOR_PROVIDER", provider) |
| 586 | + monkeypatch.setenv(credential, "sk-test") |
| 587 | + monkeypatch.setenv("SKILLSPECTOR_TEMPERATURE", "0") |
| 588 | + |
| 589 | + available, error = is_llm_available() |
| 590 | + |
| 591 | + assert available is False |
| 592 | + assert error is not None and f"not supported by {model}; unset it" in error |
| 593 | + |
| 594 | + def test_sol_effort_is_checked_on_every_slot(self, monkeypatch: pytest.MonkeyPatch) -> None: |
| 595 | + monkeypatch.setenv("SKILLSPECTOR_PROVIDER", "openai") |
| 596 | + monkeypatch.setenv("OPENAI_API_KEY", "sk-test") |
| 597 | + monkeypatch.setenv("SKILLSPECTOR_REASONING_EFFORT", "minimal") |
| 598 | + monkeypatch.setenv("SKILLSPECTOR_MODEL", "gpt-5.4") |
| 599 | + monkeypatch.setenv("SKILLSPECTOR_MODEL_META_ANALYZER", "gpt-6.1-sol") |
| 600 | + |
| 601 | + error = unsupported_control_error() |
| 602 | + |
| 603 | + assert error is not None and "SKILLSPECTOR_REASONING_EFFORT='minimal'" in error |
| 604 | + |
| 605 | + def test_supported_controls_and_missing_credentials_pass( |
| 606 | + self, monkeypatch: pytest.MonkeyPatch |
| 607 | + ) -> None: |
| 608 | + monkeypatch.setenv("SKILLSPECTOR_PROVIDER", "openai") |
| 609 | + monkeypatch.setenv("SKILLSPECTOR_TEMPERATURE", "0") |
| 610 | + # Missing credentials are reported by the availability path, not as a control error. |
| 611 | + assert unsupported_control_error() is None |
| 612 | + |
| 613 | + monkeypatch.setenv("OPENAI_API_KEY", "sk-test") |
| 614 | + monkeypatch.setenv("SKILLSPECTOR_MODEL", "gpt-5.4") |
| 615 | + assert unsupported_control_error() is None |
| 616 | + assert is_llm_available() == (True, None) |
| 617 | + |
| 618 | + def test_cli_providers_are_not_checked(self, monkeypatch: pytest.MonkeyPatch) -> None: |
| 619 | + monkeypatch.setenv("SKILLSPECTOR_PROVIDER", "claude_cli") |
| 620 | + monkeypatch.setenv("SKILLSPECTOR_MODEL", "gpt-6.1-sol") |
| 621 | + monkeypatch.setenv("SKILLSPECTOR_TEMPERATURE", "0") |
| 622 | + assert unsupported_control_error() is None |
| 623 | + |
| 624 | + def test_no_llm_scan_ignores_rejected_controls( |
| 625 | + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path |
| 626 | + ) -> None: |
| 627 | + (tmp_path / "SKILL.md").write_text("---\nname: demo\n---\nHello.\n", encoding="utf-8") |
| 628 | + monkeypatch.setenv("SKILLSPECTOR_PROVIDER", "openai") |
| 629 | + monkeypatch.setenv("OPENAI_API_KEY", "sk-test") |
| 630 | + monkeypatch.setenv("SKILLSPECTOR_TEMPERATURE", "0") |
| 631 | + |
| 632 | + with patch("skillspector.cli.unsupported_control_error") as check: |
| 633 | + result = CliRunner().invoke( |
| 634 | + app, ["scan", str(tmp_path), "--format", "json", "--no-llm"] |
| 635 | + ) |
| 636 | + |
| 637 | + check.assert_not_called() |
| 638 | + assert "SKILLSPECTOR_TEMPERATURE" not in result.output |
0 commit comments