|
| 1 | +from contextlib import contextmanager |
| 2 | +from types import SimpleNamespace |
| 3 | + |
1 | 4 | from opik import logging_messages, exceptions |
2 | 5 | from opik.evaluation.metrics.llm_judges.g_eval import parser |
| 6 | +from opik.evaluation.metrics.llm_judges.g_eval.metric import GEval |
| 7 | +from opik.evaluation.models import base_model |
3 | 8 | import pytest |
4 | 9 |
|
5 | 10 |
|
@@ -178,7 +183,12 @@ def test_duplicate_score_keys_last_wins_like_the_text_path(): |
178 | 183 | name="g_eval", |
179 | 184 | log_probs_supported=True, |
180 | 185 | ) |
181 | | - assert 0.55 < result.value < 0.68, f"scored from the stale key: {result.value}" |
| 186 | + # Exact weighted value of the second key's candidates (7 @ -0.1, |
| 187 | + # 1 @ -2.0): (7*e^-0.1 + e^-2)/(e^-0.1 + e^-2)/10. A first-match locator |
| 188 | + # scores the stale "9" key instead (0.88084...) and fails here. |
| 189 | + assert result.value == pytest.approx(0.6219349153822014, abs=1e-9), ( |
| 190 | + f"scored from the stale key: {result.value}" |
| 191 | + ) |
182 | 192 |
|
183 | 193 |
|
184 | 194 | def test_escaped_quote_echo_is_not_a_matchable_key(): |
@@ -207,4 +217,127 @@ def test_escaped_quote_echo_is_not_a_matchable_key(): |
207 | 217 | name="g_eval", |
208 | 218 | log_probs_supported=True, |
209 | 219 | ) |
210 | | - assert 0.55 < result.value < 0.68, f"unexpected value {result.value}" |
| 220 | + # Same distribution as the duplicate-key test pins the locator on the |
| 221 | + # real key; see the derivation there. |
| 222 | + assert result.value == pytest.approx(0.6219349153822014, abs=1e-9), ( |
| 223 | + f"unexpected value {result.value}" |
| 224 | + ) |
| 225 | + |
| 226 | + |
| 227 | +# --- provider-shaped GEval coverage ---------------------------------------- |
| 228 | +# The tests above call parse_litellm_model_output directly with hand-built |
| 229 | +# token dicts. These go through the public GEval.score LiteLLM branch with |
| 230 | +# a provider-shaped raw response, so an incorrectly bounded result on the |
| 231 | +# real path cannot pass unnoticed. |
| 232 | + |
| 233 | + |
| 234 | +def _provider_response(content, entries): |
| 235 | + """Provider-shaped raw LiteLLM response: a dict-style choice carrying the |
| 236 | + message content plus the logprobs token stream, as yielded by |
| 237 | + get_provider_response on the LiteLLM branch.""" |
| 238 | + return SimpleNamespace( |
| 239 | + choices=[{"message": {"content": content}, "logprobs": {"content": entries}}] |
| 240 | + ) |
| 241 | + |
| 242 | + |
| 243 | +def _geval_on_stubbed_provider(monkeypatch, content, entries): |
| 244 | + """GEval wired to a stubbed LiteLLM provider: chain-of-thought generation |
| 245 | + is stubbed, and the scoring call yields the given provider-shaped |
| 246 | + response. Returns (metric, captured_provider_kwargs).""" |
| 247 | + metric = GEval( |
| 248 | + task_introduction="provider-path split-score intro", |
| 249 | + evaluation_criteria="provider-path split-score criteria", |
| 250 | + model="gpt-4o", |
| 251 | + track=False, |
| 252 | + ) |
| 253 | + # The subject here is the logprob parser branch, not model capability |
| 254 | + # detection (covered by test_geval_passes_logprobs_only_when_supported). |
| 255 | + metric._log_probs_supported = True |
| 256 | + monkeypatch.setattr( |
| 257 | + metric._model, |
| 258 | + "generate_chat_completion", |
| 259 | + lambda *args, **kwargs: {"content": "stub chain of thought"}, |
| 260 | + ) |
| 261 | + captured = {} |
| 262 | + |
| 263 | + @contextmanager |
| 264 | + def fake_get_provider_response(model_provider, messages, **kwargs): |
| 265 | + captured.update(kwargs) |
| 266 | + yield _provider_response(content, entries) |
| 267 | + |
| 268 | + monkeypatch.setattr(base_model, "get_provider_response", fake_get_provider_response) |
| 269 | + return metric, captured |
| 270 | + |
| 271 | + |
| 272 | +def test_geval_litellm_path_split_score_scores_near_one(monkeypatch): |
| 273 | + # {"score":10} split across "1" + "0" must score near 1.0 end to end. A |
| 274 | + # legacy fixed-offset read of entries[3] alone averages to ~0.099. |
| 275 | + entries = [ |
| 276 | + _entry('{"', -0.01), |
| 277 | + _entry("score", -0.01), |
| 278 | + _entry('":', -0.01), |
| 279 | + _entry( |
| 280 | + "1", |
| 281 | + -0.05, |
| 282 | + top=[ |
| 283 | + {"token": "1", "logprob": -0.05}, |
| 284 | + {"token": "0", "logprob": -2.30}, |
| 285 | + {"token": "2", "logprob": -2.40}, |
| 286 | + ], |
| 287 | + ), |
| 288 | + _entry( |
| 289 | + "0", |
| 290 | + -0.02, |
| 291 | + top=[{"token": "0", "logprob": -0.02}, {"token": "1", "logprob": -3.00}], |
| 292 | + ), |
| 293 | + _entry(",", -0.01), |
| 294 | + _entry('"reason"', -0.01), |
| 295 | + _entry('":', -0.01), |
| 296 | + _entry(" ", -0.01), |
| 297 | + _entry('"excellent"', -0.01), |
| 298 | + _entry("}", -0.01), |
| 299 | + ] |
| 300 | + content = '{"score":10, "reason": "excellent"}' |
| 301 | + metric, captured = _geval_on_stubbed_provider(monkeypatch, content, entries) |
| 302 | + |
| 303 | + result = metric.score("any input") |
| 304 | + |
| 305 | + assert captured["logprobs"] is True |
| 306 | + assert captured["top_logprobs"] == 20 |
| 307 | + assert result.value == pytest.approx(0.9007723389857002, abs=1e-9) |
| 308 | + assert result.reason == "excellent" |
| 309 | + |
| 310 | + |
| 311 | +def test_geval_litellm_path_folded_whitespace_is_scored(monkeypatch): |
| 312 | + # The space after the colon folds into the score token (" 0"); the |
| 313 | + # end-to-end path must score it instead of raising |
| 314 | + # MetricComputationError on a perfectly parseable response. |
| 315 | + entries = [ |
| 316 | + _entry('{"', -0.01), |
| 317 | + _entry("score", -0.01), |
| 318 | + _entry('":', -0.01), |
| 319 | + _entry( |
| 320 | + " 0", |
| 321 | + -0.02, |
| 322 | + top=[ |
| 323 | + {"token": " 0", "logprob": -0.02}, |
| 324 | + {"token": " 1", "logprob": -2.00}, |
| 325 | + {"token": " 10", "logprob": -2.50}, |
| 326 | + ], |
| 327 | + ), |
| 328 | + _entry(",", -0.01), |
| 329 | + _entry('"reason"', -0.01), |
| 330 | + _entry('":', -0.01), |
| 331 | + _entry(" ", -0.01), |
| 332 | + _entry('"none"', -0.01), |
| 333 | + _entry("}", -0.01), |
| 334 | + ] |
| 335 | + content = '{"score": 0, "reason": "none"}' |
| 336 | + metric, captured = _geval_on_stubbed_provider(monkeypatch, content, entries) |
| 337 | + |
| 338 | + result = metric.score("any input") |
| 339 | + |
| 340 | + assert captured["logprobs"] is True |
| 341 | + assert captured["top_logprobs"] == 20 |
| 342 | + assert result.value == pytest.approx(0.07984052568223204, abs=1e-9) |
| 343 | + assert result.reason == "none" |
0 commit comments