Skip to content

Commit 4f14e81

Browse files
committed
test(g-eval): pin duplicate-key value and cover the LiteLLM GEval path
Tighten the duplicate-score-key regression to the exact weighted value of the second key's candidates (pytest.approx, abs=1e-9) instead of a direction-only range, and add two provider-shaped GEval.score tests through the real LiteLLM branch (split-score 10, folded whitespace), asserting the exact weighted values plus reason and provider kwargs.
1 parent ab62edc commit 4f14e81

1 file changed

Lines changed: 135 additions & 2 deletions

File tree

  • sdks/python/tests/unit/evaluation/metrics/llm_judges/g_eval

‎sdks/python/tests/unit/evaluation/metrics/llm_judges/g_eval/test_parser.py‎

Lines changed: 135 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
1+
from contextlib import contextmanager
2+
from types import SimpleNamespace
3+
14
from opik import logging_messages, exceptions
25
from opik.evaluation.metrics.llm_judges.g_eval import parser
6+
from opik.evaluation.metrics.llm_judges.g_eval.metric import GEval
7+
from opik.evaluation.models import base_model
38
import pytest
49

510

@@ -178,7 +183,12 @@ def test_duplicate_score_keys_last_wins_like_the_text_path():
178183
name="g_eval",
179184
log_probs_supported=True,
180185
)
181-
assert 0.55 < result.value < 0.68, f"scored from the stale key: {result.value}"
186+
# Exact weighted value of the second key's candidates (7 @ -0.1,
187+
# 1 @ -2.0): (7*e^-0.1 + e^-2)/(e^-0.1 + e^-2)/10. A first-match locator
188+
# scores the stale "9" key instead (0.88084...) and fails here.
189+
assert result.value == pytest.approx(0.6219349153822014, abs=1e-9), (
190+
f"scored from the stale key: {result.value}"
191+
)
182192

183193

184194
def test_escaped_quote_echo_is_not_a_matchable_key():
@@ -207,4 +217,127 @@ def test_escaped_quote_echo_is_not_a_matchable_key():
207217
name="g_eval",
208218
log_probs_supported=True,
209219
)
210-
assert 0.55 < result.value < 0.68, f"unexpected value {result.value}"
220+
# Same distribution as the duplicate-key test pins the locator on the
221+
# real key; see the derivation there.
222+
assert result.value == pytest.approx(0.6219349153822014, abs=1e-9), (
223+
f"unexpected value {result.value}"
224+
)
225+
226+
227+
# --- provider-shaped GEval coverage ----------------------------------------
228+
# The tests above call parse_litellm_model_output directly with hand-built
229+
# token dicts. These go through the public GEval.score LiteLLM branch with
230+
# a provider-shaped raw response, so an incorrectly bounded result on the
231+
# real path cannot pass unnoticed.
232+
233+
234+
def _provider_response(content, entries):
235+
"""Provider-shaped raw LiteLLM response: a dict-style choice carrying the
236+
message content plus the logprobs token stream, as yielded by
237+
get_provider_response on the LiteLLM branch."""
238+
return SimpleNamespace(
239+
choices=[{"message": {"content": content}, "logprobs": {"content": entries}}]
240+
)
241+
242+
243+
def _geval_on_stubbed_provider(monkeypatch, content, entries):
244+
"""GEval wired to a stubbed LiteLLM provider: chain-of-thought generation
245+
is stubbed, and the scoring call yields the given provider-shaped
246+
response. Returns (metric, captured_provider_kwargs)."""
247+
metric = GEval(
248+
task_introduction="provider-path split-score intro",
249+
evaluation_criteria="provider-path split-score criteria",
250+
model="gpt-4o",
251+
track=False,
252+
)
253+
# The subject here is the logprob parser branch, not model capability
254+
# detection (covered by test_geval_passes_logprobs_only_when_supported).
255+
metric._log_probs_supported = True
256+
monkeypatch.setattr(
257+
metric._model,
258+
"generate_chat_completion",
259+
lambda *args, **kwargs: {"content": "stub chain of thought"},
260+
)
261+
captured = {}
262+
263+
@contextmanager
264+
def fake_get_provider_response(model_provider, messages, **kwargs):
265+
captured.update(kwargs)
266+
yield _provider_response(content, entries)
267+
268+
monkeypatch.setattr(base_model, "get_provider_response", fake_get_provider_response)
269+
return metric, captured
270+
271+
272+
def test_geval_litellm_path_split_score_scores_near_one(monkeypatch):
273+
# {"score":10} split across "1" + "0" must score near 1.0 end to end. A
274+
# legacy fixed-offset read of entries[3] alone averages to ~0.099.
275+
entries = [
276+
_entry('{"', -0.01),
277+
_entry("score", -0.01),
278+
_entry('":', -0.01),
279+
_entry(
280+
"1",
281+
-0.05,
282+
top=[
283+
{"token": "1", "logprob": -0.05},
284+
{"token": "0", "logprob": -2.30},
285+
{"token": "2", "logprob": -2.40},
286+
],
287+
),
288+
_entry(
289+
"0",
290+
-0.02,
291+
top=[{"token": "0", "logprob": -0.02}, {"token": "1", "logprob": -3.00}],
292+
),
293+
_entry(",", -0.01),
294+
_entry('"reason"', -0.01),
295+
_entry('":', -0.01),
296+
_entry(" ", -0.01),
297+
_entry('"excellent"', -0.01),
298+
_entry("}", -0.01),
299+
]
300+
content = '{"score":10, "reason": "excellent"}'
301+
metric, captured = _geval_on_stubbed_provider(monkeypatch, content, entries)
302+
303+
result = metric.score("any input")
304+
305+
assert captured["logprobs"] is True
306+
assert captured["top_logprobs"] == 20
307+
assert result.value == pytest.approx(0.9007723389857002, abs=1e-9)
308+
assert result.reason == "excellent"
309+
310+
311+
def test_geval_litellm_path_folded_whitespace_is_scored(monkeypatch):
312+
# The space after the colon folds into the score token (" 0"); the
313+
# end-to-end path must score it instead of raising
314+
# MetricComputationError on a perfectly parseable response.
315+
entries = [
316+
_entry('{"', -0.01),
317+
_entry("score", -0.01),
318+
_entry('":', -0.01),
319+
_entry(
320+
" 0",
321+
-0.02,
322+
top=[
323+
{"token": " 0", "logprob": -0.02},
324+
{"token": " 1", "logprob": -2.00},
325+
{"token": " 10", "logprob": -2.50},
326+
],
327+
),
328+
_entry(",", -0.01),
329+
_entry('"reason"', -0.01),
330+
_entry('":', -0.01),
331+
_entry(" ", -0.01),
332+
_entry('"none"', -0.01),
333+
_entry("}", -0.01),
334+
]
335+
content = '{"score": 0, "reason": "none"}'
336+
metric, captured = _geval_on_stubbed_provider(monkeypatch, content, entries)
337+
338+
result = metric.score("any input")
339+
340+
assert captured["logprobs"] is True
341+
assert captured["top_logprobs"] == 20
342+
assert result.value == pytest.approx(0.07984052568223204, abs=1e-9)
343+
assert result.reason == "none"

0 commit comments

Comments
 (0)