Skip to content

Commit 571670d

Browse files
jazhang00copybara-github
authored andcommitted
fix: ignore thought parts when scoring response_match_score
The ROUGE-1 evaluator joined every text part of the final response, so agents that emit thoughts (e.g. with `include_thoughts` or `PlanReActPlanner`) had their reasoning scored as part of the answer. Thought parts are now skipped, matching how the rest of ADK extracts response text. Co-authored-by: Jason Zhang <[email protected]> PiperOrigin-RevId: 1008467069
1 parent 2591397 commit 571670d

2 files changed

Lines changed: 20 additions & 10 deletions

File tree

‎src/google/adk/evaluation/final_response_match_v1.py‎

Lines changed: 5 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,11 @@
1818
from typing import Optional
1919
import unicodedata
2020

21-
from google.genai import types as genai_types
2221
from typing_extensions import override
2322

2423
from ..dependencies.rouge_scorer import rouge_scorer
2524
from ..dependencies.rouge_scorer import tokenizers
25+
from ..utils.content_utils import extract_text_from_content
2626
from .eval_case import ConversationScenario
2727
from .eval_case import Invocation
2828
from .eval_metrics import _get_metric_threshold
@@ -63,8 +63,10 @@ def evaluate_invocations(
6363
for actual, expected in zip(
6464
actual_invocations, expected_invocations, strict=True
6565
):
66-
reference = _get_text_from_content(expected.final_response)
67-
response = _get_text_from_content(actual.final_response)
66+
reference = extract_text_from_content(
67+
expected.final_response, splitter="\n"
68+
)
69+
response = extract_text_from_content(actual.final_response, splitter="\n")
6870
rouge_1_scores = _calculate_rouge_1_scores(response, reference)
6971
score = rouge_1_scores.fmeasure
7072
per_invocation_results.append(
@@ -89,13 +91,6 @@ def evaluate_invocations(
8991
return EvaluationResult()
9092

9193

92-
def _get_text_from_content(content: Optional[genai_types.Content]) -> str:
93-
if content and content.parts:
94-
return "\n".join([part.text for part in content.parts if part.text])
95-
96-
return ""
97-
98-
9994
def _get_eval_status(score: float, threshold: float) -> EvalStatus:
10095
return EvalStatus.PASSED if score >= threshold else EvalStatus.FAILED
10196

‎tests/unittests/evaluation/test_final_response_match_v1.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -409,3 +409,18 @@ def test_rouge_evaluator_grades_with_either_threshold_source(
409409
def test_rouge_evaluator_rejects_metric_without_a_threshold():
410410
with pytest.raises(ValueError, match="requires a threshold"):
411411
RougeEvaluator(EvalMetric(metric_name="response_match_score"))
412+
413+
414+
def test_rouge_evaluator_ignores_thought_parts():
415+
"""Thought parts in the final response are not scored as answer text."""
416+
rouge_evaluator = _create_test_rouge_evaluator(threshold=0.8)
417+
actual, expected = _create_test_invocations(
418+
"It is sunny today.", "It is sunny today."
419+
)
420+
actual.final_response.parts.insert(
421+
0, genai_types.Part(text="Let me reason about the weather.", thought=True)
422+
)
423+
424+
evaluation_result = rouge_evaluator.evaluate_invocations([actual], [expected])
425+
426+
assert evaluation_result.overall_score == pytest.approx(1.0)

0 commit comments

Comments
 (0)