Skip to content
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
pazar: addressed feedback from PR comments
Signed-off-by: Philip Azar <[email protected]>
  • Loading branch information
philazar committed Aug 28, 2026
commit a87823aaae7a45c8d2e70115c5c0014fc4414c97
10 changes: 6 additions & 4 deletions benchmarks/buzz-dataset/memory-retrieval/README.md
Original file line number Diff line number Diff line change
@@ -1,11 +1,13 @@
# memory-retrieval

Before the agent starts, the harness runs `buzz mem set` with the agent's own
Buzz credentials to seed a GPV rule as cold memory. It then delivers
Buzz credentials to seed a revenue convention as cold memory. It then delivers
`instruction.md`, which contains only the retrieval question and does not
reveal the memory value or slug. No channel message contains the answer, so
conversation history cannot supply it.

Passing requires a threaded answer that selects `net_gpv`. The verifier does
not inspect tool calls: seeding is deterministic harness setup, and retrieval
is graded only through the observable answer.
Full credit requires an affirmative `use net_gpv_var_usd` in the threaded
answer; a non-rejecting mention of `net_gpv_var_usd` receives half credit.
Negated or contradictory selections receive no credit. The verifier does not
inspect tool calls: seeding is deterministic harness setup, and retrieval is
graded only through the observable answer.
2 changes: 1 addition & 1 deletion benchmarks/buzz-dataset/memory-retrieval/instruction.md
Original file line number Diff line number Diff line change
@@ -1 +1 @@
What column should be used for gpv calculation?
What column should be used for revenue calculation? Reply with `use <column_name>`.
39 changes: 32 additions & 7 deletions benchmarks/buzz-dataset/memory-retrieval/tests/verify.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,17 @@
from pathlib import Path
from typing import Any

EXPECTED_COLUMN = "net_gpv_var_usd"
EXPECTED_COLUMN_MENTION = re.compile(rf"\b{EXPECTED_COLUMN}\b")
USE_EXPECTED_COLUMN = re.compile(rf"\buse\s+{EXPECTED_COLUMN}\b")
REJECT_EXPECTED_COLUMN = re.compile(
rf"\b(?:do|should|must|would|could|can)(?:\s+\w+){{0,2}}\s+not\s+use\s+{EXPECTED_COLUMN}\b"
rf"|\b(?:don't|shouldn't|mustn't|wouldn't|couldn't|can't|never|not)\s+use\s+{EXPECTED_COLUMN}\b"
rf"|\buse\s+gross_gpv\b[^.!?\n]*(?:instead\s+of|rather\s+than)\s+{EXPECTED_COLUMN}\b"
rf"|\b(?:gross_gpv\b[^.!?\n]*,?\s*)?not\s+{EXPECTED_COLUMN}\b"
rf"|\b{EXPECTED_COLUMN}\b[^.!?\n]{{0,24}}\b(?:incorrect|wrong)\b"
)


def _zero() -> dict[str, float]:
return {
Expand All @@ -19,6 +30,13 @@ def _zero() -> dict[str, float]:
}


def _normalize_answer(content: str) -> str:
"""Normalize harmless presentation differences without changing meaning."""
normalized = content.strip().casefold().replace("`", "").replace("’", "'")
normalized = re.sub(r"\s+", " ", normalized)
return normalized.rstrip(".!?").rstrip()


def score_evidence(evidence: object) -> tuple[dict[str, float], dict[str, Any]]:
if not isinstance(evidence, dict):
return _zero(), {"error": "evidence root is not an object"}
Expand Down Expand Up @@ -49,7 +67,16 @@ def score_evidence(evidence: object) -> tuple[dict[str, float], dict[str, Any]]:
]
answer = replies[-1] if replies else None
content = str(answer.get("content", "")) if answer else ""
answer_correct = float(bool(re.search(r"\bnet_gpv\b", content, re.IGNORECASE)))
normalized_answer = _normalize_answer(content)
rejects_expected_column = bool(REJECT_EXPECTED_COLUMN.search(normalized_answer))
if rejects_expected_column:
answer_correct = 0.0
elif USE_EXPECTED_COLUMN.search(normalized_answer):
answer_correct = 1.0
elif EXPECTED_COLUMN_MENTION.search(normalized_answer):
answer_correct = 0.5
else:
answer_correct = 0.0
threaded_reply = float(answer is not None)
evidence_complete = float(
evidence.get("schema_version") == 1
Expand All @@ -60,13 +87,9 @@ def score_evidence(evidence: object) -> tuple[dict[str, float], dict[str, Any]]:
and isinstance(question_channel, str)
)

values = (
answer_correct,
threaded_reply,
evidence_complete,
)
structural_score = float(threaded_reply == 1.0 and evidence_complete == 1.0)
metrics = {
"reward": float(all(value == 1.0 for value in values)),
"reward": answer_correct * structural_score,
"answer_correct": answer_correct,
"threaded_reply": threaded_reply,
"evidence_complete": evidence_complete,
Expand All @@ -76,6 +99,8 @@ def score_evidence(evidence: object) -> tuple[dict[str, float], dict[str, Any]]:
"question_channel_id": question_channel,
"answer_message_id": answer.get("id") if answer else None,
"answer_content": content,
"normalized_answer": normalized_answer,
"rejects_expected_column": rejects_expected_column,
}


Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -174,13 +174,34 @@ class BuzzTaskFixture:
_MEMORY_RETRIEVAL_FIXTURE = BuzzTaskFixture(
user_display_name="Amelia Rose Bennett",
memory_seeds=(
MemorySeed(
slug="finance-vs-marketing",
value=(
"Finance measures all metrics in quarters; marketing measures all "
"metrics monthly."
),
),
MemorySeed(
slug="finance-conventions",
value=(
"For GPV calculation, always use the net_gpv column, not the "
"For revenue metrics, always use the net_gpv_var_usd column, not the "
"gross_gpv column."
),
),
MemorySeed(
slug="timezone-calculation",
value=(
"Always adjust timezone fields like created_at and payment_at from "
"UTC to PST."
),
),
MemorySeed(
slug="revenue-metric-formatting",
value=(
"Round revenue to the nearest billion. Include a $ when using the "
"gross_gpv_fxd_usd column."
),
),
),
requires_evidence=True,
)
Expand Down
Loading
Loading