|
| 1 | +#!/usr/bin/env python3 |
| 2 | +"""Deterministic verifier for exact Buzz channel creation and membership.""" |
| 3 | + |
| 4 | +from __future__ import annotations |
| 5 | + |
| 6 | +import argparse |
| 7 | +import json |
| 8 | +from pathlib import Path |
| 9 | +from typing import Any |
| 10 | + |
| 11 | +CHANNEL_NAME = "fix-pr-1234" |
| 12 | +TARGET_USERS = {"benchmark-user-07", "benchmark-user-19", "benchmark-user-42"} |
| 13 | +TARGET_BOTS = {"benchmark-bot-03", "benchmark-bot-08"} |
| 14 | + |
| 15 | + |
| 16 | +def _zero_metrics() -> dict[str, float]: |
| 17 | + return { |
| 18 | + "reward": 0.0, |
| 19 | + "channel_created": 0.0, |
| 20 | + "channel_shape": 0.0, |
| 21 | + "temporary_channel": 0.0, |
| 22 | + "exact_membership": 0.0, |
| 23 | + "expected_roles": 0.0, |
| 24 | + "evidence_complete": 0.0, |
| 25 | + } |
| 26 | + |
| 27 | + |
| 28 | +def score_evidence(evidence: object) -> tuple[dict[str, float], dict[str, Any]]: |
| 29 | + if not isinstance(evidence, dict): |
| 30 | + return _zero_metrics(), {"error": "evidence root is not an object"} |
| 31 | + |
| 32 | + directory_rows = [ |
| 33 | + row for row in evidence.get("directory", []) if isinstance(row, dict) |
| 34 | + ] |
| 35 | + directory = { |
| 36 | + row.get("name"): row |
| 37 | + for row in directory_rows |
| 38 | + if isinstance(row.get("name"), str) |
| 39 | + } |
| 40 | + channels = [ |
| 41 | + channel |
| 42 | + for channel in evidence.get("observed_channels", []) |
| 43 | + if isinstance(channel, dict) and channel.get("name") == CHANNEL_NAME |
| 44 | + ] |
| 45 | + channel = channels[0] if len(channels) == 1 else None |
| 46 | + identities = ( |
| 47 | + evidence.get("identities") |
| 48 | + if isinstance(evidence.get("identities"), dict) |
| 49 | + else {} |
| 50 | + ) |
| 51 | + orchestrators = [ |
| 52 | + row |
| 53 | + for row in identities.values() |
| 54 | + if isinstance(row, dict) and row.get("role") == "orchestrator" |
| 55 | + ] |
| 56 | + owner_pubkey = orchestrators[0].get("pubkey") if len(orchestrators) == 1 else None |
| 57 | + |
| 58 | + expected_names = TARGET_USERS | TARGET_BOTS |
| 59 | + expected_targets = { |
| 60 | + directory[name]["pubkey"]: "bot" if name in TARGET_BOTS else "member" |
| 61 | + for name in expected_names |
| 62 | + if name in directory and isinstance(directory[name].get("pubkey"), str) |
| 63 | + } |
| 64 | + expected_members = ( |
| 65 | + {owner_pubkey: "owner", **expected_targets} |
| 66 | + if isinstance(owner_pubkey, str) |
| 67 | + else expected_targets |
| 68 | + ) |
| 69 | + member_rows = ( |
| 70 | + [row for row in channel.get("members", []) if isinstance(row, dict)] |
| 71 | + if channel is not None |
| 72 | + else [] |
| 73 | + ) |
| 74 | + actual_members = { |
| 75 | + row.get("pubkey"): row.get("role") |
| 76 | + for row in member_rows |
| 77 | + if isinstance(row.get("pubkey"), str) |
| 78 | + } |
| 79 | + |
| 80 | + evidence_complete = float( |
| 81 | + evidence.get("schema_version") == 1 |
| 82 | + and evidence.get("task_name") == "create-channel-invite-users" |
| 83 | + and len(directory_rows) == 60 |
| 84 | + and len(directory) == 60 |
| 85 | + and sum(row.get("role") == "user" for row in directory_rows) == 50 |
| 86 | + and sum(row.get("role") == "bot" for row in directory_rows) == 10 |
| 87 | + and len(expected_targets) == 5 |
| 88 | + and len(orchestrators) == 1 |
| 89 | + ) |
| 90 | + channel_created = float(channel is not None) |
| 91 | + channel_shape = float( |
| 92 | + channel is not None |
| 93 | + and channel.get("channel_type") == "stream" |
| 94 | + and channel.get("visibility") == "private" |
| 95 | + and channel.get("archived") is False |
| 96 | + ) |
| 97 | + temporary_channel = float( |
| 98 | + channel is not None and channel.get("ttl_seconds") == 3600 |
| 99 | + ) |
| 100 | + exact_membership = float( |
| 101 | + len(member_rows) == len(actual_members) |
| 102 | + and set(actual_members) == set(expected_members) |
| 103 | + ) |
| 104 | + expected_roles = float(actual_members == expected_members) |
| 105 | + reward = float( |
| 106 | + all( |
| 107 | + metric == 1.0 |
| 108 | + for metric in ( |
| 109 | + evidence_complete, |
| 110 | + channel_created, |
| 111 | + channel_shape, |
| 112 | + temporary_channel, |
| 113 | + exact_membership, |
| 114 | + expected_roles, |
| 115 | + ) |
| 116 | + ) |
| 117 | + ) |
| 118 | + metrics = { |
| 119 | + "reward": reward, |
| 120 | + "channel_created": channel_created, |
| 121 | + "channel_shape": channel_shape, |
| 122 | + "temporary_channel": temporary_channel, |
| 123 | + "exact_membership": exact_membership, |
| 124 | + "expected_roles": expected_roles, |
| 125 | + "evidence_complete": evidence_complete, |
| 126 | + } |
| 127 | + details = { |
| 128 | + "matching_channel_count": len(channels), |
| 129 | + "channel_id": channel.get("channel_id") if channel is not None else None, |
| 130 | + "expected_members": expected_members, |
| 131 | + "actual_members": actual_members, |
| 132 | + } |
| 133 | + return metrics, details |
| 134 | + |
| 135 | + |
| 136 | +def main() -> int: |
| 137 | + parser = argparse.ArgumentParser() |
| 138 | + parser.add_argument("--evidence", type=Path, required=True) |
| 139 | + parser.add_argument("--reward", type=Path, required=True) |
| 140 | + parser.add_argument("--details", type=Path, required=True) |
| 141 | + args = parser.parse_args() |
| 142 | + |
| 143 | + try: |
| 144 | + evidence = json.loads(args.evidence.read_text(encoding="utf-8")) |
| 145 | + metrics, details = score_evidence(evidence) |
| 146 | + except (OSError, json.JSONDecodeError) as error: |
| 147 | + metrics, details = _zero_metrics(), {"error": str(error)} |
| 148 | + |
| 149 | + args.reward.write_text(json.dumps(metrics, sort_keys=True) + "\n", encoding="utf-8") |
| 150 | + args.details.write_text( |
| 151 | + json.dumps(details, indent=2, sort_keys=True) + "\n", encoding="utf-8" |
| 152 | + ) |
| 153 | + return 0 |
| 154 | + |
| 155 | + |
| 156 | +if __name__ == "__main__": |
| 157 | + raise SystemExit(main()) |
0 commit comments