|
| 1 | +// @ts-check |
| 2 | +/// <reference types="@actions/github-script" /> |
| 3 | + |
| 4 | +/** |
| 5 | + * run_evals — BinEval binary evaluation harness. |
| 6 | + * |
| 7 | + * This module operates in two phases selected by GH_AW_EVALS_PHASE: |
| 8 | + * |
| 9 | + * Phase "setup" (default, runs BEFORE the agentic engine): |
| 10 | + * - Reads configured eval questions from GH_AW_EVALS_QUESTIONS (JSON array) |
| 11 | + * - Reads the agent output from /tmp/gh-aw/evals/agent_output.json |
| 12 | + * - Builds a multi-question binary evaluation prompt |
| 13 | + * - Writes the prompt to /tmp/gh-aw/aw-prompts/prompt.txt for the engine |
| 14 | + * |
| 15 | + * Phase "parse" (runs AFTER the agentic engine): |
| 16 | + * - Reads the engine output log from /tmp/gh-aw/evals/evals.log |
| 17 | + * - Extracts YES/NO answer for each question by ID or by position |
| 18 | + * - Writes structured results to /tmp/gh-aw/evals.jsonl |
| 19 | + * |
| 20 | + * Environment variables: |
| 21 | + * GH_AW_EVALS_QUESTIONS JSON array of { id, question } objects |
| 22 | + * GH_AW_EVALS_PHASE "setup" (default) or "parse" |
| 23 | + * GH_AW_EVALS_MODEL LLM model name recorded in output metadata |
| 24 | + * |
| 25 | + * Design note: this file is intentionally engine-agnostic. The engine is |
| 26 | + * installed and executed by separate Go-generated GitHub Actions steps that |
| 27 | + * call engine.GetInstallationSteps / engine.GetExecutionSteps; this module |
| 28 | + * only handles prompt construction and result parsing. |
| 29 | + */ |
| 30 | + |
| 31 | +"use strict"; |
| 32 | + |
| 33 | +const fs = require("fs"); |
| 34 | +const path = require("path"); |
| 35 | + |
| 36 | +const { ERR_VALIDATION } = require("./error_codes.cjs"); |
| 37 | + |
| 38 | +const EVALS_DIR = "/tmp/gh-aw/evals"; |
| 39 | +const EVALS_LOG_PATH = "/tmp/gh-aw/evals/evals.log"; |
| 40 | +const EVALS_OUTPUT_PATH = "/tmp/gh-aw/evals.jsonl"; |
| 41 | +const AGENT_OUTPUT_FILENAME = "agent_output.json"; |
| 42 | + |
| 43 | +// --------------------------------------------------------------------------- |
| 44 | +// Phase 1 – setup: write multi-question evaluation prompt |
| 45 | +// --------------------------------------------------------------------------- |
| 46 | + |
| 47 | +/** |
| 48 | + * Reads eval questions and agent output, constructs a BinEval prompt, and |
| 49 | + * writes it to the standard GH_AW_PROMPT path for the agentic engine. |
| 50 | + * @returns {Promise<void>} |
| 51 | + */ |
| 52 | +async function setupMain() { |
| 53 | + const questionsRaw = process.env.GH_AW_EVALS_QUESTIONS; |
| 54 | + if (!questionsRaw) { |
| 55 | + core.setFailed(`${ERR_VALIDATION}: GH_AW_EVALS_QUESTIONS is not set`); |
| 56 | + return; |
| 57 | + } |
| 58 | + |
| 59 | + let questions; |
| 60 | + try { |
| 61 | + questions = JSON.parse(questionsRaw); |
| 62 | + } catch (e) { |
| 63 | + core.setFailed(`${ERR_VALIDATION}: GH_AW_EVALS_QUESTIONS is not valid JSON: ` + e.message); |
| 64 | + return; |
| 65 | + } |
| 66 | + |
| 67 | + if (!Array.isArray(questions) || questions.length === 0) { |
| 68 | + core.setFailed(`${ERR_VALIDATION}: GH_AW_EVALS_QUESTIONS must be a non-empty JSON array`); |
| 69 | + return; |
| 70 | + } |
| 71 | + |
| 72 | + fs.mkdirSync(EVALS_DIR, { recursive: true }); |
| 73 | + |
| 74 | + // Load agent output for evaluation context |
| 75 | + const agentOutputPath = path.join(EVALS_DIR, AGENT_OUTPUT_FILENAME); |
| 76 | + let agentOutputContent = ""; |
| 77 | + if (fs.existsSync(agentOutputPath)) { |
| 78 | + const stats = fs.statSync(agentOutputPath); |
| 79 | + agentOutputContent = fs.readFileSync(agentOutputPath, "utf-8"); |
| 80 | + core.info(`Agent output loaded: ${agentOutputPath} (${stats.size} bytes)`); |
| 81 | + } else { |
| 82 | + core.warning(`Agent output not found at ${agentOutputPath}. ` + "Ensure the agent artifact includes agent_output.json. " + "Evaluation will proceed without agent context."); |
| 83 | + } |
| 84 | + |
| 85 | + const prompt = buildEvalPrompt(questions, agentOutputContent); |
| 86 | + |
| 87 | + fs.mkdirSync("/tmp/gh-aw/aw-prompts", { recursive: true }); |
| 88 | + fs.writeFileSync("/tmp/gh-aw/aw-prompts/prompt.txt", prompt); |
| 89 | + core.exportVariable("GH_AW_PROMPT", "/tmp/gh-aw/aw-prompts/prompt.txt"); |
| 90 | + |
| 91 | + core.info(`BinEval setup complete: wrote prompt with ${questions.length} question(s)`); |
| 92 | + |
| 93 | + core.summary.addDetails("BinEval Evaluation Prompt", "\n\n``````markdown\n" + prompt + "\n``````\n\n"); |
| 94 | + await core.summary.write(); |
| 95 | +} |
| 96 | + |
| 97 | +// --------------------------------------------------------------------------- |
| 98 | +// Phase 2 – parse: extract answers and write evals.jsonl |
| 99 | +// --------------------------------------------------------------------------- |
| 100 | + |
| 101 | +/** |
| 102 | + * Reads the engine log, extracts per-question YES/NO answers, and writes |
| 103 | + * structured JSONL records to the evals output file. |
| 104 | + * @returns {Promise<void>} |
| 105 | + */ |
| 106 | +async function parseMain() { |
| 107 | + const questionsRaw = process.env.GH_AW_EVALS_QUESTIONS; |
| 108 | + const model = process.env.GH_AW_EVALS_MODEL || ""; |
| 109 | + |
| 110 | + /** @type {Array<{id: string, question: string}>} */ |
| 111 | + let questions = []; |
| 112 | + if (questionsRaw) { |
| 113 | + try { |
| 114 | + questions = JSON.parse(questionsRaw); |
| 115 | + } catch { |
| 116 | + core.warning("GH_AW_EVALS_QUESTIONS is not valid JSON; result IDs will be positional"); |
| 117 | + } |
| 118 | + } |
| 119 | + |
| 120 | + if (!fs.existsSync(EVALS_LOG_PATH)) { |
| 121 | + core.warning(`Evals log not found at ${EVALS_LOG_PATH}; no results written`); |
| 122 | + fs.writeFileSync(EVALS_OUTPUT_PATH, ""); |
| 123 | + return; |
| 124 | + } |
| 125 | + |
| 126 | + const logContent = fs.readFileSync(EVALS_LOG_PATH, "utf-8"); |
| 127 | + core.info(`Parsing evals log: ${EVALS_LOG_PATH} (${logContent.length} bytes)`); |
| 128 | + |
| 129 | + // Collect all positional Q1/Q2/... answers from the log for fallback lookup |
| 130 | + const positionalAnswers = extractAllPositionalAnswers(logContent); |
| 131 | + |
| 132 | + const timestamp = new Date().toISOString(); |
| 133 | + const results = []; |
| 134 | + |
| 135 | + for (let i = 0; i < questions.length; i++) { |
| 136 | + const q = questions[i]; |
| 137 | + |
| 138 | + // Try ID-specific match first (e.g. "builds: YES"), then positional (Q1: YES) |
| 139 | + let answer = extractAnswerByID(logContent, q.id); |
| 140 | + if (answer === "UNKNOWN" && i < positionalAnswers.length && positionalAnswers[i]) { |
| 141 | + answer = positionalAnswers[i]; |
| 142 | + } |
| 143 | + |
| 144 | + const record = { |
| 145 | + id: q.id, |
| 146 | + question: q.question, |
| 147 | + answer, |
| 148 | + model, |
| 149 | + timestamp, |
| 150 | + }; |
| 151 | + results.push(record); |
| 152 | + core.info(`Q[${q.id}]: ${answer}`); |
| 153 | + } |
| 154 | + |
| 155 | + // Write JSONL — one JSON object per line |
| 156 | + const jsonlLines = results.map(r => JSON.stringify(r)); |
| 157 | + fs.writeFileSync(EVALS_OUTPUT_PATH, jsonlLines.join("\n") + (jsonlLines.length > 0 ? "\n" : "")); |
| 158 | + core.info(`BinEval results written to ${EVALS_OUTPUT_PATH} (${results.length} record(s))`); |
| 159 | + |
| 160 | + const yesCount = results.filter(r => r.answer === "YES").length; |
| 161 | + const noCount = results.filter(r => r.answer === "NO").length; |
| 162 | + const unknownCount = results.filter(r => r.answer === "UNKNOWN").length; |
| 163 | + |
| 164 | + await core.summary |
| 165 | + .addHeading("BinEval Results", 2) |
| 166 | + .addTable([ |
| 167 | + [ |
| 168 | + { data: "ID", header: true }, |
| 169 | + { data: "Question", header: true }, |
| 170 | + { data: "Answer", header: true }, |
| 171 | + ], |
| 172 | + ...results.map(r => [r.id, r.question, r.answer]), |
| 173 | + ["", `YES: ${yesCount} | NO: ${noCount} | UNKNOWN: ${unknownCount}`, ""], |
| 174 | + ]) |
| 175 | + .write(); |
| 176 | +} |
| 177 | + |
| 178 | +// --------------------------------------------------------------------------- |
| 179 | +// Main entry point |
| 180 | +// --------------------------------------------------------------------------- |
| 181 | + |
| 182 | +/** |
| 183 | + * Dispatches to setupMain or parseMain based on GH_AW_EVALS_PHASE. |
| 184 | + * @returns {Promise<void>} |
| 185 | + */ |
| 186 | +async function main() { |
| 187 | + const phase = process.env.GH_AW_EVALS_PHASE || "setup"; |
| 188 | + if (phase === "parse") { |
| 189 | + await parseMain(); |
| 190 | + } else { |
| 191 | + await setupMain(); |
| 192 | + } |
| 193 | +} |
| 194 | + |
| 195 | +// --------------------------------------------------------------------------- |
| 196 | +// Helpers |
| 197 | +// --------------------------------------------------------------------------- |
| 198 | + |
| 199 | +/** |
| 200 | + * Builds a multi-question binary evaluation prompt. |
| 201 | + * @param {Array<{id: string, question: string}>} questions |
| 202 | + * @param {string} agentOutput |
| 203 | + * @returns {string} |
| 204 | + */ |
| 205 | +function buildEvalPrompt(questions, agentOutput) { |
| 206 | + const questionList = questions.map((q, i) => `<question number="${i + 1}" id="${q.id}">${q.question}</question>`).join("\n"); |
| 207 | + |
| 208 | + const agentSection = agentOutput ? `<agent_output>\n${agentOutput}\n</agent_output>` : "<agent_output>\n(no agent output available)\n</agent_output>"; |
| 209 | + |
| 210 | + return `# BinEval: Binary Evaluation |
| 211 | +
|
| 212 | +You are evaluating the output of an AI agentic workflow using BinEval (binary evaluation). |
| 213 | +For each question below, answer with exactly YES or NO based on the agent output provided. |
| 214 | +
|
| 215 | +<questions> |
| 216 | +${questionList} |
| 217 | +</questions> |
| 218 | +
|
| 219 | +${agentSection} |
| 220 | +
|
| 221 | +<instructions> |
| 222 | +Answer each question on a separate line using EXACTLY this format: |
| 223 | +Q1: YES |
| 224 | +Q2: NO |
| 225 | +
|
| 226 | +Use only YES or NO. Do not provide explanations or reasoning. |
| 227 | +Evaluate each question solely based on the agent output shown above. |
| 228 | +</instructions>`; |
| 229 | +} |
| 230 | + |
| 231 | +/** |
| 232 | + * Extracts all positional Q1/Q2/... answers from log content. |
| 233 | + * Returns a 0-indexed array where index 0 = Q1's answer. |
| 234 | + * @param {string} logContent |
| 235 | + * @returns {string[]} |
| 236 | + */ |
| 237 | +function extractAllPositionalAnswers(logContent) { |
| 238 | + /** @type {string[]} */ |
| 239 | + const answers = []; |
| 240 | + for (const line of logContent.split("\n")) { |
| 241 | + const match = line.trim().match(/^Q(\d+):\s+(YES|NO)\b/i); |
| 242 | + if (match) { |
| 243 | + const idx = parseInt(match[1], 10) - 1; // Convert 1-indexed to 0-indexed |
| 244 | + if (idx >= 0) { |
| 245 | + answers[idx] = match[2].toUpperCase(); |
| 246 | + } |
| 247 | + } |
| 248 | + } |
| 249 | + return answers; |
| 250 | +} |
| 251 | + |
| 252 | +/** |
| 253 | + * Tries to find an answer for a question by its id using flexible pattern matching. |
| 254 | + * Returns "YES", "NO", or "UNKNOWN". |
| 255 | + * @param {string} logContent |
| 256 | + * @param {string} id |
| 257 | + * @returns {string} |
| 258 | + */ |
| 259 | +function extractAnswerByID(logContent, id) { |
| 260 | + const escaped = id.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); |
| 261 | + const yesPattern = new RegExp(`\\b${escaped}\\b[:\\s]+(YES)\\b`, "i"); |
| 262 | + const noPattern = new RegExp(`\\b${escaped}\\b[:\\s]+(NO)\\b`, "i"); |
| 263 | + if (yesPattern.test(logContent)) return "YES"; |
| 264 | + if (noPattern.test(logContent)) return "NO"; |
| 265 | + return "UNKNOWN"; |
| 266 | +} |
| 267 | + |
| 268 | +module.exports = { main, setupMain, parseMain }; |
0 commit comments