Skip to content

Commit cba13b4

Browse files
chore: sync actions from [email protected] (#180)
Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
1 parent eb6dc28 commit cba13b4

5 files changed

Lines changed: 311 additions & 4 deletions

File tree

‎setup/js/allowed_issue_fields.cjs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,13 @@
33

44
const { ERR_VALIDATION } = require("./error_codes.cjs");
55

6+
/**
7+
* Builtin GitHub issue fields that have dedicated safe-output tools (e.g. update_issue).
8+
* The set_issue_field handler must refuse these fields to avoid confusion.
9+
* @type {Set<string>}
10+
*/
11+
const BUILTIN_ISSUE_FIELD_NAMES = new Set(["title", "body", "state", "labels", "assignees", "milestone"]);
12+
613
/**
714
* Parse allowed issue field names from config.
815
* @param {string[]|string|undefined} value
@@ -58,6 +65,7 @@ function validateAllowedIssueFields(issueFields, allowedFields) {
5865
}
5966

6067
module.exports = {
68+
BUILTIN_ISSUE_FIELD_NAMES,
6169
parseAllowedIssueFields,
6270
validateAllowedIssueFieldName,
6371
validateAllowedIssueFields,

‎setup/js/run_evals.cjs‎

Lines changed: 268 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,268 @@
1+
// @ts-check
2+
/// <reference types="@actions/github-script" />
3+
4+
/**
5+
* run_evals — BinEval binary evaluation harness.
6+
*
7+
* This module operates in two phases selected by GH_AW_EVALS_PHASE:
8+
*
9+
* Phase "setup" (default, runs BEFORE the agentic engine):
10+
* - Reads configured eval questions from GH_AW_EVALS_QUESTIONS (JSON array)
11+
* - Reads the agent output from /tmp/gh-aw/evals/agent_output.json
12+
* - Builds a multi-question binary evaluation prompt
13+
* - Writes the prompt to /tmp/gh-aw/aw-prompts/prompt.txt for the engine
14+
*
15+
* Phase "parse" (runs AFTER the agentic engine):
16+
* - Reads the engine output log from /tmp/gh-aw/evals/evals.log
17+
* - Extracts YES/NO answer for each question by ID or by position
18+
* - Writes structured results to /tmp/gh-aw/evals.jsonl
19+
*
20+
* Environment variables:
21+
* GH_AW_EVALS_QUESTIONS JSON array of { id, question } objects
22+
* GH_AW_EVALS_PHASE "setup" (default) or "parse"
23+
* GH_AW_EVALS_MODEL LLM model name recorded in output metadata
24+
*
25+
* Design note: this file is intentionally engine-agnostic. The engine is
26+
* installed and executed by separate Go-generated GitHub Actions steps that
27+
* call engine.GetInstallationSteps / engine.GetExecutionSteps; this module
28+
* only handles prompt construction and result parsing.
29+
*/
30+
31+
"use strict";
32+
33+
const fs = require("fs");
34+
const path = require("path");
35+
36+
const { ERR_VALIDATION } = require("./error_codes.cjs");
37+
38+
const EVALS_DIR = "/tmp/gh-aw/evals";
39+
const EVALS_LOG_PATH = "/tmp/gh-aw/evals/evals.log";
40+
const EVALS_OUTPUT_PATH = "/tmp/gh-aw/evals.jsonl";
41+
const AGENT_OUTPUT_FILENAME = "agent_output.json";
42+
43+
// ---------------------------------------------------------------------------
44+
// Phase 1 – setup: write multi-question evaluation prompt
45+
// ---------------------------------------------------------------------------
46+
47+
/**
48+
* Reads eval questions and agent output, constructs a BinEval prompt, and
49+
* writes it to the standard GH_AW_PROMPT path for the agentic engine.
50+
* @returns {Promise<void>}
51+
*/
52+
async function setupMain() {
53+
const questionsRaw = process.env.GH_AW_EVALS_QUESTIONS;
54+
if (!questionsRaw) {
55+
core.setFailed(`${ERR_VALIDATION}: GH_AW_EVALS_QUESTIONS is not set`);
56+
return;
57+
}
58+
59+
let questions;
60+
try {
61+
questions = JSON.parse(questionsRaw);
62+
} catch (e) {
63+
core.setFailed(`${ERR_VALIDATION}: GH_AW_EVALS_QUESTIONS is not valid JSON: ` + e.message);
64+
return;
65+
}
66+
67+
if (!Array.isArray(questions) || questions.length === 0) {
68+
core.setFailed(`${ERR_VALIDATION}: GH_AW_EVALS_QUESTIONS must be a non-empty JSON array`);
69+
return;
70+
}
71+
72+
fs.mkdirSync(EVALS_DIR, { recursive: true });
73+
74+
// Load agent output for evaluation context
75+
const agentOutputPath = path.join(EVALS_DIR, AGENT_OUTPUT_FILENAME);
76+
let agentOutputContent = "";
77+
if (fs.existsSync(agentOutputPath)) {
78+
const stats = fs.statSync(agentOutputPath);
79+
agentOutputContent = fs.readFileSync(agentOutputPath, "utf-8");
80+
core.info(`Agent output loaded: ${agentOutputPath} (${stats.size} bytes)`);
81+
} else {
82+
core.warning(`Agent output not found at ${agentOutputPath}. ` + "Ensure the agent artifact includes agent_output.json. " + "Evaluation will proceed without agent context.");
83+
}
84+
85+
const prompt = buildEvalPrompt(questions, agentOutputContent);
86+
87+
fs.mkdirSync("/tmp/gh-aw/aw-prompts", { recursive: true });
88+
fs.writeFileSync("/tmp/gh-aw/aw-prompts/prompt.txt", prompt);
89+
core.exportVariable("GH_AW_PROMPT", "/tmp/gh-aw/aw-prompts/prompt.txt");
90+
91+
core.info(`BinEval setup complete: wrote prompt with ${questions.length} question(s)`);
92+
93+
core.summary.addDetails("BinEval Evaluation Prompt", "\n\n``````markdown\n" + prompt + "\n``````\n\n");
94+
await core.summary.write();
95+
}
96+
97+
// ---------------------------------------------------------------------------
98+
// Phase 2 – parse: extract answers and write evals.jsonl
99+
// ---------------------------------------------------------------------------
100+
101+
/**
102+
* Reads the engine log, extracts per-question YES/NO answers, and writes
103+
* structured JSONL records to the evals output file.
104+
* @returns {Promise<void>}
105+
*/
106+
async function parseMain() {
107+
const questionsRaw = process.env.GH_AW_EVALS_QUESTIONS;
108+
const model = process.env.GH_AW_EVALS_MODEL || "";
109+
110+
/** @type {Array<{id: string, question: string}>} */
111+
let questions = [];
112+
if (questionsRaw) {
113+
try {
114+
questions = JSON.parse(questionsRaw);
115+
} catch {
116+
core.warning("GH_AW_EVALS_QUESTIONS is not valid JSON; result IDs will be positional");
117+
}
118+
}
119+
120+
if (!fs.existsSync(EVALS_LOG_PATH)) {
121+
core.warning(`Evals log not found at ${EVALS_LOG_PATH}; no results written`);
122+
fs.writeFileSync(EVALS_OUTPUT_PATH, "");
123+
return;
124+
}
125+
126+
const logContent = fs.readFileSync(EVALS_LOG_PATH, "utf-8");
127+
core.info(`Parsing evals log: ${EVALS_LOG_PATH} (${logContent.length} bytes)`);
128+
129+
// Collect all positional Q1/Q2/... answers from the log for fallback lookup
130+
const positionalAnswers = extractAllPositionalAnswers(logContent);
131+
132+
const timestamp = new Date().toISOString();
133+
const results = [];
134+
135+
for (let i = 0; i < questions.length; i++) {
136+
const q = questions[i];
137+
138+
// Try ID-specific match first (e.g. "builds: YES"), then positional (Q1: YES)
139+
let answer = extractAnswerByID(logContent, q.id);
140+
if (answer === "UNKNOWN" && i < positionalAnswers.length && positionalAnswers[i]) {
141+
answer = positionalAnswers[i];
142+
}
143+
144+
const record = {
145+
id: q.id,
146+
question: q.question,
147+
answer,
148+
model,
149+
timestamp,
150+
};
151+
results.push(record);
152+
core.info(`Q[${q.id}]: ${answer}`);
153+
}
154+
155+
// Write JSONL — one JSON object per line
156+
const jsonlLines = results.map(r => JSON.stringify(r));
157+
fs.writeFileSync(EVALS_OUTPUT_PATH, jsonlLines.join("\n") + (jsonlLines.length > 0 ? "\n" : ""));
158+
core.info(`BinEval results written to ${EVALS_OUTPUT_PATH} (${results.length} record(s))`);
159+
160+
const yesCount = results.filter(r => r.answer === "YES").length;
161+
const noCount = results.filter(r => r.answer === "NO").length;
162+
const unknownCount = results.filter(r => r.answer === "UNKNOWN").length;
163+
164+
await core.summary
165+
.addHeading("BinEval Results", 2)
166+
.addTable([
167+
[
168+
{ data: "ID", header: true },
169+
{ data: "Question", header: true },
170+
{ data: "Answer", header: true },
171+
],
172+
...results.map(r => [r.id, r.question, r.answer]),
173+
["", `YES: ${yesCount} | NO: ${noCount} | UNKNOWN: ${unknownCount}`, ""],
174+
])
175+
.write();
176+
}
177+
178+
// ---------------------------------------------------------------------------
179+
// Main entry point
180+
// ---------------------------------------------------------------------------
181+
182+
/**
183+
* Dispatches to setupMain or parseMain based on GH_AW_EVALS_PHASE.
184+
* @returns {Promise<void>}
185+
*/
186+
async function main() {
187+
const phase = process.env.GH_AW_EVALS_PHASE || "setup";
188+
if (phase === "parse") {
189+
await parseMain();
190+
} else {
191+
await setupMain();
192+
}
193+
}
194+
195+
// ---------------------------------------------------------------------------
196+
// Helpers
197+
// ---------------------------------------------------------------------------
198+
199+
/**
200+
* Builds a multi-question binary evaluation prompt.
201+
* @param {Array<{id: string, question: string}>} questions
202+
* @param {string} agentOutput
203+
* @returns {string}
204+
*/
205+
function buildEvalPrompt(questions, agentOutput) {
206+
const questionList = questions.map((q, i) => `<question number="${i + 1}" id="${q.id}">${q.question}</question>`).join("\n");
207+
208+
const agentSection = agentOutput ? `<agent_output>\n${agentOutput}\n</agent_output>` : "<agent_output>\n(no agent output available)\n</agent_output>";
209+
210+
return `# BinEval: Binary Evaluation
211+
212+
You are evaluating the output of an AI agentic workflow using BinEval (binary evaluation).
213+
For each question below, answer with exactly YES or NO based on the agent output provided.
214+
215+
<questions>
216+
${questionList}
217+
</questions>
218+
219+
${agentSection}
220+
221+
<instructions>
222+
Answer each question on a separate line using EXACTLY this format:
223+
Q1: YES
224+
Q2: NO
225+
226+
Use only YES or NO. Do not provide explanations or reasoning.
227+
Evaluate each question solely based on the agent output shown above.
228+
</instructions>`;
229+
}
230+
231+
/**
232+
* Extracts all positional Q1/Q2/... answers from log content.
233+
* Returns a 0-indexed array where index 0 = Q1's answer.
234+
* @param {string} logContent
235+
* @returns {string[]}
236+
*/
237+
function extractAllPositionalAnswers(logContent) {
238+
/** @type {string[]} */
239+
const answers = [];
240+
for (const line of logContent.split("\n")) {
241+
const match = line.trim().match(/^Q(\d+):\s+(YES|NO)\b/i);
242+
if (match) {
243+
const idx = parseInt(match[1], 10) - 1; // Convert 1-indexed to 0-indexed
244+
if (idx >= 0) {
245+
answers[idx] = match[2].toUpperCase();
246+
}
247+
}
248+
}
249+
return answers;
250+
}
251+
252+
/**
253+
* Tries to find an answer for a question by its id using flexible pattern matching.
254+
* Returns "YES", "NO", or "UNKNOWN".
255+
* @param {string} logContent
256+
* @param {string} id
257+
* @returns {string}
258+
*/
259+
function extractAnswerByID(logContent, id) {
260+
const escaped = id.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
261+
const yesPattern = new RegExp(`\\b${escaped}\\b[:\\s]+(YES)\\b`, "i");
262+
const noPattern = new RegExp(`\\b${escaped}\\b[:\\s]+(NO)\\b`, "i");
263+
if (yesPattern.test(logContent)) return "YES";
264+
if (noPattern.test(logContent)) return "NO";
265+
return "UNKNOWN";
266+
}
267+
268+
module.exports = { main, setupMain, parseMain };

‎setup/js/safe_outputs_tools.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1434,7 +1434,7 @@
14341434
},
14351435
{
14361436
"name": "set_issue_field",
1437-
"description": "Set a single GitHub issue field by name and value. Use field_name for discovery by field label (for example, \"Priority\"), or provide field_node_id to skip discovery. Supports text, number, date (YYYY-MM-DD), and single-select fields (value must match an option name).",
1437+
"description": "Set a single GitHub issue custom field by name and value. Use field_name for discovery by field label (for example, \"Priority\"), or provide field_node_id to skip discovery. Supports text, number, date (YYYY-MM-DD), and single-select fields (value must match an option name). Does NOT support builtin issue fields such as \"title\", \"body\", or \"state\" — use the update_issue tool for those (for open/closed state, use update_issue.status).",
14381438
"inputSchema": {
14391439
"type": "object",
14401440
"required": ["value"],

‎setup/js/set_issue_field.cjs‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ const { resolveTargetRepoConfig, resolveAndValidateRepo } = require("./repo_help
1010
const { logStagedPreviewInfo } = require("./staged_preview.cjs");
1111
const { isStagedMode, checkRequiredFilter } = require("./safe_output_helpers.cjs");
1212
const { createAuthenticatedGitHubClient } = require("./handler_auth.cjs");
13-
const { parseAllowedIssueFields, validateAllowedIssueFieldName } = require("./allowed_issue_fields.cjs");
13+
const { parseAllowedIssueFields, validateAllowedIssueFieldName, BUILTIN_ISSUE_FIELD_NAMES } = require("./allowed_issue_fields.cjs");
1414
const { resolveSafeOutputIssueTarget } = require("./temporary_id.cjs");
1515
const { hasIssueIntentsRuntimeFeature, normalizeIssueIntentMetadata } = require("./issue_intents.cjs");
1616

@@ -249,6 +249,14 @@ async function main(config = {}) {
249249
};
250250
}
251251

252+
if (fieldName && BUILTIN_ISSUE_FIELD_NAMES.has(fieldName.toLowerCase())) {
253+
const stateHint = fieldName.toLowerCase() === "state" ? ' For open/closed status, use update_issue.status ("open" or "closed").' : "";
254+
return {
255+
success: false,
256+
error: `Cannot set builtin issue field "${fieldName}" with set_issue_field. Use the update_issue tool instead.${stateHint}`,
257+
};
258+
}
259+
252260
if (isStaged) {
253261
const description = `Would set issue field ${JSON.stringify(fieldName || fieldNodeId)} to ${JSON.stringify(value)} on issue #${issueNumber} in ${itemRepo}`;
254262
logStagedPreviewInfo(description);

‎setup/setup.sh‎

Lines changed: 25 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -91,8 +91,31 @@ debug_log "Safe-output custom tokens support: ${SAFE_OUTPUT_CUSTOM_TOKENS_ENABLE
9191
create_dir "${DESTINATION}"
9292
debug_log "Created directory: ${DESTINATION}"
9393

94-
# Remove and recreate /tmp/gh-aw directory to ensure a clean state
95-
rm -rf /tmp/gh-aw
94+
# Remove and recreate /tmp/gh-aw directory to ensure a clean state.
95+
# On persistent runners, a previous AWF run may leave this directory (or subdirectories
96+
# like sandbox/firewall/) owned by root. Plain rm -rf fails with EACCES in that case,
97+
# so we fall back to sudo rm -rf which is available passwordless on GitHub-hosted runners.
98+
if [ -d /tmp/gh-aw ] && [ ! -w /tmp/gh-aw ]; then
99+
debug_log "/tmp/gh-aw exists but is not writable (likely root-owned from prior run); using sudo to remove"
100+
if command -v sudo >/dev/null 2>&1; then
101+
sudo -n rm -rf /tmp/gh-aw
102+
else
103+
echo "::error::/tmp/gh-aw exists but is not writable, and sudo is not available to reclaim it."
104+
exit 1
105+
fi
106+
elif [ -d /tmp/gh-aw ]; then
107+
# Directory is writable — but subdirectories may not be (e.g., sandbox/firewall/ owned by root).
108+
# Attempt plain rm first; if it fails, escalate to sudo.
109+
if ! rm -rf /tmp/gh-aw 2>/dev/null; then
110+
debug_log "/tmp/gh-aw has non-writable children (likely root-owned from prior AWF run); using sudo to remove"
111+
if command -v sudo >/dev/null 2>&1; then
112+
sudo -n rm -rf /tmp/gh-aw
113+
else
114+
echo "::error::/tmp/gh-aw contains non-writable children, and sudo is not available to reclaim it."
115+
exit 1
116+
fi
117+
fi
118+
fi
96119
mkdir -p /tmp/gh-aw
97120
debug_log "Created /tmp/gh-aw directory"
98121

0 commit comments

Comments
 (0)