Skip to content

Commit 30bd705

Browse files
committed
fix(ci): close freeform issue-quality template bypass
Unclassified API/freeform issues no longer skip the gate; non-trusted authors are closed with template guidance (including #521-shaped near-miss headings).
1 parent 6d8f05f commit 30bd705

3 files changed

Lines changed: 222 additions & 6 deletions

File tree

‎.github/scripts/issue-quality.cjs‎

Lines changed: 99 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -420,9 +420,99 @@ function isRawPlaceholder(raw) {
420420
return isPlaceholderOnlyValue(raw);
421421
}
422422

423+
/**
424+
* Near-miss freeform headings that often appear in API-opened or copy-pasted
425+
* bug reports instead of the Bug report template (e.g. Description / Log entry).
426+
*/
427+
const FREEFORM_BUG_NEAR_MISS_HEADINGS = [
428+
"Description",
429+
"Steps to reproduce",
430+
"Steps to Reproduce",
431+
"How to reproduce",
432+
"Log",
433+
"Logs",
434+
"Log entry",
435+
"Error",
436+
"Error output",
437+
"Stack trace",
438+
];
439+
440+
/**
441+
* True when an unclassified body looks like a bug report that skipped the
442+
* template (near-miss headings and/or repro/error signals).
443+
*
444+
* @param {{ title?: string, body?: string }} issue
445+
* @returns {boolean}
446+
*/
447+
function looksLikeUntemplatedBugReport(issue) {
448+
const body = typeof issue?.body === "string" ? issue.body : "";
449+
if (!body.trim()) return false;
450+
451+
const nearMissCount = countHeadings(body, FREEFORM_BUG_NEAR_MISS_HEADINGS);
452+
const hasReproduction =
453+
extractSection(body, "Reproduction") !== null ||
454+
extractSection(body, "Steps to reproduce") !== null ||
455+
extractSection(body, "Steps to Reproduce") !== null ||
456+
extractSection(body, "How to reproduce") !== null;
457+
const hasDescription = extractSection(body, "Description") !== null;
458+
const hasLogOrError =
459+
extractSection(body, "Log") !== null ||
460+
extractSection(body, "Logs") !== null ||
461+
extractSection(body, "Log entry") !== null ||
462+
extractSection(body, "Logs or error output") !== null ||
463+
extractSection(body, "Error") !== null ||
464+
extractSection(body, "Error output") !== null ||
465+
extractSection(body, "Stack trace") !== null;
466+
467+
if (hasDescription && (hasReproduction || hasLogOrError)) return true;
468+
if (hasReproduction && hasLogOrError) return true;
469+
if (nearMissCount >= 2) return true;
470+
471+
// Body-level signals for heading-free freeform dumps.
472+
const signalRe =
473+
/\b(repro(?:duce|duction| steps)?|stack\s*traces?|traceback|segfault|panic|exception|error\s*output|ECONNREFUSED|SIGSEGV)\b/i;
474+
if (signalRe.test(body) && (hasDescription || hasReproduction || hasLogOrError || nearMissCount >= 1)) {
475+
return true;
476+
}
477+
return false;
478+
}
479+
480+
/**
481+
* Reasons/guidance when no structured issue kind was detected.
482+
*
483+
* @param {{ title?: string, body?: string }} issue
484+
* @returns {{ reasons: string[], guidance: string[] }}
485+
*/
486+
function untemplatedIssueFailure(issue) {
487+
if (looksLikeUntemplatedBugReport(issue)) {
488+
return {
489+
reasons: [
490+
"This looks like a bug report but it does not use the Bug report template headings (for example Description/Log entry instead of Summary).",
491+
],
492+
guidance: [
493+
"Use the Bug report template, or edit this issue to include: Client or integration, Summary, Reproduction, Version, and Operating system.",
494+
"Retitling with `[Bug]:` and applying the `bug` label alone is not enough without those section headings filled in.",
495+
],
496+
};
497+
}
498+
return {
499+
reasons: [
500+
"This issue does not use a recognized issue template.",
501+
],
502+
guidance: [
503+
"Open a new issue with the Bug report, Feature request, Documentation, or Provider compatibility template.",
504+
"Or edit this issue so the body uses the template section headings for the kind of report you are filing.",
505+
],
506+
};
507+
}
508+
423509
/**
424510
* Validate an issue body for its detected kind.
425511
*
512+
* Unclassified (freeform / non-template) issues are invalid so API-opened
513+
* reports cannot skip the quality gate. Trusted-author exemption is enforced
514+
* by the workflow, not here.
515+
*
426516
* @param {{ title: string, body: string, labels: string[], storedKind?: string|null }} issue
427517
* @returns {{ kind: string|null, valid: boolean, softPass: boolean, reasons: string[], guidance: string[] }}
428518
*/
@@ -435,8 +525,14 @@ function validateIssue(issue) {
435525
const titleLower = title.toLowerCase();
436526

437527
if (!kind) {
438-
// Not a structured form we validate.
439-
return { kind: null, valid: true, softPass: false, reasons: [], guidance: [] };
528+
const failure = untemplatedIssueFailure(issue);
529+
return {
530+
kind: null,
531+
valid: false,
532+
softPass: false,
533+
reasons: failure.reasons,
534+
guidance: failure.guidance,
535+
};
440536
}
441537

442538
if (kind === "feature") {
@@ -720,6 +816,7 @@ module.exports = {
720816
resolveSection,
721817
detectIssueKind,
722818
validateIssue,
819+
looksLikeUntemplatedBugReport,
723820
shouldReopen,
724821
shouldEnforceClosure,
725822
isPlaceholderOnlyValue,

‎.github/scripts/issue-quality.test.cjs‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ const {
99
extractSection,
1010
detectIssueKind,
1111
validateIssue,
12+
looksLikeUntemplatedBugReport,
1213
shouldReopen,
1314
shouldEnforceClosure,
1415
labelForKind,
@@ -1090,6 +1091,111 @@ describe("labelForKind", () => {
10901091
});
10911092
});
10921093

1094+
// ---------------------------------------------------------------------------
1095+
// Freeform / non-template bypass (e.g. issue #521)
1096+
// ---------------------------------------------------------------------------
1097+
1098+
describe("validateIssue - freeform / non-template", () => {
1099+
it("rejects a plain freeform body that previously skipped validation", () => {
1100+
const result = validateIssue({
1101+
title: "How do I configure?",
1102+
body: "Just a random question about setup.",
1103+
labels: [],
1104+
});
1105+
assert.equal(result.kind, null);
1106+
assert.equal(result.valid, false);
1107+
assert.equal(result.softPass, false);
1108+
assert.ok(result.reasons.some((r) => /recognized issue template/i.test(r)));
1109+
assert.ok(result.guidance.some((g) => /Bug report|Feature request/i.test(g)));
1110+
});
1111+
1112+
it("rejects a #521-shaped Description/Reproduction/Log entry body with a clear message", () => {
1113+
const body = [
1114+
"### Description",
1115+
"Proxy returns 502 when streaming is enabled on Windows.",
1116+
"### Reproduction",
1117+
"1. ocx start",
1118+
"2. Send a streaming request",
1119+
"3. Observe 502",
1120+
"### Log entry",
1121+
"```",
1122+
"upstream connect error or disconnect/reset before headers",
1123+
"```",
1124+
].join("\n");
1125+
assert.equal(
1126+
detectIssueKind({ title: "Proxy 502 on streaming", body, labels: [] }),
1127+
null,
1128+
"near-miss headings must not silently classify as a structured bug",
1129+
);
1130+
assert.equal(looksLikeUntemplatedBugReport({ title: "Proxy 502 on streaming", body }), true);
1131+
1132+
const result = validateIssue({
1133+
title: "Proxy 502 on streaming",
1134+
body,
1135+
labels: [],
1136+
});
1137+
assert.equal(result.kind, null);
1138+
assert.equal(result.valid, false);
1139+
assert.ok(
1140+
result.reasons.some((r) => /bug report/i.test(r) && /template/i.test(r)),
1141+
`Expected bug-template reason, got: ${result.reasons.join("; ")}`,
1142+
);
1143+
assert.ok(
1144+
result.guidance.some((g) => /Client or integration|Summary|Reproduction/i.test(g)),
1145+
`Expected template-heading guidance, got: ${result.guidance.join("; ")}`,
1146+
);
1147+
});
1148+
1149+
it("still detects and validates a real structured bug as before", () => {
1150+
const body = [
1151+
"### Client or integration",
1152+
"Codex CLI",
1153+
"### Summary",
1154+
"Proxy segfaults on ARM64 when streaming is enabled.",
1155+
"### Reproduction",
1156+
"ocx start on Raspberry Pi 4, send any streaming request.",
1157+
"### Version",
1158+
"2.7.30",
1159+
"### Operating system",
1160+
"Debian 12 aarch64",
1161+
].join("\n");
1162+
const result = validateIssue({
1163+
title: "Segfault on ARM64 streaming",
1164+
body,
1165+
labels: ["bug"],
1166+
});
1167+
assert.equal(result.kind, "bug");
1168+
assert.equal(result.valid, true);
1169+
});
1170+
1171+
it("does not treat Summary+Reproduction alone as a bug without prefix or label", () => {
1172+
// Existing anti-false-positive rule; freeform gate still fails these as untemplated.
1173+
const body = [
1174+
"### Summary",
1175+
"Something went wrong in the proxy.",
1176+
"### Reproduction",
1177+
"Run ocx start.",
1178+
].join("\n");
1179+
assert.equal(detectIssueKind({ title: "Something went wrong", body, labels: [] }), null);
1180+
const result = validateIssue({ title: "Something went wrong", body, labels: [] });
1181+
assert.equal(result.kind, null);
1182+
assert.equal(result.valid, false);
1183+
});
1184+
1185+
it("keeps label-backed storedKind validation for enhancement freeform", () => {
1186+
// Workflow passes storedKind from the enhancement label; empty feature form still fails.
1187+
const result = validateIssue({
1188+
title: "How do I configure?",
1189+
body: "Just a random question about setup.",
1190+
labels: ["enhancement"],
1191+
storedKind: "feature",
1192+
});
1193+
assert.equal(result.kind, "feature");
1194+
assert.equal(result.valid, false);
1195+
assert.ok(result.reasons.some((r) => /missing or empty/i.test(r)));
1196+
});
1197+
});
1198+
10931199
// ---------------------------------------------------------------------------
10941200
// workflow_dispatch guards
10951201
// ---------------------------------------------------------------------------

‎.github/workflows/enforce-issue-quality.yml‎

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -540,8 +540,13 @@ jobs:
540540
return null;
541541
})();
542542
if (!kind && !labelBasedKind) {
543-
core.info("Issue is not a structured form; skipping validation.");
544-
return;
543+
// Freeform / API-opened issues used to skip here, which let reports
544+
// like non-template Description/Reproduction bodies bypass the gate.
545+
// Continue into validateIssue (invalid) so non-trusted authors are
546+
// closed with template guidance; trusted authors remain exempt below.
547+
core.info(
548+
"Issue is not a structured form; requiring a template for non-trusted authors.",
549+
);
545550
}
546551
const resolvedKind = kind ?? labelBasedKind;
547552
@@ -680,6 +685,7 @@ jobs:
680685
// Close (or re-close) the issue.
681686
const reasonList = result.reasons.map((r) => `- ${r}`).join("\n");
682687
const guidanceList = result.guidance.map((g) => `- ${g}`).join("\n");
688+
const missingTemplate = !resolvedKind;
683689
684690
// Close as not_planned.
685691
await github.rest.issues.update({
@@ -699,13 +705,20 @@ jobs:
699705
stateReason: closedIssue.state_reason || "not_planned",
700706
};
701707
708+
const closeHeading = missingTemplate
709+
? "### Issue closed: use an issue template"
710+
: "### Issue closed: insufficient detail";
711+
const closeIntro = missingTemplate
712+
? "Thanks for taking the time to submit this issue. It was closed automatically because it was not opened with a recognized issue template (Bug report, Feature request, Documentation, or Provider compatibility). Freeform and API-opened issues cannot skip this check."
713+
: "Thanks for taking the time to submit this issue. It was closed automatically because the structured report is missing information needed to evaluate or reproduce it.";
714+
702715
await upsertComment([
703716
BOT_MARKER,
704717
stateTag(newBotState),
705718
"",
706-
"### Issue closed: insufficient detail",
719+
closeHeading,
707720
"",
708-
"Thanks for taking the time to submit this issue. It was closed automatically because the structured report is missing information needed to evaluate or reproduce it.",
721+
closeIntro,
709722
"",
710723
reasonList,
711724
"",

0 commit comments

Comments
 (0)