Skip to content

Commit a3954aa

Browse files
fix(cli): persist what the server answered for a captured page (#3553)
* fix(cli): persist what the server answered for a captured page A page that renders is not a page that succeeded. An error page has a title, a palette, typefaces and a DOM, so every extractor downstream reads it happily and produces a design system belonging to whoever wrote the error page rather than to the site's owner. The status was already read, once, to feed `detectBlockedPage`, and then dropped. That helper cannot stand in for it: it decides whether the rendered document LOOKS like a protection wall, over a minimal-DOM heuristic that a rich error page passes. "Was this blocked?" and "what did the server answer?" are two questions, and widening the first to carry the second would leave a heuristic owning a fact. So the response status is persisted plainly, as its own record, and every consumer decides for itself what a non-success response means for its product. Written before the blocked-page check runs, so the record's absence means "navigation never produced a response" — a third state distinct from a status of 404 and from a status of null. * fix(cli): report the response status in the capture command's JSON output The status reached `CaptureResult` and was then dropped at the CLI boundary, which is the same read-once-and-discard that made the error page harvestable in the first place. `--json` is the documented programmatic surface, and an agent reading `ok: true` off a capture of a 404 has no way to see it there.
1 parent c8300c6 commit a3954aa

5 files changed

Lines changed: 93 additions & 1 deletion

File tree

‎packages/cli/src/capture/index.ts‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@ import {
4949
import type { VisionCaptionOutcome } from "./contentExtractor.js";
5050
import { loadEnvFile, generateProjectScaffold } from "./scaffolding.js";
5151
import { detectBlockedPage } from "./pageBlockDetection.js";
52+
import { writeResponseRecord } from "./responseRecord.js";
5253
import { navigateForCapture } from "./navigateForCapture.js";
5354
import {
5455
captureProtocolTimeoutMs,
@@ -302,8 +303,14 @@ export async function captureWebsite(
302303
progress("warn", message);
303304
}
304305

306+
// Persisted before the blocked-page check, so a capture that reaches navigation always leaves
307+
// a record of what the server said. That makes the file's ABSENCE mean "capture never got a
308+
// response", which is a third state distinct from a status of 404 and from a status of null.
309+
const httpStatus = navigationResponse?.status() ?? null;
310+
writeResponseRecord(join(outputDir, "extracted"), { status: httpStatus });
311+
305312
const blockedReason = detectBlockedPage({
306-
httpStatus: navigationResponse?.status() ?? null,
313+
httpStatus,
307314
...(contentCheckTimedOut
308315
? {
309316
title: "",
@@ -911,6 +918,7 @@ export async function captureWebsite(
911918
ok: true,
912919
projectDir: outputDir,
913920
url,
921+
httpStatus,
914922
title: tokens.title,
915923
extracted,
916924
screenshots,
Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
import { mkdtempSync, mkdirSync, readFileSync, rmSync } from "node:fs";
2+
import { tmpdir } from "node:os";
3+
import { join } from "node:path";
4+
import { afterEach, beforeEach, describe, expect, it } from "vitest";
5+
import {
6+
RESPONSE_RECORD_FILENAME,
7+
writeResponseRecord,
8+
type CaptureResponseRecord,
9+
} from "./responseRecord.js";
10+
11+
describe("writeResponseRecord", () => {
12+
let extractedDir: string;
13+
let projectDir: string;
14+
15+
beforeEach(() => {
16+
projectDir = mkdtempSync(join(tmpdir(), "hf-response-record-"));
17+
extractedDir = join(projectDir, "extracted");
18+
mkdirSync(extractedDir);
19+
});
20+
21+
afterEach(() => {
22+
rmSync(projectDir, { recursive: true, force: true });
23+
});
24+
25+
const readBack = (): CaptureResponseRecord =>
26+
JSON.parse(readFileSync(join(extractedDir, RESPONSE_RECORD_FILENAME), "utf-8"));
27+
28+
it("records the status the server answered", () => {
29+
writeResponseRecord(extractedDir, { status: 404 });
30+
31+
expect(readBack()).toEqual({ status: 404 });
32+
});
33+
34+
it("keeps a missing status distinguishable from a status of zero and from success", () => {
35+
writeResponseRecord(extractedDir, { status: null });
36+
const record = readBack();
37+
38+
// The three states a consumer must be able to separate: the server answered, the server
39+
// answered nothing, and — never — a falsy stand-in that reads as either.
40+
expect(record.status).toBeNull();
41+
expect(record.status).not.toBe(0);
42+
expect("status" in record).toBe(true);
43+
});
44+
});
Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
import { writeFileSync } from "node:fs";
2+
import { join } from "node:path";
3+
4+
/**
5+
* What the server answered for the captured page, written beside the extraction.
6+
*
7+
* A capture that renders is not the same fact as a capture that succeeded: an error page has a
8+
* title, colors, typefaces and a DOM, so every extractor downstream reads it happily and produces
9+
* a design system belonging to whoever wrote the error page. `detectBlockedPage` cannot answer
10+
* this — it decides whether the page LOOKS like a protection wall, which is a heuristic over the
11+
* rendered document, and a rich 404 passes it. So the status is persisted as its own plain fact
12+
* and consumers decide for themselves what a non-success response means for their product.
13+
*/
14+
export const RESPONSE_RECORD_FILENAME = "response.json";
15+
16+
export interface CaptureResponseRecord {
17+
/**
18+
* The final response's status after redirects, or null when navigation produced no response at
19+
* all. Null is NOT "fine": it means we never learned what the server said, which is a different
20+
* fact from a 200 and from a 404, and a consumer must be able to tell the three apart.
21+
*/
22+
status: number | null;
23+
}
24+
25+
/** Writes the record into an already-created `extracted/` directory. */
26+
export function writeResponseRecord(extractedDir: string, record: CaptureResponseRecord): void {
27+
writeFileSync(
28+
join(extractedDir, RESPONSE_RECORD_FILENAME),
29+
JSON.stringify(record, null, 2),
30+
"utf-8",
31+
);
32+
}

‎packages/cli/src/capture/types.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,11 @@ export interface CaptureResult {
6969
projectDir: string;
7070
/** Source URL */
7171
url: string;
72+
/**
73+
* What the server answered for `url`, after redirects; null when navigation produced no
74+
* response. Also persisted to `extracted/response.json` for out-of-process consumers.
75+
*/
76+
httpStatus: number | null;
7277
/** Page title */
7378
title: string;
7479
/** Extracted HTML data */

‎packages/cli/src/commands/capture.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -204,6 +204,9 @@ export default defineCommand({
204204
ok: result.ok,
205205
projectDir: result.projectDir,
206206
url: result.url,
207+
// Reported beside `ok`, because they answer different questions: a capture of an
208+
// error page is `ok: true` with a status the caller has to see to know it.
209+
httpStatus: result.httpStatus,
207210
title: result.title,
208211
screenshots: result.screenshots.length,
209212
assets: result.assets.length,

0 commit comments

Comments
 (0)