Skip to content

Commit 068cbe1

Browse files
committed
Add the offline secret scanner over the Secretlint preset
Integrate @secretlint/core and the recommended preset as the detector for journal content, with safe findings on top. Findings carry ruleId, messageId, location, and a fingerprint — never the matched value, and never Secretlint's result, which includes sourceContent. The fingerprint is an HMAC under a key generated per execution rather than a plain hash, so it cannot confirm a guessed secret; it is comparable only within the run that produced it. SecretDetectedError takes an immutable snapshot: each finding and its nested location are copied and frozen, not just the array, so the account of a rejection cannot be edited by whoever receives it. Scanner failures fail closed and drop the underlying error, which can quote scanned content. An internal seam over the detector call makes that provable — the real detector cannot be made to throw on demand — and is not exported from the package. Refs #199
1 parent 8183eb7 commit 068cbe1

9 files changed

Lines changed: 616 additions & 0 deletions

File tree

‎deno.json‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,9 @@
1414
"effection/experimental": "npm:[email protected]/experimental",
1515
"ajv": "npm:ajv@^8.17.1",
1616
"@effectionx/context-api": "npm:@effectionx/[email protected]",
17+
"@secretlint/core": "npm:@secretlint/[email protected]",
18+
"@secretlint/secretlint-rule-preset-recommend": "npm:@secretlint/[email protected]",
19+
"@secretlint/types": "npm:@secretlint/[email protected]",
1720
"@effectionx/converge": "npm:@effectionx/[email protected]",
1821
"@effectionx/fetch": "npm:@effectionx/[email protected]",
1922
"@effectionx/fs": "npm:@effectionx/[email protected]",

‎deno.lock‎

Lines changed: 39 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎packages/core/deno.json‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,9 @@
66
},
77
"imports": {
88
"@effectionx/context-api": "npm:@effectionx/[email protected]",
9+
"@secretlint/core": "npm:@secretlint/[email protected]",
10+
"@secretlint/secretlint-rule-preset-recommend": "npm:@secretlint/[email protected]",
11+
"@secretlint/types": "npm:@secretlint/[email protected]",
912
"acorn": "npm:acorn@^8.16.0",
1013
"ajv": "npm:ajv@^8.17.1",
1114
"gray-matter": "npm:gray-matter@^4.0.3",

‎packages/core/mod.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -139,3 +139,10 @@ export {
139139
// Shared execution config — contextual timeout (re-exported from runtime).
140140
export { Config, timeout } from "@executablemd/runtime";
141141
export type { ConfigApi } from "@executablemd/runtime";
142+
143+
// Secret detection — the offline scanner and its safe findings.
144+
// The rules themselves stay internal: they are policy, not API.
145+
export { createSecretScanner } from "./src/secrets/scanner.ts";
146+
export type { SecretScanner } from "./src/secrets/scanner.ts";
147+
export { SecretDetectedError, SecretScannerError } from "./src/secrets/findings.ts";
148+
export type { SecretFinding } from "./src/secrets/findings.ts";

‎packages/core/package.json‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,9 @@
1919
"@effectionx/timebox": "0.4.3",
2020
"@executablemd/durable-streams": "workspace:*",
2121
"@executablemd/runtime": "workspace:*",
22+
"@secretlint/core": "13.0.4",
23+
"@secretlint/secretlint-rule-preset-recommend": "13.0.4",
24+
"@secretlint/types": "13.0.4",
2225
"acorn": "^8.16.0",
2326
"effection": "4.1.0-alpha.10",
2427
"gray-matter": "^4.0.3",
Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
/**
2+
* Safe findings.
3+
*
4+
* A Secretlint result carries `sourceContent` — the complete text that was
5+
* scanned. Nothing in this module ever stores, returns, throws, or logs one.
6+
* A raw result is normalized to a {@link SecretFinding} at the moment it is
7+
* produced and then dropped, so no later code has the chance to leak it.
8+
*/
9+
10+
import { createHmac } from "node:crypto";
11+
12+
/**
13+
* One detection, carrying nothing that could reconstruct the secret.
14+
*
15+
* The matched value is deliberately absent, and so is Secretlint's message —
16+
* `maskSecrets: true` masks it today, but a field that is never populated
17+
* cannot be un-masked by a future change to how masking works.
18+
*/
19+
export interface SecretFinding {
20+
/** The rule that matched, e.g. `@secretlint/secretlint-rule-github`. */
21+
ruleId: string;
22+
/** The rule's message identifier, e.g. `GITHUB_TOKEN`. */
23+
messageId: string;
24+
/** Where in the scanned text the match began. */
25+
location: {
26+
line: number;
27+
column: number;
28+
};
29+
/**
30+
* A keyed digest of the matched value, comparable only within the execution
31+
* that produced it. See {@link createFingerprinter}.
32+
*/
33+
fingerprint: string;
34+
}
35+
36+
/**
37+
* Derives fingerprints for one execution.
38+
*
39+
* The digest is an HMAC under a key generated in memory for this execution,
40+
* not a plain hash. A plain hash would be an oracle: anyone holding a
41+
* candidate secret could confirm it by recomputing the digest and comparing.
42+
* Keying it removes that, and costs only cross-execution comparability, which
43+
* nothing needs — a fingerprint exists to tell "this same value appeared
44+
* twice in this run" from "these are two different values".
45+
*
46+
* The key never leaves this closure: it is not persisted, logged, exposed on
47+
* a finding, or reachable from one.
48+
*/
49+
export function createFingerprinter(): (matched: string) => string {
50+
const key = crypto.getRandomValues(new Uint8Array(32));
51+
52+
return function fingerprint(matched: string): string {
53+
return createHmac("sha256", key).update(matched).digest("hex").slice(0, 32);
54+
};
55+
}
56+
57+
/**
58+
* Normalize one Secretlint message into a safe finding.
59+
*
60+
* `source` is the text that was scanned. The matched slice is read from it,
61+
* handed straight to the fingerprinter, and goes out of scope when this
62+
* function returns — it is never assigned to the finding.
63+
*/
64+
export function toSecretFinding(
65+
message: {
66+
ruleId: string;
67+
messageId: string;
68+
range: readonly [number, number];
69+
loc?: { start: { line: number; column: number } };
70+
},
71+
source: string,
72+
fingerprint: (matched: string) => string,
73+
): SecretFinding {
74+
return {
75+
ruleId: message.ruleId,
76+
messageId: message.messageId,
77+
location: {
78+
line: message.loc?.start.line ?? 1,
79+
column: message.loc?.start.column ?? 0,
80+
},
81+
fingerprint: fingerprint(source.slice(message.range[0], message.range[1])),
82+
};
83+
}
84+
85+
function describe(findings: readonly SecretFinding[]): string {
86+
return findings
87+
.map((finding) => ` ${finding.ruleId} at ${finding.location.line}:${finding.location.column}`)
88+
.join("\n");
89+
}
90+
91+
/**
92+
* A credential was found in content that was about to be persisted.
93+
*
94+
* There is no allowlist, approval, or repair: a finding means a code or
95+
* data-flow defect that has to be fixed. The message names the rules and
96+
* positions and nothing else, so this error stays safe when it is serialized
97+
* into a close event or rendered as a diagnostic.
98+
*/
99+
export class SecretDetectedError extends Error {
100+
readonly findings: readonly SecretFinding[];
101+
102+
constructor(findings: readonly SecretFinding[]) {
103+
super(
104+
`secret detection rejected content before it was persisted:\n${describe(findings)}\n` +
105+
`Fix the code or data flow that produced it.`,
106+
);
107+
this.name = "SecretDetectedError";
108+
this.findings = snapshot(findings);
109+
}
110+
}
111+
112+
/**
113+
* An immutable copy of what was found.
114+
*
115+
* Freezing only the array would leave every finding — and every nested
116+
* `location` — writable through it, so a caller could edit the record of a
117+
* rejection after the fact. This error is the account of why content was
118+
* refused; it should not be editable by whoever receives it.
119+
*/
120+
function snapshot(findings: readonly SecretFinding[]): readonly SecretFinding[] {
121+
return Object.freeze(
122+
findings.map((finding) =>
123+
Object.freeze({
124+
ruleId: finding.ruleId,
125+
messageId: finding.messageId,
126+
location: Object.freeze({
127+
line: finding.location.line,
128+
column: finding.location.column,
129+
}),
130+
fingerprint: finding.fingerprint,
131+
}),
132+
),
133+
);
134+
}
135+
136+
/**
137+
* The detector itself failed, so the content could not be cleared.
138+
*
139+
* Failing closed is the point: an unusable detector must not become an open
140+
* gate. The originating error is deliberately dropped rather than attached as
141+
* `cause`, because a detector stack trace can quote the content it choked on.
142+
*/
143+
export class SecretScannerError extends Error {
144+
constructor(stage: string) {
145+
super(
146+
`secret detection failed during ${stage}, so the content was not persisted. ` +
147+
`The underlying error is withheld because it can contain scanned content.`,
148+
);
149+
this.name = "SecretScannerError";
150+
}
151+
}
Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
/**
2+
* The offline secret scanner.
3+
*
4+
* One scanner is created per execution. It pins its rule configuration at
5+
* creation and generates the HMAC key its fingerprints are derived under, so
6+
* fingerprints are comparable within an execution and meaningless outside it.
7+
*
8+
* The scanner performs no network access, no rule download, no telemetry, and
9+
* no subprocess execution. Everything it needs is resolved from the module
10+
* graph at import time.
11+
*/
12+
13+
import { until } from "effection";
14+
import type { Operation } from "effection";
15+
import { lintSource } from "@secretlint/core";
16+
import { creator as recommended } from "@secretlint/secretlint-rule-preset-recommend";
17+
import { createFingerprinter, SecretScannerError, toSecretFinding } from "./findings.ts";
18+
import type { SecretFinding } from "./findings.ts";
19+
20+
/** Scans text for credentials. Created once per execution. */
21+
export interface SecretScanner {
22+
/** Findings for `content`, empty when nothing matched. */
23+
scan(content: string): Operation<SecretFinding[]>;
24+
}
25+
26+
/** The subset of a Secretlint message a finding is built from. */
27+
interface LintMessage {
28+
ruleId: string;
29+
messageId: string;
30+
range: readonly [number, number];
31+
loc?: { start: { line: number; column: number } };
32+
}
33+
34+
/**
35+
* The detector call, isolated so a test can substitute a failing one.
36+
*
37+
* Deliberately not re-exported from the package: proving the scanner fails
38+
* closed requires a detector that throws, and there is no way to make the real
39+
* one throw on demand. Substituting it is a test concern, not an API.
40+
*/
41+
export type LintFn = (
42+
content: string,
43+
filePath: string,
44+
) => Promise<{
45+
messages: readonly LintMessage[];
46+
}>;
47+
48+
const CONFIG = {
49+
rules: [
50+
{
51+
id: "@secretlint/secretlint-rule-preset-recommend",
52+
rule: recommended,
53+
},
54+
],
55+
};
56+
57+
const lint: LintFn = (content, filePath) =>
58+
lintSource({
59+
source: { filePath, content, ext: ".jsonl", contentType: "text" },
60+
options: {
61+
config: CONFIG,
62+
// Masked messages are never read — findings carry no message at all —
63+
// but masking keeps the value out of anything Secretlint might log on
64+
// its own.
65+
maskSecrets: true,
66+
},
67+
});
68+
69+
/**
70+
* Create the scanner for one execution.
71+
*
72+
* `filePath` is what Secretlint reports findings against. It is a label only —
73+
* nothing is read from disk.
74+
*/
75+
export function createSecretScanner(filePath = "/journal.jsonl"): SecretScanner {
76+
return createScannerWith(lint, filePath);
77+
}
78+
79+
/**
80+
* Internal seam: the scanner over an arbitrary detector.
81+
*
82+
* Not exported from `mod.ts`. See {@link LintFn}.
83+
*/
84+
export function createScannerWith(detect: LintFn, filePath = "/journal.jsonl"): SecretScanner {
85+
const fingerprint = createFingerprinter();
86+
87+
return {
88+
*scan(content: string): Operation<SecretFinding[]> {
89+
if (content.length === 0) {
90+
return [];
91+
}
92+
93+
let messages: readonly LintMessage[];
94+
95+
try {
96+
// `until` releases this operation on cancellation but cannot cancel
97+
// the detector's promise; an in-flight scan may finish and be dropped.
98+
// What matters is that a cancelled scan never reaches its caller, so
99+
// nothing downstream persists.
100+
//
101+
// The result also carries `sourceContent`. It is normalized here and
102+
// never assigned anywhere that outlives this function.
103+
messages = (yield* until(detect(content, filePath))).messages;
104+
} catch {
105+
// The detector's own error can quote the content it choked on, so it
106+
// is dropped rather than wrapped.
107+
throw new SecretScannerError("scanning");
108+
}
109+
110+
try {
111+
return messages.map((message) => toSecretFinding(message, content, fingerprint));
112+
} catch {
113+
throw new SecretScannerError("normalizing findings");
114+
}
115+
},
116+
};
117+
}

0 commit comments

Comments
 (0)