Skip to content

Commit 8ea4c3b

Browse files
t3dotggclaude
andcommitted
fix(release): Windows signing retries transient Azure Trusted Signing errors
Azure Trusted Signing sometimes answers with InternalError, which failed both Windows release legs three times in one hour. jsign now retries only recognized transient errors (InternalError, HTTP 429/5xx, dropped connections) up to three times with backoff. Each attempt signs a fresh copy that replaces the file only after jsign succeeds, so a failed attempt never changes the input. Credential and input errors still fail at once. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 51675e0 commit 8ea4c3b

2 files changed

Lines changed: 239 additions & 28 deletions

File tree

‎scripts/lib/windows-jsign.test.ts‎

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
import * as NodeServices from "@effect/platform-node/NodeServices";
2+
import { assert, it } from "@effect/vitest";
3+
import * as Cause from "effect/Cause";
4+
import * as Clock from "effect/Clock";
5+
import * as ConfigProvider from "effect/ConfigProvider";
6+
import * as Duration from "effect/Duration";
7+
import * as Effect from "effect/Effect";
8+
import * as Exit from "effect/Exit";
9+
import * as FileSystem from "effect/FileSystem";
10+
import * as Option from "effect/Option";
11+
import * as Path from "effect/Path";
12+
import * as Sink from "effect/Sink";
13+
import * as Stream from "effect/Stream";
14+
import { ChildProcess, ChildProcessSpawner } from "effect/process";
15+
16+
import { signWithJsign } from "./windows-jsign.ts";
17+
18+
const UNSIGNED = "MZ unsigned payload";
19+
const TRANSIENT_CHAIN =
20+
"jsign: Unable to retrieve the certificate chain 'acct/profile'\nCaused by: java.io.IOException: InternalError - An internal error has occurred.\n";
21+
const TRANSIENT_SIGN =
22+
"jsign: Couldn't sign t3.exe\nCaused by: java.security.GeneralSecurityException: java.io.IOException: InternalError - An internal error has occurred.\n";
23+
const BAD_SECRET =
24+
"jsign: Unable to retrieve the certificate chain 'acct/profile'\nCaused by: java.io.IOException: AADSTS7000215: Invalid client secret provided.\n";
25+
26+
/** What the stub does with the file jsign was given: sign it, or write junk and fail. */
27+
type Reply = { readonly exitCode: number; readonly stderr?: string; readonly write?: string };
28+
29+
/** Stands in for jsign. Records the bytes it received and applies each reply in turn. */
30+
function stubJsign(replies: ReadonlyArray<Reply>) {
31+
const inputs: Array<{ readonly path: string; readonly content: string }> = [];
32+
const spawner = ChildProcessSpawner.make((command) =>
33+
Effect.gen(function* () {
34+
const fs = yield* FileSystem.FileSystem;
35+
const args = ChildProcess.isStandardCommand(command) ? command.args.map(String) : [];
36+
const target = args.at(-1)!;
37+
inputs.push({ path: target, content: yield* fs.readFileString(target) });
38+
const reply = replies[inputs.length - 1] ?? replies.at(-1)!;
39+
if (reply.write !== undefined) yield* fs.writeFileString(target, reply.write);
40+
return ChildProcessSpawner.makeHandle({
41+
pid: ChildProcessSpawner.ProcessId(1),
42+
exitCode: Effect.succeed(ChildProcessSpawner.ExitCode(reply.exitCode)),
43+
isRunning: Effect.succeed(false),
44+
kill: () => Effect.void,
45+
unref: Effect.succeed(Effect.void),
46+
stdin: Sink.drain,
47+
stdout: Stream.empty,
48+
stderr: reply.stderr ? Stream.make(new TextEncoder().encode(reply.stderr)) : Stream.empty,
49+
all: Stream.empty,
50+
getInputFd: () => Sink.drain,
51+
getOutputFd: () => Stream.empty,
52+
});
53+
}).pipe(Effect.provide(NodeServices.layer), Effect.orDie),
54+
);
55+
return { inputs, spawner };
56+
}
57+
58+
const signed = (content: string): Reply => ({ exitCode: 0, write: `${content}+signature` });
59+
60+
const runSign = Effect.fn("test.runSign")(function* (replies: ReadonlyArray<Reply>) {
61+
const fs = yield* FileSystem.FileSystem;
62+
const path = yield* Path.Path;
63+
const dir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-jsign-" });
64+
const file = path.join(dir, "t3.exe");
65+
yield* fs.writeFileString(file, UNSIGNED);
66+
const stub = stubJsign(replies);
67+
const sleeps: Array<number> = [];
68+
const clock = yield* Clock.Clock;
69+
const exit = yield* signWithJsign(file).pipe(
70+
Effect.provideService(ChildProcessSpawner.ChildProcessSpawner, stub.spawner),
71+
// Records the backoff instead of waiting it out.
72+
Effect.provideService(Clock.Clock, {
73+
...clock,
74+
sleep: (duration) => Effect.sync(() => sleeps.push(Duration.toSeconds(duration))),
75+
}),
76+
Effect.provide(
77+
ConfigProvider.layer(
78+
ConfigProvider.fromEnv({
79+
env: {
80+
AZURE_TRUSTED_SIGNING_ENDPOINT: "https://eus.codesigning.azure.net/",
81+
AZURE_TRUSTED_SIGNING_ACCOUNT_NAME: "acct",
82+
AZURE_TRUSTED_SIGNING_CERTIFICATE_PROFILE_NAME: "profile",
83+
},
84+
}),
85+
),
86+
),
87+
Effect.exit,
88+
);
89+
return {
90+
ok: Exit.isSuccess(exit),
91+
error: Exit.isFailure(exit)
92+
? Option.getOrUndefined(Cause.findErrorOption(exit.cause))
93+
: undefined,
94+
sleeps,
95+
inputs: stub.inputs,
96+
// jsign only ever sees a disposable copy, never the file itself.
97+
signedInPlace: stub.inputs.some((input) => input.path === file),
98+
content: yield* fs.readFileString(file),
99+
// Only the original file is left: every attempt's copy is cleaned up.
100+
entries: yield* fs.readDirectory(dir),
101+
};
102+
});
103+
104+
it.layer(NodeServices.layer)("signWithJsign", (it) => {
105+
it.effect("retries transient Azure errors on a fresh copy and keeps only the signed file", () =>
106+
Effect.gen(function* () {
107+
const result = yield* runSign([
108+
{ exitCode: 1, stderr: TRANSIENT_CHAIN },
109+
// A partial write must not reach the next attempt or the original.
110+
{ exitCode: 1, stderr: TRANSIENT_SIGN, write: "MZ half-written" },
111+
signed(UNSIGNED),
112+
]);
113+
assert.isTrue(result.ok);
114+
assert.strictEqual(result.content, `${UNSIGNED}+signature`);
115+
assert.deepStrictEqual(
116+
result.inputs.map((input) => input.content),
117+
[UNSIGNED, UNSIGNED, UNSIGNED],
118+
);
119+
assert.isFalse(result.signedInPlace);
120+
assert.deepStrictEqual(result.sleeps, [5, 10]);
121+
assert.deepStrictEqual(result.entries, ["t3.exe"]);
122+
}),
123+
);
124+
125+
it.effect("fails once without retrying on errors that are not transient", () =>
126+
Effect.gen(function* () {
127+
for (const reply of [
128+
{ exitCode: 1, stderr: BAD_SECRET },
129+
{ exitCode: 1, stderr: "jsign: Unsupported file: t3.exe\n", write: "MZ half-written" },
130+
]) {
131+
const result = yield* runSign([reply, signed(UNSIGNED)]);
132+
assert.strictEqual(result.error?._tag, "JsignFailedError");
133+
assert.isFalse(result.error?._tag === "JsignFailedError" && result.error.transient);
134+
assert.strictEqual(result.inputs.length, 1);
135+
assert.deepStrictEqual(result.sleeps, []);
136+
assert.strictEqual(result.content, UNSIGNED);
137+
assert.deepStrictEqual(result.entries, ["t3.exe"]);
138+
}
139+
}),
140+
);
141+
142+
it.effect("gives up after three retries and leaves the input unsigned", () =>
143+
Effect.gen(function* () {
144+
const result = yield* runSign([{ exitCode: 1, stderr: TRANSIENT_SIGN, write: "MZ junk" }]);
145+
assert.isTrue(result.error?._tag === "JsignFailedError" && result.error.transient);
146+
assert.strictEqual(result.inputs.length, 4);
147+
assert.deepStrictEqual(result.sleeps, [5, 10, 20]);
148+
assert.strictEqual(result.content, UNSIGNED);
149+
assert.deepStrictEqual(result.entries, ["t3.exe"]);
150+
}),
151+
);
152+
});

‎scripts/lib/windows-jsign.ts‎

Lines changed: 87 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -5,53 +5,112 @@
55
import * as NodeServices from "@effect/platform-node/NodeServices";
66
import * as Config from "effect/Config";
77
import * as Effect from "effect/Effect";
8+
import * as FileSystem from "effect/FileSystem";
9+
import * as Path from "effect/Path";
10+
import * as Schedule from "effect/Schedule";
811
import * as Schema from "effect/Schema";
12+
import * as Stream from "effect/Stream";
913
import { ChildProcess, ChildProcessSpawner } from "effect/process";
1014

15+
// Upstream failures worth another attempt: Azure's `InternalError` (seen in
16+
// release runs on both the certificate chain and the sign call), HTTP 429/5xx,
17+
// and dropped connections. Credential, permission and input errors never match.
18+
const TRANSIENT_JSIGN_ERROR =
19+
/\bInternalError - |HTTP Error (?:429|5\d\d)\b|java\.net\.(?:SocketTimeoutException|ConnectException)\b|Connection reset\b/;
20+
const JSIGN_RETRIES = 3;
21+
const STDERR_TAIL_CHARS = 64 * 1024;
22+
1123
export class JsignFailedError extends Schema.TaggedError<JsignFailedError>()("JsignFailedError", {
1224
file: Schema.String,
1325
exitCode: Schema.Number,
26+
transient: Schema.Boolean,
1427
}) {
1528
override get message() {
16-
return `jsign exited with code ${String(this.exitCode)} signing ${this.file}.`;
29+
return `jsign exited with code ${String(this.exitCode)} signing ${this.file}${this.transient ? " (transient Azure Trusted Signing error)" : ""}.`;
1730
}
1831
}
1932

33+
/** Runs jsign, forwarding its stderr as it arrives and returning a tail of it for classification. */
34+
const runJsign = Effect.fn("runJsign")(function* (jsign: string, args: ReadonlyArray<string>) {
35+
const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
36+
const child = yield* spawner.spawn(
37+
ChildProcess.make(jsign, args, { stdout: "inherit", stderr: "pipe" }),
38+
);
39+
const decoder = new TextDecoder();
40+
let stderr = "";
41+
const [exitCode] = yield* Effect.all(
42+
[
43+
child.exitCode.pipe(Effect.map(Number)),
44+
Stream.runForEach(child.stderr, (chunk) =>
45+
Effect.sync(() => {
46+
process.stderr.write(chunk);
47+
stderr = (stderr + decoder.decode(chunk, { stream: true })).slice(-STDERR_TAIL_CHARS);
48+
}),
49+
),
50+
],
51+
{ concurrency: "unbounded" },
52+
);
53+
return { exitCode, stderr };
54+
});
55+
2056
export const signWithJsign = Effect.fn("signWithJsign")(function* (file: string) {
2157
const { jsign, endpoint, account, profile } = yield* Config.all({
2258
jsign: Config.String("T3CODE_JSIGN").pipe(Config.withDefault("jsign")),
2359
endpoint: Config.String("AZURE_TRUSTED_SIGNING_ENDPOINT"),
2460
account: Config.String("AZURE_TRUSTED_SIGNING_ACCOUNT_NAME"),
2561
profile: Config.String("AZURE_TRUSTED_SIGNING_CERTIFICATE_PROFILE_NAME"),
2662
});
27-
const spawner = yield* ChildProcessSpawner.ChildProcessSpawner;
28-
const child = yield* spawner.spawn(
29-
ChildProcess.make(
30-
jsign,
31-
[
32-
"--storetype",
33-
"TRUSTEDSIGNING",
34-
"--keystore",
35-
new URL(endpoint).host,
36-
"--alias",
37-
`${account}/${profile}`,
38-
"--alg",
39-
"SHA-256",
40-
"--tsaurl",
41-
"http://timestamp.acs.microsoft.com",
42-
"--tsmode",
43-
"RFC3161",
44-
// jsign appends a nested signature by default; signtool replaces.
45-
"--replace",
46-
file,
47-
],
48-
{ stdout: "inherit", stderr: "inherit" },
49-
),
63+
const fs = yield* FileSystem.FileSystem;
64+
const path = yield* Path.Path;
65+
const args = [
66+
"--storetype",
67+
"TRUSTEDSIGNING",
68+
"--keystore",
69+
new URL(endpoint).host,
70+
"--alias",
71+
`${account}/${profile}`,
72+
"--alg",
73+
"SHA-256",
74+
"--tsaurl",
75+
"http://timestamp.acs.microsoft.com",
76+
"--tsmode",
77+
"RFC3161",
78+
// jsign appends a nested signature by default; signtool replaces.
79+
"--replace",
80+
];
81+
82+
let attempt = 0;
83+
// jsign rewrites its input in place, so each attempt signs a fresh copy and
84+
// only a copy jsign finished with replaces the file (same directory, so the
85+
// rename is atomic). A failed attempt never touches the original.
86+
const signCopy = Effect.gen(function* () {
87+
attempt++;
88+
const dir = yield* fs.makeTempDirectoryScoped({
89+
directory: path.dirname(file),
90+
prefix: ".jsign-",
91+
});
92+
const copy = path.join(dir, path.basename(file));
93+
yield* fs.copyFile(file, copy);
94+
const { exitCode, stderr } = yield* runJsign(jsign, [...args, copy]);
95+
if (exitCode !== 0) {
96+
const transient = TRANSIENT_JSIGN_ERROR.test(stderr);
97+
if (transient && attempt <= JSIGN_RETRIES) {
98+
yield* Effect.logWarning(
99+
`[jsign] Transient Azure Trusted Signing error signing ${file} (attempt ${String(attempt)} of ${String(JSIGN_RETRIES + 1)}); retrying.`,
100+
);
101+
}
102+
return yield* new JsignFailedError({ file, exitCode, transient });
103+
}
104+
yield* fs.rename(copy, file);
105+
}).pipe(Effect.scoped);
106+
107+
yield* signCopy.pipe(
108+
Effect.retry({
109+
schedule: Schedule.exponential("5 seconds"),
110+
times: JSIGN_RETRIES,
111+
while: (error) => error._tag === "JsignFailedError" && error.transient,
112+
}),
50113
);
51-
const exitCode = Number(yield* child.exitCode);
52-
if (exitCode !== 0) {
53-
return yield* new JsignFailedError({ file, exitCode });
54-
}
55114
});
56115

57116
export default function sign(configuration: { readonly path: string }): Promise<void> {

0 commit comments

Comments
 (0)