Skip to content

Commit 0f8cb9a

Browse files
committed
fix(server): preserve checkpoint pathname bytes
1 parent 585a2f8 commit 0f8cb9a

7 files changed

Lines changed: 142 additions & 24 deletions

File tree

‎apps/server/src/processRunner.test.ts‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -246,6 +246,24 @@ describe("runProcess", () => {
246246
}),
247247
);
248248

249+
it.effect("preserves raw stdout bytes when requested", () =>
250+
Effect.gen(function* () {
251+
const stdoutBytes = new Uint8Array([0x66, 0x80, 0x6f]);
252+
const spawner = makeSpawner(() =>
253+
Effect.succeed(makeHandle({ stdout: Stream.make(stdoutBytes) })),
254+
);
255+
256+
const result = yield* runWith(spawner)({
257+
command: "fake",
258+
args: ["raw-stdout"],
259+
captureStdoutBytes: true,
260+
});
261+
262+
expect(result.stdoutInvalidUtf8).toBe(true);
263+
expect(Array.from(result.stdoutBytes ?? [])).toEqual(Array.from(stdoutBytes));
264+
}),
265+
);
266+
249267
it.effect("fails fast on output limit before timeout for long-running output", () =>
250268
Effect.gen(function* () {
251269
const textChunk = "x".repeat(64);
@@ -319,6 +337,32 @@ describe("runProcess", () => {
319337
}),
320338
);
321339

340+
it.effect("writes raw stdin bytes without UTF-8 encoding", () =>
341+
Effect.gen(function* () {
342+
const stdinBytes = new Uint8Array([0x66, 0x80, 0x6f]);
343+
const receivedBytes: number[] = [];
344+
const spawner = makeSpawner(() =>
345+
Effect.succeed(
346+
makeHandle({
347+
stdin: Sink.forEach((chunk: Uint8Array) =>
348+
Effect.sync(() => {
349+
receivedBytes.push(...chunk);
350+
}),
351+
),
352+
}),
353+
),
354+
);
355+
356+
yield* runWith(spawner)({
357+
command: "fake",
358+
args: ["raw-stdin"],
359+
stdin: stdinBytes,
360+
});
361+
362+
expect(receivedBytes).toEqual(Array.from(stdinBytes));
363+
}),
364+
);
365+
322366
it.effect("returns output for non-zero exit codes", () =>
323367
Effect.gen(function* () {
324368
const spawner = makeSpawner(() => Effect.succeed(makeHandle({ stderr: "boom", code: 2 })));

‎apps/server/src/processRunner.ts‎

Lines changed: 20 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,8 @@ export interface ProcessRunInput {
2424
readonly spawnCwd?: string | undefined;
2525
readonly timeout?: Duration.Input | undefined;
2626
readonly env?: NodeJS.ProcessEnv | undefined;
27-
readonly stdin?: string | undefined;
27+
readonly stdin?: string | Uint8Array | undefined;
28+
readonly captureStdoutBytes?: boolean | undefined;
2829
readonly maxOutputBytes?: number | undefined;
2930
readonly outputMode?: "error" | "truncate" | undefined;
3031
readonly truncatedMarker?: string | undefined;
@@ -44,6 +45,7 @@ export interface ProcessRunOutput {
4445
readonly stderrTruncated: boolean;
4546
readonly stdoutInvalidUtf8: boolean;
4647
readonly stderrInvalidUtf8: boolean;
48+
readonly stdoutBytes?: Uint8Array;
4749
}
4850

4951
const ProcessInvocationFields = {
@@ -182,6 +184,7 @@ const collectText = Effect.fn("processRunner.collectText")(function* (input: {
182184
readonly maxOutputBytes: number;
183185
readonly outputMode: "error" | "truncate";
184186
readonly truncatedMarker: string;
187+
readonly preserveBytes: boolean;
185188
}) {
186189
const stream = input.stream.pipe(
187190
Stream.mapError(
@@ -202,6 +205,7 @@ const collectText = Effect.fn("processRunner.collectText")(function* (input: {
202205
stream,
203206
maxBytes: input.maxOutputBytes,
204207
truncatedMarker: input.truncatedMarker,
208+
preserveBytes: input.preserveBytes,
205209
});
206210
}
207211

@@ -239,13 +243,15 @@ const collectText = Effect.fn("processRunner.collectText")(function* (input: {
239243
});
240244
},
241245
),
242-
Effect.map(
243-
(state): CollectedUint8StreamText => ({
244-
...decodeUtf8(Buffer.concat(state.chunks, state.bytes)),
246+
Effect.map((state): CollectedUint8StreamText => {
247+
const rawBytes = Buffer.concat(state.chunks, state.bytes);
248+
return {
249+
...decodeUtf8(rawBytes),
245250
bytes: state.bytes,
246251
truncated: false,
247-
}),
248-
),
252+
...(input.preserveBytes ? { rawBytes } : {}),
253+
};
254+
}),
249255
);
250256
});
251257

@@ -335,15 +341,18 @@ const runProcessCore = Effect.fn("processRunner.runProcessCore")(function* (
335341
const writeStdin =
336342
stdin === undefined
337343
? Effect.void
338-
: Stream.run(Stream.encodeText(Stream.make(stdin)), child.stdin).pipe(
344+
: Stream.run(
345+
typeof stdin === "string" ? Stream.encodeText(Stream.make(stdin)) : Stream.make(stdin),
346+
child.stdin,
347+
).pipe(
339348
Effect.mapError(
340349
(cause) =>
341350
new ProcessStdinError({
342351
command: input.command,
343352
argumentCount: input.args.length,
344353
cwd: input.cwd,
345354
spawnCwd: input.spawnCwd,
346-
stdinBytes: Buffer.byteLength(stdin),
355+
stdinBytes: typeof stdin === "string" ? Buffer.byteLength(stdin) : stdin.byteLength,
347356
cause,
348357
}),
349358
),
@@ -361,6 +370,7 @@ const runProcessCore = Effect.fn("processRunner.runProcessCore")(function* (
361370
maxOutputBytes,
362371
outputMode,
363372
truncatedMarker,
373+
preserveBytes: input.captureStdoutBytes === true,
364374
}),
365375
collectText({
366376
command: input.command,
@@ -372,6 +382,7 @@ const runProcessCore = Effect.fn("processRunner.runProcessCore")(function* (
372382
maxOutputBytes,
373383
outputMode,
374384
truncatedMarker,
385+
preserveBytes: false,
375386
}),
376387
writeStdin,
377388
],
@@ -401,6 +412,7 @@ const runProcessCore = Effect.fn("processRunner.runProcessCore")(function* (
401412
stderrTruncated: stderr.truncated,
402413
stdoutInvalidUtf8: stdout.invalidUtf8,
403414
stderrInvalidUtf8: stderr.invalidUtf8,
415+
...(stdout.rawBytes !== undefined ? { stdoutBytes: stdout.rawBytes } : {}),
404416
} satisfies ProcessRunOutput;
405417
});
406418

‎apps/server/src/stream/collectUint8StreamText.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ export interface CollectedUint8StreamText {
77
readonly truncated: boolean;
88
readonly bytes: number;
99
readonly invalidUtf8: boolean;
10+
readonly rawBytes?: Uint8Array;
1011
}
1112

1213
export const decodeUtf8 = (
@@ -26,6 +27,7 @@ export const collectUint8StreamText = <E>(input: {
2627
readonly stream: Stream.Stream<Uint8Array, E>;
2728
readonly maxBytes?: number | undefined;
2829
readonly truncatedMarker?: string | null | undefined;
30+
readonly preserveBytes?: boolean | undefined;
2931
}): Effect.Effect<CollectedUint8StreamText, E> => {
3032
const maxBytes = input.maxBytes ?? Number.POSITIVE_INFINITY;
3133
const truncatedMarker = input.truncatedMarker ?? "";
@@ -68,7 +70,8 @@ export const collectUint8StreamText = <E>(input: {
6870
},
6971
),
7072
Effect.map((state): CollectedUint8StreamText => {
71-
const decoded = decodeUtf8(Buffer.concat(state.chunks, state.bytes));
73+
const rawBytes = Buffer.concat(state.chunks, state.bytes);
74+
const decoded = decodeUtf8(rawBytes);
7275
return {
7376
text:
7477
state.truncated && truncatedMarker.length > 0
@@ -77,6 +80,7 @@ export const collectUint8StreamText = <E>(input: {
7780
bytes: state.bytes,
7881
truncated: state.truncated,
7982
invalidUtf8: decoded.invalidUtf8,
83+
...(input.preserveBytes ? { rawBytes } : {}),
8084
};
8185
}),
8286
);

‎apps/server/src/vcs/GitVcsDriver.test.ts‎

Lines changed: 18 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,19 @@ it.effect("GitVcsDriver forwards execute env to the VCS process", () => {
112112
it.effect("GitVcsDriver stages only changed paths when capturing a checkpoint", () => {
113113
const commands: Array<VcsProcess.VcsProcessInput> = [];
114114
const changedPaths = ["changed.ts", "deleted.ts", "dir/new\nfile.ts", ":literal.ts"];
115+
const invalidUtf8Path = new Uint8Array([0xff, ...new TextEncoder().encode(".txt")]);
116+
const statusBytes = Buffer.concat([
117+
Buffer.from(
118+
` M ${changedPaths[0]}\0 D ${changedPaths[1]}\0?? ${changedPaths[2]}\0?? ${changedPaths[3]}\0?? `,
119+
),
120+
invalidUtf8Path,
121+
Buffer.from([0]),
122+
]);
123+
const expectedPathspec = Buffer.concat([
124+
Buffer.from(`${changedPaths.join("\0")}\0`),
125+
invalidUtf8Path,
126+
Buffer.from([0]),
127+
]);
115128

116129
return Effect.gen(function* () {
117130
const driver = yield* GitVcsDriver.makeVcsDriverShape();
@@ -134,6 +147,7 @@ it.effect("GitVcsDriver stages only changed paths when capturing a checkpoint",
134147
".",
135148
]);
136149
assert.strictEqual(statusCommand.env?.GIT_OPTIONAL_LOCKS, "0");
150+
assert.strictEqual(statusCommand.captureStdoutBytes, true);
137151

138152
const addCommand = commands.find((command) => command.args[3] === "add");
139153
assert.isDefined(addCommand);
@@ -144,7 +158,8 @@ it.effect("GitVcsDriver stages only changed paths when capturing a checkpoint",
144158
"--pathspec-from-file=-",
145159
"--pathspec-file-nul",
146160
]);
147-
assert.strictEqual(addCommand.stdin, `${changedPaths.join("\0")}\0`);
161+
assert.instanceOf(addCommand.stdin, Uint8Array);
162+
assert.deepStrictEqual(addCommand.stdin, expectedPathspec);
148163
}).pipe(
149164
Effect.provide(
150165
Layer.mergeAll(
@@ -156,7 +171,7 @@ it.effect("GitVcsDriver stages only changed paths when capturing a checkpoint",
156171
const args = input.args.slice(2);
157172
const stdout =
158173
args[0] === "status"
159-
? ` M ${changedPaths[0]}\0 D ${changedPaths[1]}\0?? ${changedPaths[2]}\0?? ${changedPaths[3]}\0`
174+
? new TextDecoder().decode(statusBytes)
160175
: args[0] === "rev-parse" && args[1] === "--git-common-dir"
161176
? ".git\n"
162177
: args[0] === "rev-parse" && args[1] === "--show-toplevel"
@@ -172,6 +187,7 @@ it.effect("GitVcsDriver stages only changed paths when capturing a checkpoint",
172187
stderr: "",
173188
stdoutTruncated: false,
174189
stderrTruncated: false,
190+
...(args[0] === "status" ? { stdoutBytes: statusBytes } : {}),
175191
};
176192
}),
177193
}),

‎apps/server/src/vcs/GitVcsDriver.ts‎

Lines changed: 31 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -358,10 +358,19 @@ function splitNullSeparatedPaths(input: string, truncated: boolean): string[] {
358358
return parts.filter((value) => value.length > 0);
359359
}
360360

361-
function parseCheckpointStatusPaths(output: string): string[] {
362-
return splitNullSeparatedPaths(output, false).flatMap((entry) =>
363-
entry.length > 3 && entry[2] === " " ? [entry.slice(3)] : [],
364-
);
361+
function parseCheckpointStatusPathspec(output: Uint8Array): Uint8Array {
362+
const paths: Uint8Array[] = [];
363+
let recordStart = 0;
364+
for (let index = 0; index < output.byteLength; index += 1) {
365+
if (output[index] !== 0) {
366+
continue;
367+
}
368+
if (index - recordStart > 3 && output[recordStart + 2] === 0x20) {
369+
paths.push(output.subarray(recordStart + 3, index + 1));
370+
}
371+
recordStart = index + 1;
372+
}
373+
return Buffer.concat(paths);
365374
}
366375

367376
function chunkPathsForGitCheckIgnore(relativePaths: ReadonlyArray<string>): string[][] {
@@ -432,7 +441,8 @@ const gitCommand = (
432441
cwd: string,
433442
args: ReadonlyArray<string>,
434443
options?: {
435-
readonly stdin?: string;
444+
readonly stdin?: string | Uint8Array;
445+
readonly captureStdoutBytes?: boolean;
436446
readonly env?: NodeJS.ProcessEnv;
437447
readonly allowNonZeroExit?: boolean;
438448
readonly timeoutMs?: number;
@@ -447,6 +457,9 @@ const gitCommand = (
447457
cwd,
448458
spawnCwd: globalThis.process.cwd(),
449459
...(options?.stdin !== undefined ? { stdin: options.stdin } : {}),
460+
...(options?.captureStdoutBytes !== undefined
461+
? { captureStdoutBytes: options.captureStdoutBytes }
462+
: {}),
450463
...(options?.env !== undefined ? { env: options.env } : {}),
451464
...(options?.allowNonZeroExit !== undefined
452465
? { allowNonZeroExit: options.allowNonZeroExit }
@@ -487,6 +500,9 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* (
487500
const execute: VcsDriver.VcsDriver["Service"]["execute"] = (input) =>
488501
gitCommand(vcsProcess, input.operation, input.cwd, input.args, {
489502
...(input.stdin !== undefined ? { stdin: input.stdin } : {}),
503+
...(input.captureStdoutBytes !== undefined
504+
? { captureStdoutBytes: input.captureStdoutBytes }
505+
: {}),
490506
...(input.env !== undefined ? { env: input.env } : {}),
491507
...(input.allowNonZeroExit !== undefined ? { allowNonZeroExit: input.allowNonZeroExit } : {}),
492508
...(input.timeoutMs !== undefined ? { timeoutMs: input.timeoutMs } : {}),
@@ -714,12 +730,14 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* (
714730
return path.isAbsolute(gitCommonDir) ? gitCommonDir : path.resolve(cwd, gitCommonDir);
715731
});
716732

717-
const resolveRepositoryRoot = (cwd: string) =>
718-
execute({
733+
const resolveRepositoryRoot = Effect.fn("resolveRepositoryRoot")(function* (cwd: string) {
734+
const result = yield* execute({
719735
operation: "GitVcsDriver.checkpoints.resolveRepositoryRoot",
720736
cwd,
721737
args: ["rev-parse", "--show-toplevel"],
722-
}).pipe(Effect.map((result) => result.stdout.trim()));
738+
});
739+
return result.stdout.trim();
740+
});
723741

724742
const checkpoints: VcsDriver.VcsCheckpointOps = {
725743
captureCheckpoint: Effect.fn("GitVcsDriver.checkpoints.captureCheckpoint")(function* (input) {
@@ -769,18 +787,19 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* (
769787
GIT_OPTIONAL_LOCKS: "0",
770788
},
771789
maxOutputBytes: WORKSPACE_FILES_MAX_OUTPUT_BYTES,
790+
captureStdoutBytes: true,
772791
});
773792

774-
if (statusResult.stdoutTruncated) {
793+
if (statusResult.stdoutTruncated || statusResult.stdoutBytes === undefined) {
775794
yield* execute({
776795
operation,
777796
cwd: input.cwd,
778797
args: ["add", "-A", "--", "."],
779798
env: commitEnv,
780799
});
781800
} else {
782-
const changedPaths = parseCheckpointStatusPaths(statusResult.stdout);
783-
if (changedPaths.length > 0) {
801+
const changedPathspec = parseCheckpointStatusPathspec(statusResult.stdoutBytes);
802+
if (changedPathspec.byteLength > 0) {
784803
yield* execute({
785804
operation,
786805
cwd: repositoryRoot,
@@ -791,7 +810,7 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* (
791810
"--pathspec-from-file=-",
792811
"--pathspec-file-nul",
793812
],
794-
stdin: `${changedPaths.join("\0")}\0`,
813+
stdin: changedPathspec,
795814
env: commitEnv,
796815
});
797816
}

‎apps/server/src/vcs/VcsProcess.test.ts‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,23 @@ describe("VcsProcess.run", () => {
6161
}).pipe(provideLive),
6262
);
6363

64+
it.effect("round-trips raw process bytes when requested", () =>
65+
Effect.gen(function* () {
66+
const rawBytes = new Uint8Array([0x66, 0x80, 0x6f]);
67+
const result = yield* run({
68+
operation: "test.raw-stdout",
69+
command: "node",
70+
args: ["-e", "process.stdin.on('data', chunk => process.stdout.write(chunk))"],
71+
cwd: process.cwd(),
72+
stdin: rawBytes,
73+
captureStdoutBytes: true,
74+
});
75+
76+
expect(result.stdoutInvalidUtf8).toBe(true);
77+
expect(Array.from(result.stdoutBytes ?? [])).toEqual(Array.from(rawBytes));
78+
}).pipe(provideLive),
79+
);
80+
6481
it.effect("writes stdin before waiting for exit", () =>
6582
Effect.gen(function* () {
6683
const result = yield* run({

0 commit comments

Comments
 (0)