Skip to content

Commit 67699df

Browse files
perf(azure-devops): PR Code tab loads a page's files in one az run instead of two per file (#35295)
Co-authored-by: Julius Marminge <[email protected]>
1 parent a7fd432 commit 67699df

5 files changed

Lines changed: 623 additions & 25 deletions

File tree

‎packages/source-control-azure-devops/src/server/AzureDevOpsPullRequestCli.test.ts‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,8 @@
1+
// @effect-diagnostics nodeBuiltinImport:off -- Builds the zip Azure returns and stands in for az writing it.
2+
import * as NodeFS from "node:fs";
3+
import * as NodeZlib from "node:zlib";
4+
5+
import * as NodeServices from "@effect/platform-node/NodeServices";
16
import { afterEach, assert, expect, it, vi } from "@effect/vitest";
27
import * as Effect from "effect/Effect";
38
import * as Layer from "effect/Layer";
@@ -16,6 +21,7 @@ const layer = it.layer(
1621
execute: mockedExecute,
1722
}),
1823
),
24+
Layer.provide(NodeServices.layer),
1925
),
2026
);
2127

@@ -81,6 +87,52 @@ function pullRequests(count: number, firstNumber: number): string {
8187
return JSON.stringify(pullRequestRows(count, firstNumber));
8288
}
8389

90+
/** Entries as Azure streams them: data descriptors, with sizes only in the central directory. */
91+
function streamedZip(
92+
entries: ReadonlyArray<{
93+
readonly name: string;
94+
readonly data: Uint8Array;
95+
readonly stored?: true;
96+
}>,
97+
): Buffer {
98+
const parts: Buffer[] = [];
99+
const directory: Buffer[] = [];
100+
let offset = 0;
101+
for (const entry of entries) {
102+
const name = Buffer.from(entry.name);
103+
const method = entry.stored ? 0 : 8;
104+
const body = entry.stored ? Buffer.from(entry.data) : NodeZlib.deflateRawSync(entry.data);
105+
const local = Buffer.alloc(30);
106+
local.writeUInt32LE(0x04034b50, 0);
107+
local.writeUInt16LE(0x0008, 6);
108+
local.writeUInt16LE(method, 8);
109+
local.writeUInt16LE(name.length, 26);
110+
const descriptor = Buffer.alloc(16);
111+
descriptor.writeUInt32LE(0x08074b50, 0);
112+
descriptor.writeUInt32LE(body.length, 8);
113+
descriptor.writeUInt32LE(entry.data.length, 12);
114+
const central = Buffer.alloc(46);
115+
central.writeUInt32LE(0x02014b50, 0);
116+
central.writeUInt16LE(0x0008, 8);
117+
central.writeUInt16LE(method, 10);
118+
central.writeUInt32LE(body.length, 20);
119+
central.writeUInt32LE(entry.data.length, 24);
120+
central.writeUInt16LE(name.length, 28);
121+
central.writeUInt32LE(offset, 42);
122+
parts.push(local, name, body, descriptor);
123+
directory.push(central, name);
124+
offset += local.length + name.length + body.length + descriptor.length;
125+
}
126+
const centralDirectory = Buffer.concat(directory);
127+
const end = Buffer.alloc(22);
128+
end.writeUInt32LE(0x06054b50, 0);
129+
end.writeUInt16LE(entries.length, 8);
130+
end.writeUInt16LE(entries.length, 10);
131+
end.writeUInt32LE(centralDirectory.length, 12);
132+
end.writeUInt32LE(offset, 16);
133+
return Buffer.concat([...parts, centralDirectory, end]);
134+
}
135+
84136
function argsOfCall(index: number): ReadonlyArray<string> {
85137
const call = mockedExecute.mock.calls[index];
86138
assert.isDefined(call);
@@ -850,6 +902,80 @@ layer("AzureDevOpsPullRequestCli.layer", (it) => {
850902
}),
851903
);
852904

905+
it.effect(
906+
"reads a batch of files from one blobs zip, keeping only what Azure's read would say",
907+
() =>
908+
Effect.gen(function* () {
909+
const id = (digit: string) => digit.repeat(40);
910+
const utf8 = (text: string) => new TextEncoder().encode(text);
911+
let sent = "";
912+
mockedExecute.mockImplementationOnce((input) =>
913+
Effect.sync(() => {
914+
const flag = (name: string) => input.args[input.args.indexOf(name) + 1] ?? "";
915+
sent = NodeFS.readFileSync(flag("--in-file"), "utf8");
916+
NodeFS.writeFileSync(
917+
flag("--out-file"),
918+
streamedZip([
919+
{ name: id("a"), data: utf8("plain text\n") },
920+
{ name: id("b"), data: utf8("\uFEFFcafé ✓\n") },
921+
{ name: id("c"), data: Uint8Array.of(0x89, 0x50, 0x00, 0x01) },
922+
// Windows-1252 "café" and UTF-16: Azure decodes these itself.
923+
{ name: id("d"), data: Uint8Array.of(0x63, 0x61, 0x66, 0xe9) },
924+
{ name: id("e"), data: Uint8Array.of(0xff, 0xfe, 0x68, 0x00) },
925+
{ name: id("f"), data: utf8("stored\n"), stored: true },
926+
{ name: id("9"), data: utf8("never asked for\n") },
927+
]),
928+
);
929+
return output("");
930+
}),
931+
);
932+
const cli = yield* AzureDevOpsPullRequestCli.AzureDevOpsPullRequestCli;
933+
934+
const blobs = yield* cli.readBlobs({
935+
cwd: "/w",
936+
location: { project: "platform", repository: "web" },
937+
objectIds: [id("A"), id("b"), id("C"), id("d"), id("e"), id("f"), id("a"), "../etc"],
938+
});
939+
940+
expect(JSON.parse(sent)).toEqual([id("A"), id("b"), id("C"), id("d"), id("e"), id("f")]);
941+
expect(argsOfCall(0)).toEqual(
942+
expect.arrayContaining(["blobs", "POST", "application/zip", "project=platform"]),
943+
);
944+
expect(Object.fromEntries(blobs)).toEqual({
945+
[id("a")]: { contents: "plain text\n", isBinary: false },
946+
[id("b")]: { contents: "café ✓\n", isBinary: false },
947+
[id("c")]: { contents: "", isBinary: true },
948+
[id("f")]: { contents: "stored\n", isBinary: false },
949+
});
950+
}),
951+
);
952+
953+
it.effect("leaves blobs past the batch's total size to per-file reads", () =>
954+
Effect.gen(function* () {
955+
const ids = ["1", "2", "3", "4", "5", "6"].map((digit) => digit.repeat(40));
956+
// Compresses to a few KiB each, so only the inflated total can bound memory.
957+
const blob = new Uint8Array(3 * 1024 * 1024).fill(0x61);
958+
mockedExecute.mockImplementationOnce((input) =>
959+
Effect.sync(() => {
960+
NodeFS.writeFileSync(
961+
input.args[input.args.indexOf("--out-file") + 1] ?? "",
962+
streamedZip(ids.map((name) => ({ name, data: blob }))),
963+
);
964+
return output("");
965+
}),
966+
);
967+
const cli = yield* AzureDevOpsPullRequestCli.AzureDevOpsPullRequestCli;
968+
969+
const blobs = yield* cli.readBlobs({
970+
cwd: "/w",
971+
location: { project: "platform", repository: "web" },
972+
objectIds: ids,
973+
});
974+
975+
expect([...blobs.keys()]).toEqual(ids.slice(0, 5));
976+
}),
977+
);
978+
853979
it.effect("escapes characters in a file path that would cut the query string", () =>
854980
Effect.gen(function* () {
855981
mockedExecute.mockReturnValueOnce(Effect.succeed(output(json({ content: "x" }))));

‎packages/source-control-azure-devops/src/server/AzureDevOpsPullRequestCli.ts‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
import * as Context from "effect/Context";
22
import * as Effect from "effect/Effect";
3+
import * as FileSystem from "effect/FileSystem";
34
import * as Layer from "effect/Layer";
45
import * as Result from "effect/Result";
56
import * as Schema from "effect/Schema";
@@ -12,6 +13,7 @@ import type {
1213
} from "@t3tools/contracts";
1314

1415
import * as AzureDevOpsCli from "./AzureDevOpsCli.ts";
16+
import { blobContent, readBlobZip } from "./azureDevOpsBlobZip.ts";
1517
import {
1618
decodeItemContentJson,
1719
decodeIterationChangesJson,
@@ -108,6 +110,13 @@ const PULL_REQUEST_LIST_MAX_OUTPUT_BYTES = 16 * 1024 * 1024;
108110
const CHANGE_ENTRIES_MAX_OUTPUT_BYTES = 8 * 1024 * 1024;
109111
/** JSON escaping and base64 can exceed the decoded file's size. */
110112
const ITEM_CONTENT_MAX_OUTPUT_BYTES = 4 * 1024 * 1024;
113+
/** A larger batch is left to per-file reads rather than held in memory at once. */
114+
const BLOB_ARCHIVE_MAX_BYTES = 64 * 1024 * 1024;
115+
/** Matches what a per-file read could carry. */
116+
const BLOB_MAX_BYTES = ITEM_CONTENT_MAX_OUTPUT_BYTES;
117+
/** Blobs past this, together, are left to per-file reads, which hold a few at a time. */
118+
const BLOB_BATCH_MAX_BYTES = 16 * 1024 * 1024;
119+
const BLOB_ID = /^[0-9a-f]{40}$/i;
111120
/** Review history is unpaged and independent of diff size. */
112121
const REVIEW_HISTORY_MAX_OUTPUT_BYTES = 8 * 1024 * 1024;
113122

@@ -184,6 +193,19 @@ export class AzureDevOpsPullRequestCli extends Context.Service<
184193
readonly commit: string;
185194
}) => Effect.Effect<AzureDevOpsItemContent, AzureDevOpsPullRequestCliError>;
186195

196+
/**
197+
* Many files in one `az` run, by blob id, keyed by lowercase id. A blob only Azure's item read
198+
* decodes faithfully, or one too large to carry, is left out.
199+
*/
200+
readonly readBlobs: (input: {
201+
readonly cwd: string;
202+
readonly location: AzureDevOpsRepositoryLocation;
203+
readonly objectIds: ReadonlyArray<string>;
204+
}) => Effect.Effect<
205+
ReadonlyMap<string, AzureDevOpsItemContent>,
206+
AzureDevOpsPullRequestCliError
207+
>;
208+
187209
readonly runPullRequestAction: (input: {
188210
readonly cwd: string;
189211
readonly number: number;
@@ -276,6 +298,7 @@ function isReviewerName(value: string): boolean {
276298
/** @public */
277299
export const make = Effect.gen(function* () {
278300
const azure = yield* AzureDevOpsCli.AzureDevOpsCli;
301+
const fileSystem = yield* FileSystem.FileSystem;
279302

280303
// Let `az` parse all three remote formats.
281304
const detectArgs = ["--detect", "true"] as const;
@@ -596,6 +619,78 @@ export const make = Effect.gen(function* () {
596619
},
597620
),
598621

622+
readBlobs: Effect.fn("AzureDevOpsPullRequestCli.readBlobs")(function* (input) {
623+
const requested = new Map<string, string>();
624+
// Azure states ids in uppercase and names entries in lowercase.
625+
for (const id of input.objectIds) {
626+
const key = id.toLowerCase();
627+
if (BLOB_ID.test(id) && !requested.has(key)) requested.set(key, id);
628+
}
629+
if (requested.size === 0) return new Map<string, AzureDevOpsItemContent>();
630+
const readFailure = (cause: unknown) =>
631+
new AzureDevOpsPullRequestReadError({
632+
command: "az",
633+
cwd: input.cwd,
634+
operation: "readBlobs",
635+
cause,
636+
});
637+
return yield* Effect.scoped(
638+
Effect.gen(function* () {
639+
const directory = yield* fileSystem
640+
.makeTempDirectoryScoped({ prefix: "t3-azure-blobs-" })
641+
.pipe(Effect.mapError(readFailure));
642+
const request = `${directory}/ids.json`;
643+
// invoke refuses to overwrite its output, so the archive gets a name nothing holds yet.
644+
const archive = `${directory}/blobs.zip`;
645+
yield* fileSystem
646+
.writeFileString(
647+
request,
648+
`[${[...requested.values()].map((id) => `"${id}"`).join(",")}]`,
649+
)
650+
.pipe(Effect.mapError(readFailure));
651+
yield* azure.execute({
652+
cwd: input.cwd,
653+
args: [
654+
"devops",
655+
"invoke",
656+
...detectArgs,
657+
"--area",
658+
"git",
659+
"--resource",
660+
"blobs",
661+
"--api-version",
662+
REST_API_VERSION,
663+
"--route-parameters",
664+
...repositoryRoute(input.location),
665+
"--http-method",
666+
"POST",
667+
"--in-file",
668+
request,
669+
"--accept-media-type",
670+
"application/zip",
671+
"--out-file",
672+
archive,
673+
"--only-show-errors",
674+
],
675+
});
676+
const size = yield* fileSystem.stat(archive).pipe(
677+
Effect.map((info) => Number(info.size)),
678+
Effect.mapError(readFailure),
679+
);
680+
const contents = new Map<string, AzureDevOpsItemContent>();
681+
if (size > BLOB_ARCHIVE_MAX_BYTES) return contents;
682+
const zip = yield* fileSystem.readFile(archive).pipe(Effect.mapError(readFailure));
683+
const entries = readBlobZip(zip, BLOB_MAX_BYTES, BLOB_BATCH_MAX_BYTES);
684+
if (entries === null) return yield* readFailure("Azure returned an unreadable archive.");
685+
for (const [id, bytes] of entries) {
686+
const content = requested.has(id) ? blobContent(bytes) : null;
687+
if (content !== null) contents.set(id, content);
688+
}
689+
return contents;
690+
}),
691+
);
692+
}),
693+
599694
readItemContent: Effect.fn("AzureDevOpsPullRequestCli.readItemContent")(function (input) {
600695
return invoke({
601696
cwd: input.cwd,

0 commit comments

Comments
 (0)