Skip to content

Commit ab00399

Browse files
maria-rcksmaria-rcks
committed
chore: merge current main into office file previews
# Conflicts: # third-party-licenses.config.json
2 parents 00bddf6 + e856833 commit ab00399

42 files changed

Lines changed: 2630 additions & 187 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎apps/server/package.json‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,11 +56,13 @@
5656
"@t3tools/source-control-gitcafe": "workspace:*",
5757
"@t3tools/source-control-github": "workspace:*",
5858
"@t3tools/source-control-gitlab": "workspace:*",
59+
"character-entities": "2.0.2",
5960
"effect": "catalog:",
6061
"jose": "catalog:",
6162
"node-pty": "^1.2.0-beta.15",
6263
"playwright-core": "1.60.0",
6364
"proper-lockfile": "4.1.2",
65+
"saxes": "6.0.0",
6466
"stream-chain": "^4.2.5",
6567
"stream-json": "3.6.0",
6668
"undici": "catalog:",

‎apps/server/src/assets/AttachmentUpload.test.ts‎

Lines changed: 81 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import * as DocumentText from "../provider/DocumentText.ts";
12
// @effect-diagnostics nodeBuiltinImport:off
23
import * as NodeFS from "node:fs";
34
import * as NodePath from "node:path";
@@ -8,7 +9,9 @@ import { ChatAttachmentId } from "@t3tools/contracts";
89
import * as Deferred from "effect/Deferred";
910
import * as Effect from "effect/Effect";
1011
import * as Fiber from "effect/Fiber";
12+
import * as FileSystem from "effect/FileSystem";
1113
import * as Layer from "effect/Layer";
14+
import * as PlatformError from "effect/PlatformError";
1215
import * as Schema from "effect/Schema";
1316
import * as Stream from "effect/Stream";
1417

@@ -29,7 +32,7 @@ import {
2932

3033
const layerTest = ServerSecretStore.layer.pipe(
3134
Layer.provideMerge(ServerConfig.layerTest(process.cwd(), { prefix: "t3-attachment-upload-" })),
32-
Layer.provideMerge(NodeServices.layer),
35+
Layer.provideMerge(DocumentText.layer.pipe(Layer.provideMerge(NodeServices.layer))),
3336
);
3437

3538
const uploadInput = {
@@ -52,6 +55,83 @@ const encodeLegacyAttachmentUploadClaims = Schema.encodeEffect(
5255
);
5356

5457
describe("AttachmentUpload", () => {
58+
it.effect.each([false, true])(
59+
"persists and replays immutable uploads (hard links unsupported: %s)",
60+
(unsupported) =>
61+
Effect.gen(function* () {
62+
const config = yield* ServerConfig.ServerConfig;
63+
const fileSystem = yield* FileSystem.FileSystem;
64+
const store = (claims: Parameters<typeof storeAttachmentUpload>[0], bytes: Uint8Array) =>
65+
storeAttachmentUpload(claims, bytes).pipe(
66+
Effect.provideService(
67+
FileSystem.FileSystem,
68+
unsupported
69+
? {
70+
...fileSystem,
71+
link: () =>
72+
Effect.fail(
73+
PlatformError.systemError({
74+
_tag: "Unknown",
75+
module: "FileSystem",
76+
method: "link",
77+
}),
78+
),
79+
}
80+
: fileSystem,
81+
),
82+
);
83+
const bytes = new TextEncoder().encode(
84+
JSON.stringify({
85+
cells: [{ cell_type: "markdown", source: ["# Cargo\n", "Violet crates: 19"] }],
86+
}),
87+
);
88+
const issued = yield* issueAttachmentUploadUrl({
89+
type: "file",
90+
name: "cargo.ipynb",
91+
mimeType: "application/x-ipynb+json",
92+
sizeBytes: bytes.length,
93+
});
94+
const claims = yield* validateAttachmentUploadToken(
95+
issued.relativeUrl.slice(`${ATTACHMENT_UPLOAD_ROUTE_PREFIX}/`.length),
96+
);
97+
if (!claims) throw new Error("Expected upload claims");
98+
expect(yield* store(claims, bytes)).toEqual({ ok: true });
99+
expect(yield* store(claims, bytes)).toEqual({ ok: true });
100+
const path = NodePath.join(config.attachmentsDir, `${issued.attachmentId}.ipynb`);
101+
expect(NodeFS.readFileSync(`${path}.txt`, "utf8")).toContain("Violet crates: 19");
102+
const replacement = new TextEncoder().encode(
103+
new TextDecoder().decode(bytes).replace("Violet crates: 19", "Violet crates: 28"),
104+
);
105+
expect(yield* store(claims, replacement)).toMatchObject({
106+
ok: false,
107+
status: 409,
108+
});
109+
expect(NodeFS.readFileSync(path)).toEqual(Buffer.from(bytes));
110+
expect(NodeFS.readFileSync(`${path}.txt`, "utf8")).toContain("Violet crates: 19");
111+
const claimed = yield* claimPendingAttachments({
112+
threadId: "document-thread",
113+
attachments: [
114+
{
115+
type: "file",
116+
id: ChatAttachmentId.make(issued.attachmentId),
117+
name: claims.name,
118+
mimeType: claims.mimeType,
119+
sizeBytes: bytes.length,
120+
},
121+
],
122+
});
123+
expect(claimed.claimedPaths).toHaveLength(2);
124+
expect(NodeFS.readFileSync(claimed.claimedPaths[1]!, "utf8")).toContain(
125+
"Violet crates: 19",
126+
);
127+
yield* deletePendingAttachment(issued.attachmentId);
128+
expect(NodeFS.existsSync(path)).toBe(false);
129+
expect(NodeFS.existsSync(`${path}.txt`)).toBe(false);
130+
expect(NodeFS.existsSync(claimed.claimedPaths[1]!)).toBe(true);
131+
yield* releaseClaimedAttachments(claimed.claimedPaths);
132+
expect(NodeFS.existsSync(claimed.claimedPaths[1]!)).toBe(false);
133+
}).pipe(Effect.provide(layerTest)),
134+
);
55135
it.effect("rejects tampered and malformed upload tokens", () =>
56136
Effect.gen(function* () {
57137
const issued = yield* issueAttachmentUploadUrl(uploadInput);

‎apps/server/src/assets/AttachmentUpload.ts‎

Lines changed: 51 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
// @effect-diagnostics nodeBuiltinImport:off -- exclusive copy and streaming hashes are filesystem boundaries missing from Effect's API.
2+
import * as NodeCrypto from "node:crypto";
3+
import * as NodeFSP from "node:fs/promises";
14
import {
25
ATTACHMENT_UPLOAD_URL_TTL_MS,
36
type AttachmentCreateUploadUrlInput,
@@ -10,6 +13,7 @@ import * as Effect from "effect/Effect";
1013
import * as FileSystem from "effect/FileSystem";
1114
import * as Option from "effect/Option";
1215
import * as Path from "effect/Path";
16+
import * as PlatformError from "effect/PlatformError";
1317
import * as Schema from "effect/Schema";
1418
import * as Stream from "effect/Stream";
1519
import type * as HttpServerRequest from "effect/http/HttpServerRequest";
@@ -32,6 +36,7 @@ import {
3236
import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
3337
import * as ServerConfig from "../config.ts";
3438
import { inferImageExtension } from "../imageMime.ts";
39+
import * as DocumentText from "../provider/DocumentText.ts";
3540

3641
export const ATTACHMENT_UPLOAD_ROUTE_PREFIX = "/api/attachments/upload";
3742

@@ -205,7 +210,51 @@ export const storeAttachmentUpload = Effect.fn("AttachmentUpload.store")(functio
205210
detail: `Body was ${receivedBytes} bytes, expected ${claims.sizeBytes}.`,
206211
} satisfies StoreAttachmentUploadResult;
207212
}
208-
yield* fileSystem.rename(partPath, finalPath);
213+
// Publishing without replacement keeps an upload and its cached text immutable.
214+
const published = yield* fileSystem.link(partPath, finalPath).pipe(
215+
Effect.map(() => true),
216+
Effect.catch((error) => {
217+
if (error.reason._tag === "AlreadyExists") return Effect.succeed(false);
218+
return Effect.tryPromise({
219+
try: () => NodeFSP.copyFile(partPath, finalPath, NodeFSP.constants.COPYFILE_EXCL),
220+
catch: (cause) =>
221+
PlatformError.systemError({
222+
_tag:
223+
cause instanceof Error && "code" in cause && cause.code === "EEXIST"
224+
? "AlreadyExists"
225+
: "Unknown",
226+
module: "FileSystem",
227+
method: "copyFile",
228+
pathOrDescriptor: finalPath,
229+
cause,
230+
}),
231+
}).pipe(
232+
Effect.as(true),
233+
Effect.catch((error) =>
234+
error.reason._tag === "AlreadyExists" ? Effect.succeed(false) : Effect.fail(error),
235+
),
236+
);
237+
}),
238+
);
239+
if (!published) {
240+
if (Number((yield* fileSystem.stat(finalPath)).size) !== receivedBytes) {
241+
return { ok: false, status: 409, detail: "Attachment already uploaded." };
242+
}
243+
const hashes = yield* Effect.forEach([partPath, finalPath], (file) =>
244+
fileSystem.stream(file).pipe(
245+
Stream.runFold(
246+
() => NodeCrypto.createHash("sha256"),
247+
(hash, chunk) => hash.update(chunk),
248+
),
249+
Effect.map((hash) => hash.digest("hex")),
250+
),
251+
);
252+
if (hashes[0] !== hashes[1]) {
253+
return { ok: false, status: 409, detail: "Attachment already uploaded." };
254+
}
255+
}
256+
const documents = yield* DocumentText.DocumentText;
257+
yield* documents.ensure(finalPath);
209258
return { ok: true } satisfies StoreAttachmentUploadResult;
210259
}).pipe(
211260
Effect.catch((cause) =>
@@ -244,4 +293,5 @@ export const deletePendingAttachment = Effect.fn("AttachmentUpload.deletePending
244293

245294
const fileSystem = yield* FileSystem.FileSystem;
246295
yield* fileSystem.remove(attachmentPath, { force: true }).pipe(Effect.orElseSucceed(() => {}));
296+
yield* fileSystem.remove(`${attachmentPath}.txt`, { force: true }).pipe(Effect.ignore);
247297
});

‎apps/server/src/attachmentStore.test.ts‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -133,16 +133,18 @@ describe("attachmentStore", () => {
133133
const uuid = "00000000-0000-4000-8000-000000000002";
134134
const pendingPath = NodePath.join(attachmentsDir, `pending-${uuid}.png`);
135135
const pendingFilePath = NodePath.join(attachmentsDir, `pending-${uuid}-pdf.pdf`);
136+
const sidecarPath = `${pendingFilePath}.txt`;
136137
const threadPath = NodePath.join(attachmentsDir, `thread-1-${uuid}.png`);
137138
const partialPath = NodePath.join(attachmentsDir, `${uuid}.part`);
138-
for (const filePath of [pendingPath, pendingFilePath, threadPath, partialPath]) {
139+
for (const filePath of [pendingPath, pendingFilePath, sidecarPath, threadPath, partialPath]) {
139140
NodeFS.writeFileSync(filePath, Buffer.from("pixels"));
140141
NodeFS.utimesSync(filePath, oldTimeSeconds, oldTimeSeconds);
141142
}
142143

143-
expect(sweepStalePendingAttachments({ attachmentsDir, nowMs: now })).toEqual({ deleted: 3 });
144+
expect(sweepStalePendingAttachments({ attachmentsDir, nowMs: now })).toEqual({ deleted: 4 });
144145
expect(NodeFS.existsSync(pendingPath)).toBe(false);
145146
expect(NodeFS.existsSync(pendingFilePath)).toBe(false);
147+
expect(NodeFS.existsSync(sidecarPath)).toBe(false);
146148
expect(NodeFS.existsSync(partialPath)).toBe(false);
147149
expect(NodeFS.existsSync(threadPath)).toBe(true);
148150
} finally {

‎apps/server/src/attachmentStore.ts‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -269,7 +269,9 @@ export function sweepStalePendingAttachments(input: {
269269
for (const entry of entries) {
270270
const isPartial = entry.endsWith(".part");
271271
if (!isPartial) {
272-
const attachmentId = parseAttachmentIdFromRelativePath(entry);
272+
const attachmentId = parseAttachmentIdFromRelativePath(
273+
entry.endsWith(".txt") && entry.slice(0, -4).includes(".") ? entry.slice(0, -4) : entry,
274+
);
273275
if (
274276
!attachmentId ||
275277
parseThreadSegmentFromAttachmentId(attachmentId) !== PENDING_ATTACHMENT_THREAD_SEGMENT

‎apps/server/src/bin.ts‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,18 @@ if (
1717
// Guard sockets before either dispatch path can load outbound-capable modules.
1818
const { ignoreMacosTypeOfServiceEinval } = await import("./outboundHttp.ts");
1919
ignoreMacosTypeOfServiceEinval();
20-
if (command === "__managed-chatgpt-token") {
20+
if (command === "__document-text") {
21+
const { extractDocumentText } = await import("./provider/documentTextWorker.ts");
22+
try {
23+
process.stdout.write(await extractDocumentText(process.argv[3]!, process.argv[4]!));
24+
} catch (error) {
25+
if (error instanceof Error && "code" in error && /^E[A-Z0-9]+$/.test(String(error.code))) {
26+
throw error;
27+
}
28+
// Exit 2 identifies document rejections; OS and process failures remain retryable.
29+
process.exitCode = 2;
30+
}
31+
} else if (command === "__managed-chatgpt-token") {
2132
const { runManagedChatGptTokenCli } = await import("./cli/managedChatGptToken.ts");
2233
await runManagedChatGptTokenCli(process.argv.slice(3));
2334
} else if (command === "acp-mcp-bridge" || command === "acp-mcp-call") {

‎apps/server/src/mcp/toolkits/attachment/tools.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import * as Schema from "effect/Schema";
1515
import { Tool, Toolkit } from "effect/ai";
1616
import * as ServerSecretStore from "../../../auth/ServerSecretStore.ts";
1717
import * as ServerConfig from "../../../config.ts";
18+
import * as DocumentText from "../../../provider/DocumentText.ts";
1819
import * as ThreadManagementService from "../../../orchestration-v2/ThreadManagementService.ts";
1920
import * as McpInvocationContext from "../../McpInvocationContext.ts";
2021

@@ -45,6 +46,7 @@ const AttachmentDiscardTool = Tool.make("t3_attachment_discard", {
4546
}).annotate(Tool.Destructive, true);
4647
const AttachmentSendTool = Tool.make("t3_thread_send_attachments", {
4748
...shared,
49+
dependencies: [...shared.dependencies, DocumentText.DocumentText],
4850
description:
4951
"Send uploaded attachments as a new message to a thread. No retry key. Target modes cannot exceed yours. Success does not mean the provider read them or finished.",
5052
parameters: Schema.Struct({

‎apps/server/src/mcp/toolkits/core.test.ts‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import * as DocumentText from "../../provider/DocumentText.ts";
12
import * as NodeCrypto from "@effect/platform-node/NodeCrypto";
23
import * as NodeServices from "@effect/platform-node/NodeServices";
34
import { expect, it } from "@effect/vitest";
@@ -268,7 +269,7 @@ it.effect("returns an HTML render reference that Codex and Claude tool rows both
268269
Layer.provide(McpProviderSessions.layer),
269270
Layer.provide(PreviewBrowser.layer),
270271
Layer.provide(ServerConfig.layerTest(process.cwd(), { prefix: "t3-mcp-html-render-" })),
271-
Layer.provide(NodeServices.layer),
272+
Layer.provide(DocumentText.layer.pipe(Layer.provideMerge(NodeServices.layer))),
272273
Layer.provide(FetchHttpClient.layer),
273274
Layer.provide(
274275
Layer.mock(ThreadManagement.ThreadManagementService)({
@@ -882,7 +883,7 @@ it.effect("only the caller that prepared a pending upload can discard it", () =>
882883
Layer.provide(McpToolAccessTestkit.liveThreadsLayer),
883884
Layer.provide(ServerSecretStore.layer),
884885
Layer.provide(ServerConfig.layerTest(process.cwd(), { prefix: "t3-mcp-attachment-" })),
885-
Layer.provide(NodeServices.layer),
886+
Layer.provide(DocumentText.layer.pipe(Layer.provideMerge(NodeServices.layer))),
886887
),
887888
),
888889
),

‎apps/server/src/mcp/toolkits/project/handlers.test.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ import * as Layer from "effect/Layer";
1616
import * as Option from "effect/Option";
1717
import * as Stream from "effect/Stream";
1818

19+
import * as DocumentText from "../../../provider/DocumentText.ts";
1920
import * as ThreadLaunch from "../../../orchestration-v2/ThreadLaunchService.ts";
2021
import * as ThreadManagement from "../../../orchestration-v2/ThreadManagementService.ts";
2122
import * as ServerConfig from "../../../config.ts";
@@ -103,7 +104,7 @@ const clientLaunchHarness = (input: {
103104
),
104105
}),
105106
Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({ namedProjectsRoot: "/projects" }),
106-
NodeServices.layer,
107+
DocumentText.layer.pipe(Layer.provideMerge(NodeServices.layer)),
107108
).pipe(
108109
Layer.provideMerge(GitVcsDriver.layer),
109110
Layer.provideMerge(VcsProcess.layer),

‎apps/server/src/mcp/toolkits/project/tools.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import {
1818
SourceControlCloneRepositoryResult,
1919
} from "@t3tools/contracts";
2020
import * as FileSystem from "effect/FileSystem";
21+
import * as DocumentText from "../../../provider/DocumentText.ts";
2122
import * as ServerConfig from "../../../config.ts";
2223
import * as ServerSettings from "../../../serverSettings.ts";
2324
import * as ThreadLaunchService from "../../../orchestration-v2/ThreadLaunchService.ts";
@@ -142,6 +143,7 @@ const ThreadLaunchTool = Tool.make("t3_thread_launch", {
142143
FileSystem.FileSystem,
143144
ServerConfig.ServerConfig,
144145
ServerSettings.ServerSettingsService,
146+
DocumentText.DocumentText,
145147
],
146148
})
147149
.annotate(Tool.Destructive, true)

0 commit comments

Comments
 (0)