Skip to content

Commit 80a5ffa

Browse files
only21milclaudemaria-rcks
authored
fix(server): T3 Connect keeps a log file of its tunnel connector (#24485)
Co-authored-by: Claude Opus 5.5 <[email protected]> Co-authored-by: maria-rcks <[email protected]> Co-authored-by: maria-rcks <[email protected]>
1 parent fc1d604 commit 80a5ffa

6 files changed

Lines changed: 174 additions & 1 deletion

File tree

‎apps/server/src/cloud/ManagedEndpointRuntime.test.ts‎

Lines changed: 105 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,15 @@ import * as Duration from "effect/Duration";
66
import * as Effect from "effect/Effect";
77
import * as Exit from "effect/Exit";
88
import * as Fiber from "effect/Fiber";
9+
import * as FileSystem from "effect/FileSystem";
910
import * as Layer from "effect/Layer";
1011
import * as Logger from "effect/Logger";
1112
import * as Option from "effect/Option";
13+
import * as Path from "effect/Path";
1214
import * as PlatformError from "effect/PlatformError";
1315
import * as Queue from "effect/Queue";
1416
import * as References from "effect/References";
17+
import * as Schema from "effect/Schema";
1518
import * as Sink from "effect/Sink";
1619
import * as Stream from "effect/Stream";
1720
import * as TestClock from "effect/testing/TestClock";
@@ -42,6 +45,15 @@ const layerRelayClientAvailable = Layer.succeed(
4245

4346
const unusedHttpClient = HttpClient.make(() => Effect.die("unused"));
4447

48+
const decodeLogEntry = Schema.decodeUnknownSync(
49+
Schema.fromJsonString(
50+
Schema.Struct({
51+
level: Schema.String,
52+
message: Schema.Tuple([Schema.String, Schema.Unknown]),
53+
}),
54+
),
55+
);
56+
4557
const emptySecretStoreLayer = Layer.mock(ServerSecretStore.ServerSecretStore)({
4658
get: () => Effect.succeedNone,
4759
set: () => Effect.void,
@@ -59,7 +71,8 @@ const runtimeDependencies = (
5971
Layer.succeed(HttpClient.HttpClient, httpClient),
6072
relayClientLayer,
6173
secretStoreLayer,
62-
);
74+
ServerConfig.layerTest(process.cwd(), { prefix: "t3-managed-endpoint-" }),
75+
).pipe(Layer.provideMerge(NodeServices.layer));
6376

6477
const buildCloudManagedEndpointRuntime = (
6578
spawner: ReturnType<typeof ChildProcessSpawner.make>,
@@ -543,6 +556,97 @@ describe("CloudManagedEndpointRuntime", () => {
543556
}),
544557
);
545558

559+
it.effect("keeps redacted connector logs bounded even when rotation fails", () =>
560+
Effect.gen(function* () {
561+
const fs = yield* FileSystem.FileSystem;
562+
const path = yield* Path.Path;
563+
for (const rotationBlocked of [false, true]) {
564+
const baseDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3-relay-client-log-" });
565+
const logPath = path.join(baseDir, "userdata", "logs", "relay-client.ndjson");
566+
const existingLog = "x".repeat(1024 * 1024);
567+
if (rotationBlocked) {
568+
yield* fs.makeDirectory(`${logPath}.2`, { recursive: true });
569+
yield* fs.writeFileString(`${logPath}.2/blocked`, "blocks rotation");
570+
yield* fs.writeFileString(logPath, existingLog);
571+
}
572+
const warnings: unknown[] = [];
573+
const logger = Logger.make(({ message }) => {
574+
if (Array.isArray(message) && message[0] === "Relay client log unavailable") {
575+
warnings.push(message);
576+
}
577+
});
578+
const fixture = yield* makeTransportSpawner();
579+
yield* Effect.gen(function* () {
580+
const context = yield* Layer.build(
581+
ManagedEndpointRuntime.layer.pipe(
582+
Layer.provide(ServerConfig.layerTest(process.cwd(), baseDir)),
583+
Layer.provide(runtimeDependencies(fixture.spawner)),
584+
),
585+
);
586+
const runtime = yield* Effect.service(
587+
ManagedEndpointRuntime.CloudManagedEndpointRuntime,
588+
).pipe(Effect.provide(context));
589+
yield* runtime.applyConfig({ ...tunnelConfig, connectorToken: "connector-secret" });
590+
yield* fixture.observe(
591+
runtime,
592+
0,
593+
'2026-10-16T20:29:40Z ERR Failed to serve tunnel connection error="rejected connector-secret" connIndex=0\n',
594+
);
595+
}).pipe(Effect.scoped, Effect.withLogger(logger));
596+
597+
const log = yield* fs.readFileString(logPath);
598+
if (rotationBlocked) {
599+
expect(log.length).toBe(existingLog.length);
600+
expect(warnings).toHaveLength(1);
601+
continue;
602+
}
603+
const entries = log
604+
.trim()
605+
.split("\n")
606+
.map((line) => decodeLogEntry(line));
607+
expect(entries.map((entry) => [entry.level, entry.message[0]])).toEqual([
608+
["INFO", "Relay client process started; waiting for tunnel connection"],
609+
["WARN", "Relay client reported a transport warning"],
610+
["INFO", "Relay client tunnel connection registered"],
611+
["INFO", "Relay client stopped"],
612+
]);
613+
expect(log).toContain("rejected <redacted>");
614+
expect(log).not.toContain("connector-secret");
615+
const oversizedFixture = yield* makeTransportSpawner();
616+
yield* Effect.gen(function* () {
617+
const context = yield* Layer.build(
618+
ManagedEndpointRuntime.layer.pipe(
619+
Layer.provide(ServerConfig.layerTest(process.cwd(), baseDir)),
620+
Layer.provide(runtimeDependencies(oversizedFixture.spawner)),
621+
),
622+
);
623+
const runtime = yield* Effect.service(
624+
ManagedEndpointRuntime.CloudManagedEndpointRuntime,
625+
).pipe(Effect.provide(context));
626+
yield* runtime.applyConfig(tunnelConfig);
627+
yield* oversizedFixture.observe(runtime, 0, "ERR " + "界".repeat(1024 * 1024) + "\n");
628+
}).pipe(Effect.scoped);
629+
const files = yield* fs.readDirectory(path.dirname(logPath));
630+
const allEntries = [];
631+
for (const file of files.filter((name) => name.startsWith("relay-client.ndjson"))) {
632+
const contents = yield* fs.readFileString(path.join(path.dirname(logPath), file));
633+
expect(Buffer.byteLength(contents)).toBeLessThanOrEqual(1024 * 1024);
634+
allEntries.push(
635+
...contents
636+
.trim()
637+
.split("\n")
638+
.map((line) => decodeLogEntry(line)),
639+
);
640+
}
641+
expect(
642+
allEntries.some(
643+
(entry) => entry.message[0] === "Relay client log entry omitted: exceeds 1 MiB",
644+
),
645+
).toBe(true);
646+
}
647+
}).pipe(Effect.provide(NodeServices.layer)),
648+
);
649+
546650
it.effect("discards a pending fallback when its connector is replaced or removed", () =>
547651
Effect.gen(function* () {
548652
for (const nextConfig of [{ ...tunnelConfig, connectorToken: "replacement" }, null]) {

‎apps/server/src/cloud/ManagedEndpointRuntime.ts‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import type { RelayManagedEndpointRuntimeConfig } from "@t3tools/contracts/relay";
2+
import { RotatingFileSink } from "@t3tools/shared/logging";
23
import * as RelayClient from "@t3tools/shared/relayClient";
34
import * as Clock from "effect/Clock";
45
import * as Context from "effect/Context";
@@ -7,7 +8,9 @@ import * as Duration from "effect/Duration";
78
import * as Effect from "effect/Effect";
89
import * as Exit from "effect/Exit";
910
import * as Layer from "effect/Layer";
11+
import * as Logger from "effect/Logger";
1012
import * as Option from "effect/Option";
13+
import * as Path from "effect/Path";
1114
import * as Queue from "effect/Queue";
1215
import * as Ref from "effect/Ref";
1316
import * as Result from "effect/Result";
@@ -20,6 +23,7 @@ import * as ChildProcess from "effect/process/ChildProcess";
2023
import * as ChildProcessSpawner from "effect/process/ChildProcessSpawner";
2124

2225
import * as ServerSecretStore from "../auth/ServerSecretStore.ts";
26+
import * as ServerConfig from "../config.ts";
2327
import * as ManagedLimits from "../policy/managedLimits.ts";
2428
import * as ManagedPolicy from "../policy/ManagedPolicy.ts";
2529

@@ -123,6 +127,7 @@ const encodeHttp2Fallback = Schema.encodeSync(Http2Fallback);
123127
// than this, so a crash-looping connector or an offline host does not redownload
124128
// on every reconcile.
125129
const RELAY_CLIENT_INSTALL_RETRY_INTERVAL_MS = 10 * 60_000;
130+
export const RELAY_CLIENT_LOG_FILE = "relay-client.ndjson";
126131

127132
/** Preserve user-chosen PATH binaries and overrides; CLI downloads require consent. */
128133
function needsPinnedRelayClient(executable: RelayClient.RelayClientStatus): boolean {
@@ -227,6 +232,60 @@ export const make = Effect.gen(function* () {
227232
const quicStallsRef = yield* Ref.make<ReadonlyArray<number>>([]);
228233
let reconcileConfig: CloudManagedEndpointRuntime["Service"]["applyConfig"];
229234

235+
const { logsDir } = yield* ServerConfig.ServerConfig;
236+
const path = yield* Path.Path;
237+
const relayClientLogPath = path.join(logsDir, RELAY_CLIENT_LOG_FILE);
238+
// Synchronous appends avoid a flush timer for this low-volume log.
239+
const fallbackLoggers = yield* Effect.service(Logger.CurrentLoggers);
240+
let writeFailureReported = false;
241+
const relayClientLog = yield* Effect.try(
242+
() =>
243+
new RotatingFileSink({
244+
filePath: relayClientLogPath,
245+
maxBytes: 1024 * 1024,
246+
maxFiles: 2,
247+
throwOnError: true,
248+
}),
249+
).pipe(
250+
Effect.map((sink) =>
251+
Logger.make((options) => {
252+
try {
253+
const entry = Logger.formatJson.log(options);
254+
const bytes = Buffer.byteLength(entry) + 1;
255+
sink.write(
256+
bytes <= 1024 * 1024
257+
? `${entry}\n`
258+
: `${JSON.stringify({
259+
timestamp: options.date.toISOString(),
260+
level: options.logLevel.toUpperCase(),
261+
message: ["Relay client log entry omitted: exceeds 1 MiB", { bytes }],
262+
})}\n`,
263+
);
264+
} catch {
265+
if (!writeFailureReported) {
266+
writeFailureReported = true;
267+
for (const logger of fallbackLoggers) {
268+
logger.log({
269+
...options,
270+
logLevel: "Warn",
271+
message: ["Relay client log unavailable", { filePath: relayClientLogPath }],
272+
});
273+
}
274+
}
275+
}
276+
}),
277+
),
278+
Effect.catch((error) =>
279+
Effect.logWarning("Relay client log unavailable", {
280+
filePath: relayClientLogPath,
281+
cause: error,
282+
}).pipe(Effect.as(undefined)),
283+
),
284+
);
285+
// Forked connector fibers inherit this logger after the applyConfig span ends.
286+
const withRelayClientLog = <A, E, R>(effect: Effect.Effect<A, E, R>) =>
287+
relayClientLog ? Effect.withLogger(effect, relayClientLog) : effect;
288+
230289
const stopActive = Effect.gen(function* () {
231290
const active = yield* Ref.getAndSet(activeRef, null);
232291
yield* stopConnector(active);
@@ -760,6 +819,7 @@ export const make = Effect.gen(function* () {
760819
return yield* reconcileConfig(config);
761820
}),
762821
),
822+
withRelayClientLog,
763823
);
764824

765825
const readMetrics = <S extends Schema.Decoder<unknown>>(url: string, schema: S) =>

‎apps/server/src/triage/triagePrompt.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ const contextInput = {
7272
logsDir: "/home/u/.t3/userdata/logs",
7373
serviceLogPath: "/home/u/.t3/userdata/logs/boot-service.log",
7474
desktopBackendLogGlob: "/home/u/.t3/userdata/logs/server-child*.log*",
75+
relayClientLogPath: "/home/u/.t3/userdata/logs/relay-client.ndjson",
7576
serverTracePath: "/home/u/.t3/userdata/logs/server.trace.ndjson",
7677
providerEventLogPath: "/home/u/.t3/userdata/logs/provider/events.log",
7778
terminalLogsDir: "/home/u/.t3/userdata/logs/terminals",

‎apps/server/src/triage/triagePrompt.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -217,6 +217,7 @@ export interface TriageContextInput {
217217
readonly logsDir: string;
218218
readonly serviceLogPath: string;
219219
readonly desktopBackendLogGlob: string;
220+
readonly relayClientLogPath: string;
220221
readonly serverTracePath: string;
221222
readonly providerEventLogPath: string;
222223
readonly terminalLogsDir: string;
@@ -302,6 +303,7 @@ ${input.entry.kind === "app" ? IN_APP_SECTION : ""}${input.sourceClone === undef
302303
- Logs dir: ${input.paths.logsDir}
303304
- Service log (systemd/launchd service installs only): ${input.paths.serviceLogPath}
304305
- Desktop backend logs (glob; one file per backend plus rotated copies; written only when a backend crashes or fails to start): ${input.paths.desktopBackendLogGlob}
306+
- T3 Connect tunnel log (ndjson; what the server logged about its cloudflared connector, with rotated copies \`.1\` and \`.2\`): ${input.paths.relayClientLogPath}
305307
- Server trace (ndjson): ${input.paths.serverTracePath}
306308
- Provider event log: ${input.paths.providerEventLogPath}
307309
- Terminal logs: ${input.paths.terminalLogsDir}

‎apps/server/src/triage/triageSession.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import * as Option from "effect/Option";
1010
import * as Path from "effect/Path";
1111

1212
import * as BootService from "../cloud/bootService.ts";
13+
import * as ManagedEndpointRuntime from "../cloud/ManagedEndpointRuntime.ts";
1314
import type * as ServerConfig from "../config.ts";
1415
import { isProcessAlive, readPersistedServerRuntimeState } from "../serverRuntimeState.ts";
1516
import {
@@ -80,6 +81,7 @@ export const writeTriageSession = Effect.fn("triage.writeTriageSession")(functio
8081
logsDir: paths.logsDir,
8182
serviceLogPath: path.join(paths.logsDir, BootService.BOOT_SERVICE_LOG_FILE),
8283
desktopBackendLogGlob: path.join(paths.logsDir, "server-child*.log*"),
84+
relayClientLogPath: path.join(paths.logsDir, ManagedEndpointRuntime.RELAY_CLIENT_LOG_FILE),
8385
serverTracePath: paths.serverTracePath,
8486
providerEventLogPath: paths.providerEventLogPath,
8587
terminalLogsDir: paths.terminalLogsDir,

‎docs/user/t3-connect-troubleshooting.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,3 +151,7 @@ If none of this helps, open an issue on
151151
- The text from **Copy diagnostics** in the environment's **Details**.
152152
- The output of `t3 connect status` on the host.
153153
- The trace ID, if one is shown.
154+
- If the tunnel dropped earlier, the host's tunnel log, `relay-client.ndjson`
155+
in the logs folder (`~/.t3/userdata/logs` by default). It records when the
156+
tunnel connected, warned, exited, and restarted, with the tunnel token
157+
removed.

0 commit comments

Comments
 (0)