Skip to content

Commit 28c32d2

Browse files
t3dotggclaude
andauthored
feat(server): send telemetry through ingest.t3.codes (#15028)
Co-authored-by: Claude Opus 5.5 (1M context) <[email protected]>
1 parent d8e9fd2 commit 28c32d2

8 files changed

Lines changed: 178 additions & 69 deletions

File tree

‎.github/workflows/desktop-macos-preview.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,10 @@ jobs:
7878
- name: Install browser secret helper build libraries
7979
run: sudo apt-get update && sudo apt-get install -y libsecret-1-dev pkg-config
8080

81+
# desktop-macos-preview-publish.yml packages this bundle as the preview channel.
8182
- name: Build JS bundle
83+
env:
84+
T3CODE_TELEMETRY_CHANNEL: ${{ github.repository == 'pingdotgg/t3code' && 'preview' || 'dev' }}
8285
run: vp run build:desktop
8386

8487
# Same layout as release.yml's js-bundle so release-desktop.yml can

‎.github/workflows/release-desktop.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -501,6 +501,9 @@ jobs:
501501
# the runner, and only exact versions have a dist directory. Keep in
502502
# step with SEA_NODE_VERSION in apps/server/vite.config.ts.
503503
VP_NODE_VERSION: "26.8.2"
504+
# The executable bundles the server again, so it needs the same stamp
505+
# as release.yml's JS bundle.
506+
T3CODE_TELEMETRY_CHANNEL: ${{ github.repository == 'pingdotgg/t3code' && inputs.release_channel || 'dev' }}
504507
run: node apps/server/scripts/cli.ts build-exe --verbose
505508

506509
- name: Import macOS signing certificate for the CLI archive

‎.github/workflows/release.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -525,7 +525,11 @@ jobs:
525525
# Runs t3#build (which depends on @t3tools/web#build) and
526526
# @t3tools/desktop#build, so apps/server/dist holds the server bundle
527527
# plus the web client and apps/desktop/dist-electron the Electron main.
528+
# The release channel is already stable, nightly, or preview. Builds in
529+
# other repositories report "dev" telemetry.
528530
- name: Build JS bundle
531+
env:
532+
T3CODE_TELEMETRY_CHANNEL: ${{ github.repository == 'pingdotgg/t3code' && needs.preflight.outputs.release_channel || 'dev' }}
529533
run: vp run build:desktop
530534

531535
# Two paths under apps/ so the artifact root is apps/; consumers download

‎apps/server/src/telemetry/AnalyticsService.test.ts‎

Lines changed: 108 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,15 @@
11
import * as NodeHttpServer from "@effect/platform-node/NodeHttpServer";
22
import * as NodeServices from "@effect/platform-node/NodeServices";
3-
import { assert, it } from "@effect/vitest";
3+
import { assert, it, vi } from "@effect/vitest";
44
import * as ConfigProvider from "effect/ConfigProvider";
55
import * as Effect from "effect/Effect";
66
import * as Layer from "effect/Layer";
77
import * as Schema from "effect/Schema";
88
import * as TestClock from "effect/testing/TestClock";
99
import * as HttpClient from "effect/http/HttpClient";
1010
import * as HttpClientError from "effect/http/HttpClientError";
11+
import * as HttpClientRequest from "effect/http/HttpClientRequest";
12+
import * as HttpClientResponse from "effect/http/HttpClientResponse";
1113
import * as HttpServer from "effect/http/HttpServer";
1214
import * as HttpServerRequest from "effect/http/HttpServerRequest";
1315
import * as HttpServerResponse from "effect/http/HttpServerResponse";
@@ -52,27 +54,84 @@ interface RecordedBatchBody {
5254

5355
const SentBatch = Schema.fromJsonString(
5456
Schema.Struct({
55-
batch: Schema.Array(Schema.Struct({ uuid: Schema.String })),
57+
batch: Schema.Array(
58+
Schema.Struct({
59+
uuid: Schema.String,
60+
properties: Schema.Struct({ telemetryChannel: Schema.String }),
61+
}),
62+
),
5663
}),
5764
);
65+
const decodeSentBatch = Schema.decodeEffect(SentBatch);
66+
type SentEvent = (typeof SentBatch.Type)["batch"][number];
67+
type Respond = (
68+
request: HttpClientRequest.HttpClientRequest,
69+
) => Effect.Effect<HttpClientResponse.HttpClientResponse, HttpClientError.HttpClientError>;
70+
71+
/** The connection drops after the endpoint read the batch, so it may be stored. */
72+
const connectionReset: Respond = (request) =>
73+
Effect.fail(
74+
new HttpClientError.HttpClientError({
75+
reason: new HttpClientError.TransportError({ request, cause: "connection reset" }),
76+
}),
77+
);
5878

59-
const layerAcceptThenFailClient = (batches: Array<ReadonlyArray<{ readonly uuid: string }>>) =>
60-
Layer.succeed(
61-
HttpClient.HttpClient,
62-
HttpClient.make((request) =>
79+
const respondWithStatus =
80+
(status: number): Respond =>
81+
(request) =>
82+
Effect.succeed(HttpClientResponse.fromWeb(request, new Response(null, { status })));
83+
84+
/**
85+
* Records 20 events, then lets ten minutes of background flushes run against
86+
* a client that answers every send with `respond`. Returns the sent batches.
87+
*/
88+
const sendForTenMinutes = (respond: Respond) =>
89+
Effect.gen(function* () {
90+
const batches: Array<ReadonlyArray<SentEvent>> = [];
91+
const client = HttpClient.make((request) =>
6392
Effect.gen(function* () {
6493
if (request.body._tag === "Uint8Array") {
65-
const body = yield* Schema.decodeEffect(SentBatch)(
66-
new TextDecoder().decode(request.body.body),
67-
).pipe(Effect.orDie);
94+
const body = yield* decodeSentBatch(new TextDecoder().decode(request.body.body)).pipe(
95+
Effect.orDie,
96+
);
6897
batches.push(body.batch);
6998
}
70-
return yield* new HttpClientError.HttpClientError({
71-
reason: new HttpClientError.TransportError({ request, cause: "connection reset" }),
72-
});
99+
return yield* respond(request);
73100
}),
74-
),
75-
);
101+
);
102+
const runtimeLayer = AnalyticsService.layer.pipe(
103+
Layer.provideMerge(
104+
ServerConfig.ServerConfig.layerTest(process.cwd(), { prefix: "t3-telemetry-retry-" }),
105+
),
106+
Layer.provide(
107+
ConfigProvider.layer(
108+
ConfigProvider.fromUnknown({
109+
T3CODE_TELEMETRY_ENABLED: true,
110+
T3CODE_POSTHOG_KEY: "phc_test_key",
111+
T3CODE_POSTHOG_HOST: "http://localhost",
112+
}),
113+
),
114+
),
115+
Layer.provide(
116+
Layer.mergeAll(
117+
Layer.succeed(HostProcess.Platform, "win32"),
118+
Layer.succeed(HostProcess.Architecture, "x64"),
119+
Layer.succeed(HttpClient.HttpClient, client),
120+
),
121+
),
122+
);
123+
124+
yield* Effect.gen(function* () {
125+
const analytics = yield* AnalyticsService.AnalyticsService;
126+
for (let index = 0; index < 20; index += 1) {
127+
yield* analytics.record("test.retry", { index });
128+
}
129+
for (let second = 0; second < 600; second += 1) {
130+
yield* TestClock.adjust("1 second");
131+
}
132+
}).pipe(Effect.provide(runtimeLayer));
133+
return batches;
134+
});
76135

77136
it("retryDelayMs doubles from 2s and stays under the 5 minute cap", () => {
78137
assert.equal(AnalyticsService.retryDelayMs(1, 0), 1_000);
@@ -82,48 +141,46 @@ it("retryDelayMs doubles from 2s and stays under the 5 minute cap", () => {
82141
});
83142

84143
it.layer(NodeServices.layer)("AnalyticsService test", (it) => {
85-
it.effect("a batch that keeps failing is retried with backoff, then dropped", () =>
144+
it.effect("a batch that keeps failing is retried with the same uuids, then dropped", () =>
86145
Effect.gen(function* () {
87-
const batches: Array<ReadonlyArray<{ readonly uuid: string }>> = [];
88-
const layerRuntime = AnalyticsService.layer.pipe(
89-
Layer.provideMerge(
90-
ServerConfig.ServerConfig.layerTest(process.cwd(), { prefix: "t3-telemetry-retry-" }),
91-
),
92-
Layer.provide(
93-
ConfigProvider.layer(
94-
ConfigProvider.fromUnknown({
95-
T3CODE_TELEMETRY_ENABLED: true,
96-
T3CODE_POSTHOG_KEY: "phc_test_key",
97-
T3CODE_POSTHOG_HOST: "http://localhost",
98-
}),
99-
),
100-
),
101-
Layer.provide(
102-
Layer.mergeAll(
103-
Layer.succeed(HostProcess.Platform, "win32"),
104-
Layer.succeed(HostProcess.Architecture, "x64"),
105-
layerAcceptThenFailClient(batches),
106-
),
107-
),
108-
);
109-
110-
yield* Effect.gen(function* () {
111-
const analytics = yield* AnalyticsService.AnalyticsService;
112-
for (let index = 0; index < 20; index += 1) {
113-
yield* analytics.record("test.retry", { index });
114-
}
115-
for (let second = 0; second < 600; second += 1) {
116-
yield* TestClock.adjust("1 second");
117-
}
118-
}).pipe(Effect.provide(layerRuntime));
146+
const failures = {
147+
"connection reset": connectionReset,
148+
"408": respondWithStatus(408),
149+
"429": respondWithStatus(429),
150+
"503": respondWithStatus(503),
151+
};
152+
for (const [failure, respond] of Object.entries(failures)) {
153+
const batches = yield* sendForTenMinutes(respond);
154+
assert.equal(batches.length, 5, failure);
155+
const uuids = batches.map((batch) => batch.map((event) => event.uuid).join(","));
156+
assert.equal(new Set(uuids).size, 1, `${failure}: every retry carries the same uuids`);
157+
assert.equal(new Set(batches[0]?.map((event) => event.uuid)).size, 20, failure);
158+
}
159+
}),
160+
);
119161

120-
assert.equal(batches.length, 5);
121-
const uuids = batches.map((batch) => batch.map((event) => event.uuid).join(","));
122-
assert.equal(new Set(uuids).size, 1, "every retry carries the same uuids");
123-
assert.equal(new Set(batches[0]?.map((event) => event.uuid)).size, 20);
162+
it.effect("a batch the endpoint rejects is dropped without a retry", () =>
163+
Effect.gen(function* () {
164+
for (const status of [400, 401, 413]) {
165+
const batches = yield* sendForTenMinutes(respondWithStatus(status));
166+
assert.equal(batches.length, 1, `status ${status}`);
167+
}
124168
}),
125169
);
126170

171+
it.effect("events carry the build's telemetry channel, or dev when none is set", () =>
172+
Effect.gen(function* () {
173+
const channels = (batches: ReadonlyArray<ReadonlyArray<SentEvent>>) => [
174+
...new Set(batches.flat().map((event) => event.properties.telemetryChannel)),
175+
];
176+
assert.deepEqual(channels(yield* sendForTenMinutes(respondWithStatus(204))), ["dev"]);
177+
178+
// Stands in for the value the bundler writes in at build time.
179+
vi.stubGlobal("__T3CODE_TELEMETRY_CHANNEL__", "nightly");
180+
assert.deepEqual(channels(yield* sendForTenMinutes(respondWithStatus(204))), ["nightly"]);
181+
}).pipe(Effect.ensuring(Effect.sync(() => vi.unstubAllGlobals()))),
182+
);
183+
127184
it.effect("flush drains all buffered events across multiple batches", () =>
128185
Effect.gen(function* () {
129186
const capturedRequests: Array<RecordedBatchRequest> = [];

‎apps/server/src/telemetry/AnalyticsService.ts‎

Lines changed: 37 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
/**
2-
* Each event carries a uuid, so PostHog can tell a retried copy from a new
3-
* event.
2+
* Events go to the T3 Code ingest endpoint, which keeps a copy and forwards
3+
* them to PostHog. Each event carries a uuid, so PostHog can tell a retried
4+
* copy from a new event.
45
*/
56
import * as HostProcess from "@t3tools/shared/HostProcess";
67
import type { ClientOs } from "@t3tools/contracts";
@@ -39,12 +40,26 @@ interface DeliveryState {
3940
readonly retryAt: number;
4041
}
4142

42-
const FLUSH_INTERVAL_MS = 1_000;
43+
// The background flush checks for work at this interval, so a retry delay
44+
// shorter than it waits for the next check.
45+
const FLUSH_INTERVAL_MS = 10_000;
4346
// A hung send would hold the flush lock, and with it the shutdown flush.
4447
const SEND_TIMEOUT = "10 seconds";
4548
const MAX_BATCH_ATTEMPTS = 5;
4649
const RETRY_BASE_DELAY_MS = 2_000;
4750
const RETRY_MAX_DELAY_MS = 300_000;
51+
// The endpoint answers these when the batch itself is wrong (bad body, wrong
52+
// api_key, too large), so sending it again cannot help. 408 and 429 are not
53+
// here because a later send can succeed.
54+
const REJECTED_BATCH_STATUSES = new Set([400, 401, 413]);
55+
56+
/**
57+
* Release channel of this build. Release workflows set it for
58+
* pingdotgg/t3code builds (see apps/server/vite.config.ts); every other build
59+
* reports "dev". It is separate from `__T3CODE_BUILD_CHANNEL__`, which names
60+
* stable "latest" and preview "nightly" for the sign-in page.
61+
*/
62+
declare const __T3CODE_TELEMETRY_CHANNEL__: "stable" | "nightly" | "preview" | "dev" | undefined;
4863

4964
/** `random` is in [0, 1). */
5065
export function retryDelayMs(failures: number, random: number): number {
@@ -57,7 +72,7 @@ const TelemetryEnvConfig = Config.all({
5772
Config.withDefault("phc_XOWci4oZP4VvLiEyrFqkFjP4CZn55mjYYBMREK5Wd6m"),
5873
),
5974
posthogHost: Config.String("T3CODE_POSTHOG_HOST").pipe(
60-
Config.withDefault("https://us.i.posthog.com"),
75+
Config.withDefault("https://ingest.t3.codes"),
6176
),
6277
enabled: Config.Boolean("T3CODE_TELEMETRY_ENABLED").pipe(Config.withDefault(true)),
6378
flushBatchSize: Config.Number("T3CODE_TELEMETRY_FLUSH_BATCH_SIZE").pipe(Config.withDefault(20)),
@@ -120,6 +135,8 @@ export const make = Effect.gen(function* () {
120135
// The background flush and the shutdown flush must not send the same batch at once.
121136
const flushLock = yield* Semaphore.make(1);
122137
const clientType = serverConfig.mode === "desktop" ? "desktop-app" : "cli-web-client";
138+
const telemetryChannel =
139+
typeof __T3CODE_TELEMETRY_CHANNEL__ === "undefined" ? "dev" : __T3CODE_TELEMETRY_CHANNEL__;
123140
const hostPlatform = yield* HostProcess.Platform;
124141
const hostArchitecture = yield* HostProcess.Architecture;
125142

@@ -173,6 +190,7 @@ export const make = Effect.gen(function* () {
173190
wsl: Option.getOrUndefined(telemetryConfig.wslDistroName),
174191
arch: hostArchitecture,
175192
t3CodeVersion: packageJson.version,
193+
telemetryChannel,
176194
clientType,
177195
serverOs: serverOsFromNodePlatform(hostPlatform),
178196
serverArch: hostArchitecture,
@@ -197,6 +215,8 @@ export const make = Effect.gen(function* () {
197215
return [nextBatch, current.slice(nextBatch.length)] as const;
198216
});
199217

218+
// A rejected batch is dropped at once, but the next batch still waits for the
219+
// backoff, so a wrong api_key does not send every batch as fast as it can.
200220
const flush: AnalyticsService["Service"]["flush"] = Effect.gen(function* () {
201221
while (true) {
202222
const delivery = yield* Ref.get(deliveryRef);
@@ -214,20 +234,26 @@ export const make = Effect.gen(function* () {
214234
const failures = delivery.failures + 1;
215235
const batchAttempts = delivery.batchAttempts + 1;
216236
const retryAt = (yield* Clock.currentTimeMillis) + retryDelayMs(failures, yield* Random.next);
217-
if (batchAttempts < MAX_BATCH_ATTEMPTS) {
237+
const error = sent.failure;
238+
const rejected =
239+
error._tag === "HttpClientError" &&
240+
error.reason._tag === "StatusCodeError" &&
241+
REJECTED_BATCH_STATUSES.has(error.reason.response.status);
242+
if (!rejected && batchAttempts < MAX_BATCH_ATTEMPTS) {
218243
yield* Ref.set(deliveryRef, { failedBatch: batch, batchAttempts, failures, retryAt });
219244
yield* Effect.logDebug("Failed to send telemetry batch; will retry", {
220245
attempt: batchAttempts,
221-
cause: sent.failure,
246+
cause: error,
222247
});
223248
return;
224249
}
225250
yield* Ref.set(deliveryRef, { failedBatch: [], batchAttempts: 0, failures, retryAt });
226-
yield* Effect.logWarning("Dropped telemetry batch after repeated send failures", {
227-
events: batch.length,
228-
attempts: batchAttempts,
229-
cause: sent.failure,
230-
});
251+
yield* Effect.logWarning(
252+
rejected
253+
? "Dropped telemetry batch the endpoint rejected"
254+
: "Dropped telemetry batch after repeated send failures",
255+
{ events: batch.length, attempts: batchAttempts, cause: error },
256+
);
231257
return;
232258
}
233259
}).pipe(flushLock.withPermit);

‎apps/server/vite.config.ts‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,16 @@ const repoEnv = loadRepoEnv();
1818
const cliBuildChannel = /^[^-+]+-(?:nightly|preview)\./.test(packageJson.version)
1919
? "nightly"
2020
: "latest";
21+
// Sent with every analytics event. Release workflows set it for pingdotgg/t3code
22+
// builds only, so local builds and forks report "dev". A typo fails the build
23+
// instead of mislabeling events.
24+
const TELEMETRY_CHANNELS = ["stable", "nightly", "preview", "dev"];
25+
const telemetryChannel = process.env.T3CODE_TELEMETRY_CHANNEL?.trim() || "dev";
26+
if (!TELEMETRY_CHANNELS.includes(telemetryChannel)) {
27+
throw new Error(
28+
`T3CODE_TELEMETRY_CHANNEL must be one of ${TELEMETRY_CHANNELS.join(", ")}, got "${telemetryChannel}".`,
29+
);
30+
}
2131

2232
// `build:exe` wraps the same bundle in a Node single-executable. tsdown's exe
2333
// step refuses multi-chunk output and counts the sourcemap as a chunk, and the
@@ -93,6 +103,7 @@ export default mergeConfig(
93103
},
94104
define: {
95105
__T3CODE_BUILD_CHANNEL__: JSON.stringify(cliBuildChannel),
106+
__T3CODE_TELEMETRY_CHANNEL__: JSON.stringify(telemetryChannel),
96107
__T3CODE_BUILD_RELAY_URL__: JSON.stringify(repoEnv.T3CODE_RELAY_URL?.trim() ?? ""),
97108
__T3CODE_BUILD_CLERK_PUBLISHABLE_KEY__: JSON.stringify(
98109
repoEnv.T3CODE_CLERK_PUBLISHABLE_KEY?.trim() ?? "",

‎docs/internals/product-analytics.md‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# Product analytics
22

3-
The server owns PostHog delivery, opt-out, and identity for every connected client.
3+
The server owns analytics delivery, opt-out, and identity for every connected client.
44
[Identity selection](../../apps/server/src/telemetry/Identify.ts) hashes an available
55
provider account ID, falling back to an installation-scoped ID. This identity can
66
span several clients; it does not identify a browser session. Clients do not load
@@ -43,10 +43,13 @@ normalization or building reports.
4343

4444
## Delivery
4545

46-
A send can fail after PostHog has stored the batch, so every retry is a copy.
46+
The server sends batches to the ingest endpoint at `ingest.t3.codes`, which archives
47+
every event and forwards some of them to PostHog. A send can fail after the endpoint
48+
has stored the batch, so every retry is a copy.
4749
[Delivery](../../apps/server/src/telemetry/AnalyticsService.ts) gives each event a
48-
uuid when it is recorded, backs off after a failed send, and drops a batch after a
49-
few tries. Without these limits, one stuck batch was sent every second for days.
50+
uuid when it is recorded (the endpoint rejects events without one), backs off after
51+
a failed send, and drops a batch after a few tries. Without these limits, one stuck
52+
batch was sent every second for days.
5053

5154
## Collection boundary
5255

0 commit comments

Comments
 (0)