Skip to content

Commit a21b42c

Browse files
t3dotggclaude
andauthored
fix(server): prune expired replay-protection files from the secrets directory (pingdotgg#13695)
Co-authored-by: Claude Opus 5.5 (1M context) <[email protected]>
1 parent 5d707bb commit a21b42c

6 files changed

Lines changed: 293 additions & 1 deletion

File tree

‎apps/server/src/auth/dpop.ts‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,9 @@ import {
1818
} from "./EnvironmentAuth.ts";
1919
import * as ServerSecretStore from "./ServerSecretStore.ts";
2020

21+
/** Secret store name prefix of DPoP replay markers. The server prunes expired ones. */
22+
export const DPOP_REPLAY_MARKER_PREFIX = "dpop-proof-";
23+
2124
export const mapDpopFailureReason = (code: DpopVerificationFailureCodeType): DpopFailureReason => {
2225
switch (code) {
2326
case "time_window":
@@ -96,7 +99,7 @@ export const verifyRequestDpopProof = (input: {
9699
);
97100
yield* secretStore
98101
.create(
99-
`dpop-proof-${replayKey}`,
102+
`${DPOP_REPLAY_MARKER_PREFIX}${replayKey}`,
100103
new TextEncoder().encode(
101104
[
102105
`thumbprint=${result.thumbprint}`,
Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
import * as NodeServices from "@effect/platform-node/NodeServices";
2+
import { assert, it } from "@effect/vitest";
3+
import * as DateTime from "effect/DateTime";
4+
import * as Duration from "effect/Duration";
5+
import * as Effect from "effect/Effect";
6+
import * as FileSystem from "effect/FileSystem";
7+
import * as Layer from "effect/Layer";
8+
import * as Path from "effect/Path";
9+
import * as TestClock from "effect/testing/TestClock";
10+
11+
import * as ServerConfig from "../config.ts";
12+
import { pruneExpiredReplayMarkers, REPLAY_MARKER_MAX_AGE } from "./replayMarkers.ts";
13+
import * as ServerSecretStore from "./ServerSecretStore.ts";
14+
15+
// Every secret name the server stores today. The last three stand for names
16+
// built from an id at runtime.
17+
const REAL_SECRET_NAMES = [
18+
"server-signing-key",
19+
"asset-access-signing-key",
20+
"cloud-cli-oauth-token",
21+
"cloud-cli-desired-link",
22+
"cloud-link-ed25519-key-pair",
23+
"cloud-link-ed25519-private-key",
24+
"cloud-link-ed25519-public-key",
25+
"cloud-mint-ed25519-public-key",
26+
"cloud-endpoint-runtime-config",
27+
"cloud-endpoint-confirmed-origin",
28+
"cloud-linked-user-id",
29+
"cloud-relay-url",
30+
"cloud-relay-issuer",
31+
"cloud-relay-environment-credential",
32+
"cloud-publish-agent-activity",
33+
"provider-env-Y29kZXg-T1BFTkFJX0FQSV9LRVk",
34+
"provider-auth-0f1e2d3c4b5a69788796a5b4c3d2e1f00f1e2d3c4b5a69788796a5b4c3d2e1f0",
35+
"usage-limit-source-aHVi",
36+
];
37+
38+
it.layer(NodeServices.layer)("replayMarkers", (it) => {
39+
it.effect("prunes only replay markers older than the max age", () =>
40+
Effect.gen(function* () {
41+
const secretStore = yield* ServerSecretStore.ServerSecretStore;
42+
const { secretsDir } = yield* ServerConfig.ServerConfig;
43+
const fileSystem = yield* FileSystem.FileSystem;
44+
const path = yield* Path.Path;
45+
const now = DateTime.makeUnsafe("2026-01-01T00:00:00Z");
46+
const setAge = (fileName: string, age: Duration.Duration) => {
47+
const mtime = DateTime.toDateUtc(DateTime.subtractDuration(now, age));
48+
return fileSystem.utimes(path.join(secretsDir, fileName), mtime, mtime);
49+
};
50+
const writeAged = (name: string, age: Duration.Duration) =>
51+
secretStore
52+
.create(name, Uint8Array.from([1]))
53+
.pipe(Effect.andThen(setAge(`${name}.bin`, age)));
54+
55+
const justExpired = Duration.sum(REPLAY_MARKER_MAX_AGE, Duration.seconds(1));
56+
const expiredMarkers = [
57+
"dpop-proof-old",
58+
"cloud-mint-jti-old",
59+
"cloud-mint-nonce-old",
60+
"cloud-health-jti-old",
61+
"cloud-health-nonce-old",
62+
];
63+
for (const name of expiredMarkers) yield* writeAged(name, justExpired);
64+
yield* writeAged("dpop-proof-at-max-age", REPLAY_MARKER_MAX_AGE);
65+
for (const name of REAL_SECRET_NAMES) yield* writeAged(name, Duration.days(30));
66+
const pendingSetFile = "dpop-proof-pending.bin.0000.tmp";
67+
yield* fileSystem.writeFile(path.join(secretsDir, pendingSetFile), Uint8Array.from([1]));
68+
yield* setAge(pendingSetFile, Duration.days(30));
69+
yield* TestClock.setTime(DateTime.toEpochMillis(now));
70+
71+
yield* pruneExpiredReplayMarkers();
72+
73+
const remaining = yield* fileSystem.readDirectory(secretsDir);
74+
assert.deepStrictEqual(
75+
remaining.toSorted(),
76+
[
77+
...REAL_SECRET_NAMES.map((name) => `${name}.bin`),
78+
"dpop-proof-at-max-age.bin",
79+
pendingSetFile,
80+
].toSorted(),
81+
);
82+
}).pipe(
83+
Effect.provide(
84+
ServerSecretStore.layer.pipe(
85+
Layer.provideMerge(
86+
ServerConfig.layerTest(process.cwd(), { prefix: "t3-replay-markers-test-" }),
87+
),
88+
),
89+
),
90+
),
91+
);
92+
});
Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
import * as Clock from "effect/Clock";
2+
import * as Duration from "effect/Duration";
3+
import * as Effect from "effect/Effect";
4+
import * as FileSystem from "effect/FileSystem";
5+
import * as Layer from "effect/Layer";
6+
import * as Option from "effect/Option";
7+
import * as Path from "effect/Path";
8+
import * as Schedule from "effect/Schedule";
9+
10+
import { CLOUD_REPLAY_MARKER_PREFIXES } from "../cloud/http.ts";
11+
import * as ServerConfig from "../config.ts";
12+
import { forkParked } from "../serverActivation.ts";
13+
import { DPOP_REPLAY_MARKER_PREFIX } from "./dpop.ts";
14+
15+
const REPLAY_MARKER_PREFIXES = [DPOP_REPLAY_MARKER_PREFIX, ...CLOUD_REPLAY_MARKER_PREFIXES];
16+
17+
/**
18+
* How long a replay marker stays on disk. A marker only matters while its proof
19+
* can pass the time check (about 5 minutes for DPoP, 7 for cloud proofs). After
20+
* that, the time check rejects a replay by itself. The sweep and the time check
21+
* both use the wall clock, so a pruned marker can let a replay through only if
22+
* the clock moves back by almost a day, or if the filesystem stamps mtimes almost
23+
* a day behind. Markers are files, so a restart does not reset them.
24+
*/
25+
export const REPLAY_MARKER_MAX_AGE = Duration.days(1);
26+
27+
/**
28+
* Deletes replay markers whose mtime is older than `REPLAY_MARKER_MAX_AGE`.
29+
* `ServerSecretStore` saves each secret as `<name>.bin`, so only
30+
* `<marker prefix>*.bin` names match. Other secrets and the `*.bin.<uuid>.tmp`
31+
* files that `set` writes are never touched.
32+
*/
33+
export const pruneExpiredReplayMarkers = Effect.fn("replayMarkers.pruneExpired")(function* () {
34+
const fileSystem = yield* FileSystem.FileSystem;
35+
const path = yield* Path.Path;
36+
const { secretsDir } = yield* ServerConfig.ServerConfig;
37+
const cutoff = (yield* Clock.currentTimeMillis) - Duration.toMillis(REPLAY_MARKER_MAX_AGE);
38+
const markers = (yield* fileSystem.readDirectory(secretsDir)).filter(
39+
(name) =>
40+
name.endsWith(".bin") && REPLAY_MARKER_PREFIXES.some((prefix) => name.startsWith(prefix)),
41+
);
42+
// `partition` visits every marker, so one locked file does not stop the sweep.
43+
const [failures, removed] = yield* Effect.partition(markers, (name) => {
44+
const markerPath = path.join(secretsDir, name);
45+
return fileSystem.stat(markerPath).pipe(
46+
Effect.flatMap((info) =>
47+
Option.exists(info.mtime, (mtime) => mtime.getTime() < cutoff)
48+
? fileSystem.remove(markerPath).pipe(Effect.as(true))
49+
: Effect.succeed(false),
50+
),
51+
Effect.catchReason("PlatformError", "NotFound", () => Effect.succeed(false)),
52+
);
53+
});
54+
yield* Effect.annotateCurrentSpan({
55+
"replay_markers.matched": markers.length,
56+
"replay_markers.removed": removed.filter(Boolean).length,
57+
"replay_markers.failed": failures.length,
58+
});
59+
if (failures.length > 0) {
60+
yield* Effect.logWarning("Failed to prune some replay markers", {
61+
failed: failures.length,
62+
cause: failures[0],
63+
});
64+
}
65+
});
66+
67+
/** Prunes expired replay markers after server activation, then every hour. */
68+
export const layer = Layer.effectDiscard(
69+
forkParked(
70+
pruneExpiredReplayMarkers().pipe(
71+
Effect.catch((cause) =>
72+
Effect.logWarning("Failed to prune expired replay markers", { cause }),
73+
),
74+
Effect.repeat(Schedule.spaced(Duration.hours(1))),
75+
),
76+
),
77+
);

‎apps/server/src/cloud/http.ts‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,13 @@ const CLOUD_MINT_NONCE_PREFIX = "cloud-mint-nonce-";
104104
const CLOUD_MINT_JTI_PREFIX = "cloud-mint-jti-";
105105
const CLOUD_HEALTH_NONCE_PREFIX = "cloud-health-nonce-";
106106
const CLOUD_HEALTH_JTI_PREFIX = "cloud-health-jti-";
107+
/** Secret store name prefixes of cloud replay markers. The server prunes expired ones. */
108+
export const CLOUD_REPLAY_MARKER_PREFIXES = [
109+
CLOUD_MINT_NONCE_PREFIX,
110+
CLOUD_MINT_JTI_PREFIX,
111+
CLOUD_HEALTH_NONCE_PREFIX,
112+
CLOUD_HEALTH_JTI_PREFIX,
113+
] as const;
107114
const CLOUD_PROOF_MAX_LIFETIME_SECONDS = 5 * 60;
108115
const CLOUD_PROOF_CLOCK_SKEW_SECONDS = 60;
109116
// The desktop app stops its backends within seconds of writing the marker.

‎apps/server/src/server.test.ts‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,6 +176,7 @@ import * as VcsProcess from "./vcs/VcsProcess.ts";
176176
import * as GitWorkflowService from "./git/GitWorkflowService.ts";
177177
import * as ReviewService from "./review/ReviewService.ts";
178178
import * as SourceControlRepositoryService from "./sourceControl/SourceControlRepositoryService.ts";
179+
import { REPLAY_MARKER_MAX_AGE } from "./auth/replayMarkers.ts";
179180
import * as ServerSecretStore from "./auth/ServerSecretStore.ts";
180181
import * as EnvironmentAuth from "./auth/EnvironmentAuth.ts";
181182
import * as PairingGrantStore from "./auth/PairingGrantStore.ts";
@@ -2643,6 +2644,40 @@ it.layer(NodeServices.layer)("server router seam", (it) => {
26432644
}).pipe(Effect.provide(NodeHttpServer.layerTest)),
26442645
);
26452646

2647+
it.effect("rejects a DPoP replay by time alone once its marker can be pruned", () =>
2648+
Effect.gen(function* () {
2649+
yield* buildAppUnderTest();
2650+
2651+
const ownerCookie = yield* getAuthenticatedSessionCookieHeader();
2652+
const credentialResponse = yield* HttpClient.post("/api/auth/pairing-token", {
2653+
headers: { cookie: ownerCookie },
2654+
body: yield* HttpBody.json({}),
2655+
});
2656+
const credential = (yield* credentialResponse.json) as { readonly credential: string };
2657+
const tokenUrl = yield* getHttpServerUrl("/oauth/token");
2658+
const acceptedAt = yield* DateTime.now;
2659+
// The longest-lived proof: `iat` at the 5 s future skew the verifier allows.
2660+
const dpop = makeDpopProof({
2661+
method: "POST",
2662+
url: tokenUrl,
2663+
iat: Math.floor(acceptedAt.epochMilliseconds / 1_000) + 5,
2664+
});
2665+
const exchange = exchangeAccessToken(credential.credential, {
2666+
headers: { dpop: dpop.proof },
2667+
scope: "orchestration:read orchestration:operate terminal:operate review:write",
2668+
});
2669+
2670+
assert.equal((yield* exchange).response.status, 200);
2671+
// While the proof is fresh, only the replay marker rejects it.
2672+
assert.equal((yield* exchange).body.dpopFailureReason, "replay");
2673+
// Once the marker can be pruned, the time check rejects the proof by itself.
2674+
yield* TestClock.setTime(
2675+
acceptedAt.epochMilliseconds + Duration.toMillis(REPLAY_MARKER_MAX_AGE),
2676+
);
2677+
assert.equal((yield* exchange).body.dpopFailureReason, "time_window");
2678+
}).pipe(Effect.provide(NodeHttpServer.layerTest)),
2679+
);
2680+
26462681
it.effect("ignores forwarded host headers when validating token exchange DPoP URLs", () =>
26472682
Effect.gen(function* () {
26482683
yield* buildAppUnderTest();
@@ -3710,6 +3745,82 @@ it.layer(NodeServices.layer)("server router seam", (it) => {
37103745
}).pipe(Effect.provide(NodeHttpServer.layerTest)),
37113746
);
37123747

3748+
it.effect("rejects cloud replays by time alone once their markers can be pruned", () =>
3749+
Effect.gen(function* () {
3750+
yield* buildAppUnderTest();
3751+
3752+
const cloudKeyPair = NodeCrypto.generateKeyPairSync("ed25519", {
3753+
privateKeyEncoding: { format: "pem", type: "pkcs8" },
3754+
publicKeyEncoding: { format: "pem", type: "spki" },
3755+
});
3756+
const ownerCookie = yield* getAuthenticatedSessionCookieHeader();
3757+
const relayConfigResponse = yield* fetchEffect(
3758+
yield* getHttpServerUrl("/api/connect/relay-config"),
3759+
{
3760+
method: "POST",
3761+
headers: { cookie: ownerCookie, "content-type": "application/json" },
3762+
body: jsonRequestBody({
3763+
relayUrl: "https://relay.example.test",
3764+
cloudUserId: "user_123",
3765+
environmentCredential: "t3env_test_credential",
3766+
cloudMintPublicKey: cloudKeyPair.publicKey,
3767+
endpointRuntime: null,
3768+
}),
3769+
},
3770+
);
3771+
assert.equal(relayConfigResponse.status, 200);
3772+
3773+
const acceptedAt = yield* DateTime.now;
3774+
// The longest-lived proofs: `iat` at the 60 s future skew the handlers
3775+
// allow, and the 5 minute maximum lifetime.
3776+
const issuedAt = DateTime.add(acceptedAt, { minutes: 1 });
3777+
const proofTimes = {
3778+
issuedAt: DateTime.formatIso(issuedAt),
3779+
expiresAt: DateTime.formatIso(DateTime.add(issuedAt, { minutes: 5 })),
3780+
};
3781+
const requests = [
3782+
[
3783+
"/api/t3-connect/health",
3784+
makeCloudEnvironmentHealthRequest({
3785+
privateKey: cloudKeyPair.privateKey,
3786+
environmentId: testEnvironmentDescriptor.environmentId,
3787+
nonce: "cloud-health-nonce-pruned",
3788+
...proofTimes,
3789+
}),
3790+
],
3791+
[
3792+
"/api/t3-connect/mint-credential",
3793+
makeCloudMintCredentialRequest({
3794+
privateKey: cloudKeyPair.privateKey,
3795+
environmentId: testEnvironmentDescriptor.environmentId,
3796+
clientProofKeyThumbprint: "client-proof-key-thumbprint",
3797+
nonce: "cloud-mint-nonce-pruned",
3798+
...proofTimes,
3799+
}),
3800+
],
3801+
] as const;
3802+
const postAll = Effect.forEach(requests, ([pathname, request]) =>
3803+
Effect.gen(function* () {
3804+
const response = yield* fetchEffect(yield* getHttpServerUrl(pathname), {
3805+
method: "POST",
3806+
headers: { "content-type": "application/json" },
3807+
body: jsonRequestBody(request),
3808+
});
3809+
return response.status;
3810+
}),
3811+
);
3812+
3813+
assert.deepStrictEqual(yield* postAll, [200, 200]);
3814+
// While the proofs are fresh, only the replay markers reject them (409).
3815+
assert.deepStrictEqual(yield* postAll, [409, 409]);
3816+
// Once the markers can be pruned, the time checks reject the proofs by themselves (401).
3817+
yield* TestClock.setTime(
3818+
acceptedAt.epochMilliseconds + Duration.toMillis(REPLAY_MARKER_MAX_AGE),
3819+
);
3820+
assert.deepStrictEqual(yield* postAll, [401, 401]);
3821+
}).pipe(Effect.provide(NodeHttpServer.layerTest)),
3822+
);
3823+
37133824
it.effect(
37143825
"validates cloud proofs against the configured relay issuer, not the transport URL",
37153826
() =>

‎apps/server/src/server.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,7 @@ import * as EventLoopMonitor from "./observability/EventLoopMonitor.ts";
124124
import * as ServerEnvironment from "./environment/ServerEnvironment.ts";
125125
import * as RemoteOpenTargets from "./environment/RemoteOpenTargets.ts";
126126
import { authHttpApiLayer, environmentAuthenticatedAuthLayer } from "./auth/http.ts";
127+
import * as ReplayMarkers from "./auth/replayMarkers.ts";
127128
import * as ServerSecretStore from "./auth/ServerSecretStore.ts";
128129
import * as EnvironmentAuth from "./auth/EnvironmentAuth.ts";
129130
import {
@@ -502,6 +503,7 @@ const AntigravityInstallationRefreshLive = Layer.effectDiscard(
502503

503504
const RuntimeCoreDependenciesLive = ReactorLayerLive.pipe(
504505
Layer.provideMerge(AntigravityInstallationRefreshLive),
506+
Layer.provideMerge(ReplayMarkers.layer),
505507
Layer.provideMerge(ProviderAuthServiceLive),
506508
// Core Services
507509
Layer.provideMerge(ServerSettingsLayerLive),

0 commit comments

Comments
 (0)