Skip to content

Commit 68c7081

Browse files
fix(relay): isolate FCM queue message failures
1 parent b970b65 commit 68c7081

5 files changed

Lines changed: 109 additions & 13 deletions

File tree

‎infra/relay/src/agentActivity/FcmDeliveries.test.ts‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,13 @@ import type { RelayAgentActivityState } from "@t3tools/contracts/relay";
33
import { describe, expect, it } from "@effect/vitest";
44
import * as NodeCryptoLayer from "@effect/platform-node/NodeCrypto";
55
import * as Effect from "effect/Effect";
6+
import * as DateTime from "effect/DateTime";
67
import * as Layer from "effect/Layer";
78
import * as Redacted from "effect/Redacted";
89
import * as Schema from "effect/Schema";
10+
import * as Stream from "effect/Stream";
11+
import type * as Cloudflare from "alchemy/Cloudflare";
12+
import * as FcmDeliveryQueueConsumer from "./FcmDeliveryQueueConsumer.ts";
913

1014
import { RelayConfiguration } from "../Config.ts";
1115
import { RelayDb } from "../db.ts";
@@ -757,3 +761,50 @@ it("stops reducing five-character row fields and fits the remaining alert", () =
757761
expect(data.activity_line_0).toBe("Approval\taaaaa\tbbbbb");
758762
expect(new TextEncoder().encode(encodeJson(data)).length).toBeLessThanOrEqual(3800);
759763
});
764+
765+
describe("FCM queue message isolation", () => {
766+
for (const failure of ["invalid-job", "fcm-rejection"] as const) {
767+
it.effect(`retries only the ${failure} message and delivers the rest of its batch`, () => {
768+
const h = harness();
769+
const outcomes = new Map<string, "ack" | "retry">();
770+
const message = (id: string, body: unknown): Cloudflare.Queues.Message<unknown> => ({
771+
id,
772+
body,
773+
timestamp: DateTime.toDateUtc(DateTime.makeUnsafe(0)),
774+
attempts: 1,
775+
ack: () => {
776+
if (!outcomes.has(id)) outcomes.set(id, "ack");
777+
},
778+
retry: () => {
779+
if (!outcomes.has(id)) outcomes.set(id, "retry");
780+
},
781+
});
782+
const batch = [
783+
message("failed", failure === "invalid-job" ? {} : h.job),
784+
message("healthy", h.job),
785+
];
786+
return Effect.gen(function* () {
787+
yield* Stream.fromIterable(batch).pipe(
788+
Stream.tap((item) =>
789+
Effect.sync(() => {
790+
h.current.deliveryFailure =
791+
item.id === "failed" && failure === "fcm-rejection"
792+
? new FcmClientError({ operation: "send", status: 400 })
793+
: null;
794+
}),
795+
),
796+
Stream.runForEach(FcmDeliveryQueueConsumer.processMessage),
797+
);
798+
// Alchemy acknowledges the batch after a successful stream. Cloudflare
799+
// ignores those acknowledgements for messages explicitly retried earlier.
800+
for (const item of batch) item.ack();
801+
expect([...outcomes]).toEqual([
802+
["failed", "retry"],
803+
["healthy", "ack"],
804+
]);
805+
expect(h.sent).toHaveLength(1);
806+
expect(h.marked).toHaveLength(1);
807+
}).pipe(Effect.provide(h.layer));
808+
});
809+
}
810+
});
Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
import type * as Cloudflare from "alchemy/Cloudflare";
2+
import * as Effect from "effect/Effect";
3+
4+
import * as FcmDeliveries from "./FcmDeliveries.ts";
5+
6+
export const processMessage = Effect.fn("relay.fcm_delivery_queue.process_message")(function* (
7+
message: Cloudflare.Queues.Message<unknown>,
8+
) {
9+
const deliveries = yield* FcmDeliveries.FcmDeliveries;
10+
yield* deliveries.process(message.body).pipe(
11+
Effect.matchEffect({
12+
onFailure: (error) =>
13+
Effect.gen(function* () {
14+
// Decide this message's outcome before Alchemy acknowledges the batch.
15+
// Cloudflare keeps the first ack/retry decision for each message.
16+
message.retry();
17+
yield* Effect.logWarning("FCM queue delivery failed; retrying message", {
18+
messageId: message.id,
19+
errorTag: error._tag,
20+
});
21+
}),
22+
onSuccess: () => Effect.sync(() => message.ack()),
23+
}),
24+
);
25+
});

‎infra/relay/src/http/Api.test.ts‎

Lines changed: 28 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ import * as HttpRouter from "effect/unstable/http/HttpRouter";
1616
import * as HttpServerRequest from "effect/unstable/http/HttpServerRequest";
1717
import * as HttpServerResponse from "effect/unstable/http/HttpServerResponse";
1818
import * as HttpServer from "effect/unstable/http/HttpServer";
19-
import * as HttpApiTest from "effect/unstable/httpapi/HttpApiTest";
19+
import * as HttpApiBuilder from "effect/unstable/httpapi/HttpApiBuilder";
2020
import * as HttpApi from "effect/unstable/httpapi/HttpApi";
2121
import { EnvironmentId } from "@t3tools/contracts";
2222
import {
@@ -136,14 +136,34 @@ describe("device listing compatibility", () => {
136136
),
137137
);
138138
return Effect.gen(function* () {
139-
const client = yield* HttpApiTest.groups(
140-
HttpApi.make("RelayApi").add(RelayApi.groups.client),
141-
["client"],
139+
const app = yield* Effect.acquireRelease(
140+
Effect.sync(() =>
141+
HttpRouter.toWebHandler(
142+
HttpApiBuilder.layer(HttpApi.make("RelayApi").add(RelayApi.groups.client)).pipe(
143+
Layer.provide(handlers),
144+
Layer.provide(HttpServer.layerServices),
145+
),
146+
{ disableLogger: true },
147+
),
148+
),
149+
(app) => Effect.promise(() => app.dispose()),
142150
);
143-
const request = { headers: { authorization: "Bearer test-token" } };
144-
expect(yield* client.client.listDevices(request)).toEqual({ devices: [iphone] });
145-
expect(yield* client.client.listDevicesV2(request)).toEqual({ devices: [iphone, android] });
146-
}).pipe(Effect.provide(Layer.mergeAll(handlers, HttpServer.layerServices)), Effect.scoped);
151+
for (const [version, devices] of [
152+
["v1", [iphone]],
153+
["v2", [iphone, android]],
154+
] as const) {
155+
const response = yield* Effect.promise(() =>
156+
app.handler(
157+
new Request(`https://relay.example.test/${version}/client/devices`, {
158+
headers: { authorization: "Bearer test-token" },
159+
}),
160+
),
161+
);
162+
expect(response.status).toBe(200);
163+
expect(response.headers.get("cache-control")).toBe("no-store");
164+
expect(yield* Effect.promise(() => response.json())).toEqual({ devices });
165+
}
166+
}).pipe(Effect.scoped);
147167
});
148168
});
149169

‎infra/relay/src/http/Api.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -545,6 +545,7 @@ export const clientApi = HttpApiBuilder.group(
545545
.handle(
546546
"listDevices",
547547
Effect.fn("relay.api.client.listDevices")(function* () {
548+
yield* appendRelayCredentialResponseHeaders;
548549
const { userId } = yield* RelayClientPrincipal;
549550
const registered = yield* devices.listForUser({ userId });
550551
return {
@@ -559,6 +560,7 @@ export const clientApi = HttpApiBuilder.group(
559560
.handle(
560561
"listDevicesV2",
561562
Effect.fn("relay.api.client.listDevicesV2")(function* () {
563+
yield* appendRelayCredentialResponseHeaders;
562564
const { userId } = yield* RelayClientPrincipal;
563565
return { devices: yield* devices.listForUser({ userId }) };
564566
}, mapRelayCommonApiErrors("not_authorized")),

‎infra/relay/src/worker.ts‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@ import * as FcmAssertionSigner from "./agentActivity/FcmAssertionSigner.ts";
5757
import * as FcmClient from "./agentActivity/FcmClient.ts";
5858
import * as FcmDeliveryQueueSender from "./agentActivity/FcmDeliveryQueueSender.ts";
5959
import * as FcmDeliveries from "./agentActivity/FcmDeliveries.ts";
60+
import * as FcmDeliveryQueueConsumer from "./agentActivity/FcmDeliveryQueueConsumer.ts";
6061
import * as RelayConfiguration from "./Config.ts";
6162
import * as AgentActivityPublisher from "./agentActivity/AgentActivityPublisher.ts";
6263
import * as ApnsClient from "./agentActivity/ApnsClient.ts";
@@ -308,11 +309,8 @@ export const ApiLive = Api.make(
308309
},
309310
(stream) =>
310311
stream.pipe(
311-
Stream.runForEach((message) =>
312-
FcmDeliveries.FcmDeliveries.pipe(
313-
Effect.flatMap((deliveries) => deliveries.process(message.body)),
314-
),
315-
),
312+
Stream.withSpan("relay.fcm_delivery_queue.process_batch"),
313+
Stream.runForEach(FcmDeliveryQueueConsumer.processMessage),
316314
Effect.provide(runtimeLayer),
317315
),
318316
);

0 commit comments

Comments
 (0)