Skip to content

Commit f92847a

Browse files
fix(server): release stalled speech model preparations
1 parent 91773c0 commit f92847a

2 files changed

Lines changed: 74 additions & 26 deletions

File tree

‎apps/server/src/speech/SpeechService.test.ts‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,47 @@ it.effect("prepares a batch model before audio arrives and reuses it for transcr
148148
}).pipe(Effect.provide(layer)),
149149
);
150150

151+
it.effect.each(["cancellation", "timeout"] as const)(
152+
"releases a stalled model preparation on %s",
153+
(reason) =>
154+
Effect.gen(function* () {
155+
const speech = yield* SpeechService.SpeechService;
156+
const started = Promise.withResolvers<AbortSignal>();
157+
const stalled = Promise.withResolvers<typeof native>();
158+
const disposed = Promise.withResolvers<void>();
159+
const abandoned = { ...native, dispose: vi.fn(() => disposed.resolve()) };
160+
loadNative.mockImplementationOnce((_path, signal) => {
161+
started.resolve(signal);
162+
return stalled.promise;
163+
});
164+
const request = yield* speech.prepareModel.pipe(Effect.forkChild);
165+
const signal = yield* Effect.promise(() => started.promise);
166+
try {
167+
if (reason === "cancellation") {
168+
yield* Fiber.interrupt(request).pipe(Effect.forkChild({ startImmediately: true }));
169+
}
170+
yield* TestClock.adjust(reason === "timeout" ? "2 minutes" : 0);
171+
expect(signal.aborted).toBe(true);
172+
const exit = yield* Fiber.await(request);
173+
expect(Exit.isFailure(exit)).toBe(true);
174+
if (reason === "timeout") {
175+
expect(Exit.findErrorOption(exit)).toMatchObject({
176+
value: { _tag: "SpeechOperationError", operation: "model preparation" },
177+
});
178+
}
179+
yield* speech.selectModel("fallback-model");
180+
const transcribe = speech.transcribe(pcm(), DEFAULT_SPEECH_TRANSCRIPTION_OPTIONS);
181+
expect(yield* transcribe).toBe("hello");
182+
stalled.resolve(abandoned);
183+
yield* Effect.promise(() => disposed.promise);
184+
expect(yield* transcribe).toBe("hello");
185+
expect(loadNative).toHaveBeenCalledTimes(2);
186+
} finally {
187+
stalled.resolve(abandoned);
188+
}
189+
}).pipe(Effect.provide(layer)),
190+
);
191+
151192
it.effect("rejects model mutations outside the offered registry", () =>
152193
Effect.gen(function* () {
153194
const speech = yield* SpeechService.SpeechService;

‎apps/server/src/speech/SpeechService.ts‎

Lines changed: 33 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -197,7 +197,7 @@ const make = Effect.gen(function* () {
197197
let activeOperation: Promise<unknown> | undefined;
198198
const operationBusy = () => Boolean(activeOperation || downloading);
199199
let closing = false;
200-
// @effect-diagnostics-next-line abortControllerInEffect:off - Native model work outlives cancelled requests and aborts on service shutdown.
200+
// @effect-diagnostics-next-line abortControllerInEffect:off - Cached native workers must also stop on service shutdown.
201201
const lifetime = new AbortController();
202202
let unloadTimeout: SpeechModelUnloadTimeout = "min_15";
203203

@@ -218,7 +218,6 @@ const make = Effect.gen(function* () {
218218
model = undefined;
219219
loadedModelId = undefined;
220220
loadedAcceleration = undefined;
221-
loading = undefined;
222221
};
223222
const disposeModel = async () => {
224223
const target = model;
@@ -342,8 +341,9 @@ const make = Effect.gen(function* () {
342341
}
343342
if (loading) return loading;
344343
await disposeModel();
345-
const pending = loadNativeSpeechModel(modelPath, signal, undefined, acceleration)
346-
.then(async (loaded) => {
344+
signal.throwIfAborted();
345+
const pending = loadNativeSpeechModel(modelPath, signal, undefined, acceleration).then(
346+
async (loaded) => {
347347
if (closing || signal.aborted) {
348348
await loaded.dispose();
349349
signal.throwIfAborted();
@@ -352,22 +352,27 @@ const make = Effect.gen(function* () {
352352
model = loaded;
353353
loadedModelId = definition.id;
354354
loadedAcceleration = acceleration;
355-
loading = undefined;
356355
scheduleUnload();
357356
return loaded;
358-
})
359-
.catch((error: unknown) => {
360-
loading = undefined;
361-
throw error;
362-
});
357+
},
358+
);
359+
const release = () => {
360+
if (loading === pending) loading = undefined;
361+
};
363362
loading = pending;
364-
return pending;
363+
signal.addEventListener("abort", release, { once: true });
364+
try {
365+
return await pending;
366+
} finally {
367+
signal.removeEventListener("abort", release);
368+
release();
369+
}
365370
};
366371

367372
const speechError = (operation: string, cause: unknown): SpeechError =>
368373
isSpeechError(cause) ? cause : new SpeechOperationError({ operation, cause });
369374

370-
const attempt = <A>(operation: string, run: () => Promise<A>) =>
375+
const attempt = <A>(operation: string, run: (signal: AbortSignal) => Promise<A>) =>
371376
Effect.tryPromise({ try: run, catch: (cause) => speechError(operation, cause) });
372377

373378
// The busy slot is released by a finalizer, so interrupting the fiber frees it at
@@ -503,20 +508,21 @@ const make = Effect.gen(function* () {
503508
exclusive(
504509
"model preparation",
505510
requireSupported.pipe(
506-
Effect.andThen(
507-
// Keep ownership until the native load settles, even if its request is cancelled.
508-
installedModelPath(selectedModel(settings)).pipe(
509-
Effect.flatMap((modelPath) =>
510-
attempt("model preparation", () =>
511-
loadModel(
512-
modelPath,
513-
selectedModel(settings),
514-
lifetime.signal,
515-
settings.speechAcceleration,
516-
),
517-
),
511+
Effect.andThen(installedModelPath(selectedModel(settings))),
512+
Effect.flatMap((modelPath) =>
513+
attempt("model preparation", (signal) =>
514+
loadModel(
515+
modelPath,
516+
selectedModel(settings),
517+
AbortSignal.any([signal, lifetime.signal]),
518+
settings.speechAcceleration,
518519
),
519-
Effect.uninterruptible,
520+
).pipe(
521+
Effect.timeout("2 minutes"),
522+
Effect.catchTags({
523+
TimeoutError: (cause) =>
524+
Effect.fail(new SpeechOperationError({ operation: "model preparation", cause })),
525+
}),
520526
),
521527
),
522528
),
@@ -643,8 +649,9 @@ const make = Effect.gen(function* () {
643649
closed = true;
644650
if (!finished) {
645651
const dispose = Effect.promise(async () => {
652+
const pending = loading;
646653
controller.abort();
647-
await loading?.catch(() => undefined);
654+
await pending?.catch(() => undefined);
648655
if (loaded) {
649656
forgetModel(loaded);
650657
await loaded.dispose();

0 commit comments

Comments
 (0)