Skip to content

Commit e591dcf

Browse files
happydog-botsteipeteHappy Dog Support Bot
authored
fix(voice-call): persist complete realtime transcripts (#84161)
Co-authored-by: Peter Steinberger <[email protected]> Co-authored-by: Happy Dog Support Bot <[email protected]>
1 parent e94fde2 commit e591dcf

9 files changed

Lines changed: 500 additions & 43 deletions

File tree

‎extensions/google/realtime-voice-provider.test.ts‎

Lines changed: 269 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -592,22 +592,56 @@ describe("buildGoogleRealtimeVoiceProvider", () => {
592592
expect(lastConnectParams().config).not.toHaveProperty("sessionResumption");
593593
});
594594

595-
it("captures Google Live resumption handles and reuses them on reconnect", async () => {
595+
it("preserves transcript fragments while reusing a resumption handle", async () => {
596596
const provider = buildGoogleRealtimeVoiceProvider();
597+
const onTranscript = vi.fn();
597598
const bridge = provider.createBridge({
598599
providerConfig: { apiKey: "gemini-key" },
599600
onAudio: vi.fn(),
600601
onClearAudio: vi.fn(),
602+
onTranscript,
601603
});
602604

603605
await bridge.connect();
604606
lastConnectParams().callbacks.onmessage({
605607
sessionResumptionUpdate: { resumable: true, newHandle: "resume-1" },
608+
serverContent: { inputTranscription: { text: "Before " } },
606609
});
607610

608611
await bridge.connect();
612+
lastConnectParams().callbacks.onmessage({
613+
serverContent: { inputTranscription: { text: "after", finished: true } },
614+
});
609615

610616
expect(lastConnectParams().config.sessionResumption).toEqual({ handle: "resume-1" });
617+
expect(onTranscript.mock.calls.filter((call) => call[2] === true)).toEqual([
618+
["user", "Before after", true],
619+
]);
620+
});
621+
622+
it("drops unfinished hypotheses when a new session has no continuity", async () => {
623+
const provider = buildGoogleRealtimeVoiceProvider();
624+
const onTranscript = vi.fn();
625+
const bridge = provider.createBridge({
626+
providerConfig: { apiKey: "gemini-key", sessionResumption: false },
627+
onAudio: vi.fn(),
628+
onClearAudio: vi.fn(),
629+
onTranscript,
630+
});
631+
632+
await bridge.connect();
633+
lastConnectParams().callbacks.onmessage({
634+
serverContent: { inputTranscription: { text: "Old fragment " } },
635+
});
636+
637+
await bridge.connect();
638+
lastConnectParams().callbacks.onmessage({
639+
serverContent: { inputTranscription: { text: "New turn", finished: true } },
640+
});
641+
642+
expect(onTranscript.mock.calls.filter((call) => call[2] === true)).toEqual([
643+
["user", "New turn", true],
644+
]);
611645
});
612646

613647
it("reconnects unexpected Google Live closes with the latest resumption handle", async () => {
@@ -629,6 +663,9 @@ describe("buildGoogleRealtimeVoiceProvider", () => {
629663
setupComplete: { sessionId: "session-1" },
630664
sessionResumptionUpdate: { resumable: true, newHandle: "resume-1" },
631665
});
666+
lastConnectParams().callbacks.onmessage({
667+
sessionResumptionUpdate: { resumable: false },
668+
});
632669
lastConnectParams().callbacks.onclose({
633670
code: 1011,
634671
reason: "temporary upstream close",
@@ -648,6 +685,106 @@ describe("buildGoogleRealtimeVoiceProvider", () => {
648685
}
649686
});
650687

688+
it("does not finalize or merge a hypothesis across a fresh automatic reconnect", async () => {
689+
vi.useFakeTimers();
690+
const provider = buildGoogleRealtimeVoiceProvider();
691+
const onTranscript = vi.fn();
692+
const bridge = provider.createBridge({
693+
providerConfig: { apiKey: "gemini-key" },
694+
onAudio: vi.fn(),
695+
onClearAudio: vi.fn(),
696+
onTranscript,
697+
});
698+
699+
await bridge.connect();
700+
const firstSession = lastConnectParams().callbacks;
701+
firstSession.onmessage({
702+
setupComplete: {},
703+
serverContent: { inputTranscription: { text: "Interrupted hypothesis" } },
704+
});
705+
firstSession.onclose({ code: 1011, reason: "temporary" });
706+
707+
expect(onTranscript.mock.calls.filter((call) => call[2] === true)).toEqual([]);
708+
await vi.advanceTimersByTimeAsync(250);
709+
lastConnectParams().callbacks.onmessage({
710+
setupComplete: {},
711+
serverContent: { inputTranscription: { text: "New utterance", finished: true } },
712+
});
713+
714+
expect(onTranscript.mock.calls.filter((call) => call[2] === true)).toEqual([
715+
["user", "New utterance", true],
716+
]);
717+
});
718+
719+
it("keeps transcript fragments pending across a resumable reconnect", async () => {
720+
vi.useFakeTimers();
721+
const provider = buildGoogleRealtimeVoiceProvider();
722+
const onTranscript = vi.fn();
723+
const bridge = provider.createBridge({
724+
providerConfig: { apiKey: "gemini-key" },
725+
onAudio: vi.fn(),
726+
onClearAudio: vi.fn(),
727+
onTranscript,
728+
});
729+
730+
await bridge.connect();
731+
const firstSession = lastConnectParams().callbacks;
732+
firstSession.onmessage({
733+
setupComplete: {},
734+
sessionResumptionUpdate: { resumable: true, newHandle: "resume-1" },
735+
serverContent: {
736+
outputTranscription: { text: "Before " },
737+
turnComplete: true,
738+
},
739+
});
740+
await vi.advanceTimersByTimeAsync(500);
741+
firstSession.onclose({ code: 1011, reason: "temporary" });
742+
await vi.advanceTimersByTimeAsync(1_000);
743+
744+
expect(onTranscript.mock.calls).toEqual([["assistant", "Before ", false]]);
745+
746+
lastConnectParams().callbacks.onmessage({
747+
setupComplete: {},
748+
serverContent: { outputTranscription: { text: "after", finished: true } },
749+
});
750+
expect(onTranscript.mock.calls.at(-1)).toEqual(["assistant", "Before after", true]);
751+
});
752+
753+
it("flushes pending transcripts before closing after reconnect failures", async () => {
754+
vi.useFakeTimers();
755+
const provider = buildGoogleRealtimeVoiceProvider();
756+
const onClose = vi.fn();
757+
const onTranscript = vi.fn();
758+
const bridge = provider.createBridge({
759+
providerConfig: { apiKey: "gemini-key" },
760+
onAudio: vi.fn(),
761+
onClearAudio: vi.fn(),
762+
onClose,
763+
onTranscript,
764+
});
765+
766+
await bridge.connect();
767+
const firstSession = lastConnectParams().callbacks;
768+
firstSession.onmessage({
769+
setupComplete: {},
770+
sessionResumptionUpdate: { resumable: true, newHandle: "resume-1" },
771+
serverContent: { inputTranscription: { text: "Last words" } },
772+
});
773+
connectMock
774+
.mockRejectedValueOnce(new Error("connect failed 1"))
775+
.mockRejectedValueOnce(new Error("connect failed 2"))
776+
.mockRejectedValueOnce(new Error("connect failed 3"));
777+
firstSession.onclose({ code: 1011, reason: "temporary" });
778+
779+
await vi.advanceTimersByTimeAsync(1_750);
780+
781+
expect(onTranscript.mock.calls.at(-1)).toEqual(["user", "Last words", true]);
782+
expect(onClose).toHaveBeenCalledWith("error");
783+
expect(onTranscript.mock.invocationCallOrder.at(-1)).toBeLessThan(
784+
onClose.mock.invocationCallOrder[0] ?? Number.MAX_SAFE_INTEGER,
785+
);
786+
});
787+
651788
it("waits for setup completion before draining audio and firing ready", async () => {
652789
const provider = buildGoogleRealtimeVoiceProvider();
653790
const onReady = vi.fn();
@@ -855,7 +992,7 @@ describe("buildGoogleRealtimeVoiceProvider", () => {
855992
expect(requireFirstAudio(onAudio)).toEqual(pcm24k);
856993
});
857994

858-
it("does not forward Google thought text as assistant transcript", async () => {
995+
it("uses official output transcription instead of model-turn text", async () => {
859996
const provider = buildGoogleRealtimeVoiceProvider();
860997
const onTranscript = vi.fn();
861998
const bridge = provider.createBridge({
@@ -870,14 +1007,143 @@ describe("buildGoogleRealtimeVoiceProvider", () => {
8701007
setupComplete: {},
8711008
serverContent: {
8721009
modelTurn: {
873-
parts: [{ text: "internal reasoning", thought: true }],
1010+
parts: [
1011+
{ text: "internal reasoning", thought: true },
1012+
{ text: "uncorrelated model text" },
1013+
],
8741014
},
8751015
},
8761016
});
8771017

8781018
expect(onTranscript).not.toHaveBeenCalled();
8791019
});
8801020

1021+
it("emits one complete transcript after Google marks a transcription finished", async () => {
1022+
vi.useFakeTimers();
1023+
const provider = buildGoogleRealtimeVoiceProvider();
1024+
const onTranscript = vi.fn();
1025+
const bridge = provider.createBridge({
1026+
providerConfig: { apiKey: "gemini-key" },
1027+
onAudio: vi.fn(),
1028+
onClearAudio: vi.fn(),
1029+
onTranscript,
1030+
});
1031+
1032+
await bridge.connect();
1033+
const onmessage = lastConnectParams().callbacks.onmessage;
1034+
onmessage({ serverContent: { outputTranscription: { text: "Hi, " } } });
1035+
onmessage({
1036+
serverContent: { outputTranscription: { text: "how can I help?", finished: true } },
1037+
});
1038+
onmessage({ serverContent: { modelTurn: { parts: [{ text: "ignored fallback" }] } } });
1039+
onmessage({ serverContent: { turnComplete: true } });
1040+
1041+
expect(onTranscript.mock.calls).toEqual([
1042+
["assistant", "Hi, ", false],
1043+
["assistant", "how can I help?", false],
1044+
["assistant", "Hi, how can I help?", true],
1045+
]);
1046+
});
1047+
1048+
it("honors a finish-only transcription message", async () => {
1049+
const provider = buildGoogleRealtimeVoiceProvider();
1050+
const onTranscript = vi.fn();
1051+
const bridge = provider.createBridge({
1052+
providerConfig: { apiKey: "gemini-key" },
1053+
onAudio: vi.fn(),
1054+
onClearAudio: vi.fn(),
1055+
onTranscript,
1056+
});
1057+
1058+
await bridge.connect();
1059+
const onmessage = lastConnectParams().callbacks.onmessage;
1060+
onmessage({ serverContent: { inputTranscription: { text: "Last words" } } });
1061+
onmessage({ serverContent: { inputTranscription: { finished: true } } });
1062+
1063+
expect(onTranscript.mock.calls).toEqual([
1064+
["user", "Last words", false],
1065+
["user", "Last words", true],
1066+
]);
1067+
});
1068+
1069+
it("retains unordered transcript chunks until a protocol terminal or close", async () => {
1070+
const provider = buildGoogleRealtimeVoiceProvider();
1071+
const onTranscript = vi.fn();
1072+
const bridge = provider.createBridge({
1073+
providerConfig: { apiKey: "gemini-key" },
1074+
onAudio: vi.fn(),
1075+
onClearAudio: vi.fn(),
1076+
onTranscript,
1077+
});
1078+
1079+
await bridge.connect();
1080+
const onmessage = lastConnectParams().callbacks.onmessage;
1081+
onmessage({ serverContent: { inputTranscription: { text: "Earlier question. " } } });
1082+
onmessage({ serverContent: { outputTranscription: { text: "Interrupted response " } } });
1083+
onmessage({ serverContent: { interrupted: true } });
1084+
onmessage({ serverContent: { turnComplete: true } });
1085+
onmessage({
1086+
serverContent: {
1087+
inputTranscription: { text: "New question" },
1088+
outputTranscription: { text: "ending" },
1089+
turnComplete: true,
1090+
},
1091+
});
1092+
1093+
expect(onTranscript.mock.calls.filter((call) => call[2] === true)).toEqual([]);
1094+
1095+
bridge.close();
1096+
expect(onTranscript.mock.calls.filter((call) => call[2] === true)).toEqual([
1097+
["user", "Earlier question. New question", true],
1098+
["assistant", "Interrupted response ending", true],
1099+
]);
1100+
});
1101+
1102+
it("flushes pending transcripts when the bridge closes", async () => {
1103+
const provider = buildGoogleRealtimeVoiceProvider();
1104+
const onTranscript = vi.fn();
1105+
const bridge = provider.createBridge({
1106+
providerConfig: { apiKey: "gemini-key" },
1107+
onAudio: vi.fn(),
1108+
onClearAudio: vi.fn(),
1109+
onTranscript,
1110+
});
1111+
1112+
await bridge.connect();
1113+
lastConnectParams().callbacks.onmessage({
1114+
serverContent: { inputTranscription: { text: "Last words" } },
1115+
});
1116+
bridge.close();
1117+
1118+
expect(onTranscript.mock.calls.at(-1)).toEqual(["user", "Last words", true]);
1119+
});
1120+
1121+
it("closes the Live session when the final transcript callback throws", async () => {
1122+
const provider = buildGoogleRealtimeVoiceProvider();
1123+
const callbackError = new Error("transcript persistence failed");
1124+
const onError = vi.fn();
1125+
const bridge = provider.createBridge({
1126+
providerConfig: { apiKey: "gemini-key" },
1127+
onAudio: vi.fn(),
1128+
onClearAudio: vi.fn(),
1129+
onError,
1130+
onTranscript: vi.fn((_role, _text, isFinal) => {
1131+
if (isFinal) {
1132+
throw callbackError;
1133+
}
1134+
}),
1135+
});
1136+
1137+
await bridge.connect();
1138+
lastConnectParams().callbacks.onmessage({
1139+
serverContent: { inputTranscription: { text: "Last words" } },
1140+
});
1141+
1142+
expect(() => bridge.close()).not.toThrow();
1143+
expect(onError).toHaveBeenCalledWith(callbackError);
1144+
expect(session.close).toHaveBeenCalledTimes(1);
1145+
});
1146+
8811147
it("reports provider-confirmed input interruption as barge-in", async () => {
8821148
const provider = buildGoogleRealtimeVoiceProvider();
8831149
const onClearAudio = vi.fn();

0 commit comments

Comments
 (0)