Skip to content

Commit 606400e

Browse files
committed
fix: keep dictation compatible with current composer and checks
1 parent 4d59059 commit 606400e

21 files changed

Lines changed: 276 additions & 382 deletions

‎apps/desktop/src/window/MainWindowPermissions.ts‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,6 @@
11
import * as Electron from "electron";
22

3-
/**
4-
* Permission policy for the main window's session. A session without a
5-
* handler grants every request, and this keeps that. The one addition is
6-
* microphone capture from the main window on macOS: it asks the OS for
7-
* consent first, so dictation gets the system prompt instead of silence
8-
* when the app has not been allowed to record yet.
9-
*/
3+
// Ask macOS for microphone consent before granting main-window capture; preserve other grants.
104
export function installMainWindowPermissions(
115
webContents: Electron.WebContents,
126
platform: NodeJS.Platform,

‎apps/server/src/dictation/Dictation.integration.test.ts‎

Lines changed: 131 additions & 141 deletions
Original file line numberDiff line numberDiff line change
@@ -27,152 +27,142 @@ const model = candidate;
2727
const home = process.env.T3_DICTATION_E2E_HOME;
2828

2929
/** Run explicitly with the spike model directory. Runtime + tokens use real HTTP; ONNX bytes stream from the verified cache. */
30-
it.live.skipIf(!fixture)(
31-
"installs and transcribes through the real native runtime",
32-
() =>
33-
Effect.gen(function* () {
34-
const fs = yield* FileSystem.FileSystem;
35-
const path = yield* Path.Path;
36-
const network = yield* HttpClient.HttpClient;
37-
const http = HttpClient.make((request) => {
38-
const name = request.url.split("/").at(-1)!;
39-
const file =
40-
model === "parakeet-v3"
41-
? models[model].files.find(
42-
(candidate) => candidate.name === name && name.endsWith(".onnx"),
43-
)
44-
: undefined;
45-
if (!file) return network.execute(request);
46-
return Stream.toReadableStreamEffect(fs.stream(path.join(fixture!, name))).pipe(
47-
Effect.map((stream) =>
48-
HttpClientResponse.fromWeb(
49-
request,
50-
new Response(stream, { headers: { "content-length": String(file.bytes) } }),
51-
),
30+
it.live.skipIf(!fixture)("installs and transcribes through the real native runtime", () =>
31+
Effect.gen(function* () {
32+
const fs = yield* FileSystem.FileSystem;
33+
const path = yield* Path.Path;
34+
const network = yield* HttpClient.HttpClient;
35+
const http = HttpClient.make((request) => {
36+
const name = request.url.split("/").at(-1)!;
37+
const file =
38+
model === "parakeet-v3"
39+
? models[model].files.find(
40+
(candidate) => candidate.name === name && name.endsWith(".onnx"),
41+
)
42+
: undefined;
43+
if (!file) return network.execute(request);
44+
return Stream.toReadableStreamEffect(fs.stream(path.join(fixture!, name))).pipe(
45+
Effect.map((stream) =>
46+
HttpClientResponse.fromWeb(
47+
request,
48+
new Response(stream, { headers: { "content-length": String(file.bytes) } }),
5249
),
53-
Effect.orDie,
54-
);
55-
});
56-
const artifactsLayer = DictationArtifacts.layer.pipe(
57-
Layer.provide(Layer.succeed(HttpClient.HttpClient, http)),
58-
);
59-
const layer = Dictation.layer.pipe(
60-
Layer.provide(DictationRecognizer.layer.pipe(Layer.provide(artifactsLayer))),
61-
Layer.provideMerge(artifactsLayer),
62-
Layer.provideMerge(
63-
ServerSettings.layerTest({ dictation: { engine: "local", localModel: model } }),
64-
),
65-
Layer.provide(
66-
ServerConfig.layerTest(process.cwd(), home ?? { prefix: "t3-dictation-e2e-" }),
6750
),
51+
Effect.orDie,
6852
);
69-
yield* Effect.gen(function* () {
70-
const service = yield* Dictation.Dictation;
71-
const settings = yield* ServerSettings.ServerSettingsService;
72-
yield* service.stateStream.pipe(
73-
Stream.filter((state) => state.ready),
74-
Stream.take(1),
75-
Stream.runDrain,
76-
);
77-
yield* service.install({ model });
78-
assert.strictEqual((yield* service.state).ready, true);
79-
const wav = Buffer.from(yield* fs.readFile(path.join(fixture!, "test_wavs", "en.wav")));
80-
assert.strictEqual(wav.toString("ascii", 0, 4), "RIFF");
81-
let sampleRate = 0;
82-
let channels = 0;
83-
let bits = 0;
84-
let audio = Buffer.alloc(0);
85-
for (let offset = 12; offset + 8 <= wav.length;) {
86-
const name = wav.toString("ascii", offset, offset + 4);
87-
const size = wav.readUInt32LE(offset + 4);
88-
if (name === "fmt ") {
89-
assert.strictEqual(wav.readUInt16LE(offset + 8), 1);
90-
channels = wav.readUInt16LE(offset + 10);
91-
sampleRate = wav.readUInt32LE(offset + 12);
92-
bits = wav.readUInt16LE(offset + 22);
93-
}
94-
if (name === "data") audio = wav.subarray(offset + 8, offset + 8 + size);
95-
offset += 8 + size + (size % 2);
53+
});
54+
const artifactsLayer = DictationArtifacts.layer.pipe(
55+
Layer.provide(Layer.succeed(HttpClient.HttpClient, http)),
56+
);
57+
const layer = Dictation.layer.pipe(
58+
Layer.provide(DictationRecognizer.layer.pipe(Layer.provide(artifactsLayer))),
59+
Layer.provideMerge(artifactsLayer),
60+
Layer.provideMerge(
61+
ServerSettings.layerTest({ dictation: { engine: "local", localModel: model } }),
62+
),
63+
Layer.provide(ServerConfig.layerTest(process.cwd(), home ?? { prefix: "t3-dictation-e2e-" })),
64+
);
65+
yield* Effect.gen(function* () {
66+
const service = yield* Dictation.Dictation;
67+
const settings = yield* ServerSettings.ServerSettingsService;
68+
yield* service.stateStream.pipe(
69+
Stream.filter((state) => state.ready),
70+
Stream.take(1),
71+
Stream.runDrain,
72+
);
73+
yield* service.install({ model });
74+
assert.strictEqual((yield* service.state).ready, true);
75+
const wav = Buffer.from(yield* fs.readFile(path.join(fixture!, "test_wavs", "en.wav")));
76+
assert.strictEqual(wav.toString("ascii", 0, 4), "RIFF");
77+
let sampleRate = 0;
78+
let channels = 0;
79+
let bits = 0;
80+
let audio = Buffer.alloc(0);
81+
for (let offset = 12; offset + 8 <= wav.length;) {
82+
const name = wav.toString("ascii", offset, offset + 4);
83+
const size = wav.readUInt32LE(offset + 4);
84+
if (name === "fmt ") {
85+
assert.strictEqual(wav.readUInt16LE(offset + 8), 1);
86+
channels = wav.readUInt16LE(offset + 10);
87+
sampleRate = wav.readUInt32LE(offset + 12);
88+
bits = wav.readUInt16LE(offset + 22);
9689
}
97-
assert.strictEqual(channels, 1);
98-
assert.strictEqual(bits, 16);
99-
if (sampleRate !== 16000) {
100-
const converted = Buffer.alloc(Math.floor(((audio.length / 2) * 16000) / sampleRate) * 2);
101-
for (let i = 0; i < converted.length / 2; i++) {
102-
const position = (i * sampleRate) / 16000;
103-
const left = Math.floor(position);
104-
const right = Math.min(left + 1, audio.length / 2 - 1);
105-
converted.writeInt16LE(
106-
Math.round(
107-
audio.readInt16LE(left * 2) * (1 - (position - left)) +
108-
audio.readInt16LE(right * 2) * (position - left),
109-
),
110-
i * 2,
111-
);
112-
}
113-
audio = converted;
90+
if (name === "data") audio = wav.subarray(offset + 8, offset + 8 + size);
91+
offset += 8 + size + (size % 2);
92+
}
93+
assert.strictEqual(channels, 1);
94+
assert.strictEqual(bits, 16);
95+
if (sampleRate !== 16000) {
96+
const converted = Buffer.alloc(Math.floor(((audio.length / 2) * 16000) / sampleRate) * 2);
97+
for (let i = 0; i < converted.length / 2; i++) {
98+
const position = (i * sampleRate) / 16000;
99+
const left = Math.floor(position);
100+
const right = Math.min(left + 1, audio.length / 2 - 1);
101+
converted.writeInt16LE(
102+
Math.round(
103+
audio.readInt16LE(left * 2) * (1 - (position - left)) +
104+
audio.readInt16LE(right * 2) * (position - left),
105+
),
106+
i * 2,
107+
);
114108
}
115-
const runs: {
116-
text: string;
117-
locale: string;
118-
finishMs: number;
119-
recordingSeconds: number;
120-
startMs: number;
121-
}[] = [];
122-
for (let run = 0; run < (model === "whisper-turbo" ? 4 : 3); run++) {
123-
if (model === "whisper-turbo" && run === 1)
124-
yield* settings.updateSettings({ dictation: { language: "en" } });
125-
if (model === "whisper-turbo" && run === 3)
126-
yield* settings.updateSettings({ dictation: { language: "nl" } });
127-
const recording =
128-
run !== 2
129-
? audio
130-
: Buffer.concat(
131-
Array.from({ length: 4 }, () => Buffer.concat([audio, Buffer.alloc(12800)])),
132-
);
133-
const beforeStart = yield* Clock.currentTimeMillis;
134-
const { sessionId } = yield* service.start;
135-
const startMs = (yield* Clock.currentTimeMillis) - beforeStart;
136-
let chunkCount = 0;
137-
for (let offset = 0; offset < recording.length; offset += 3200) {
138-
yield* service.append({
139-
sessionId,
140-
seq: chunkCount++,
141-
pcm: recording.subarray(offset, offset + 3200).toString("base64"),
142-
});
143-
// Pace the recording as a microphone would, so background decoding overlaps speech.
144-
yield* Effect.sleep("100 millis");
145-
}
146-
const beforeFinish = yield* Clock.currentTimeMillis;
147-
const result = yield* service.finish({ sessionId, chunkCount });
148-
runs.push({
149-
text: result.text,
150-
locale: result.locale,
151-
finishMs: (yield* Clock.currentTimeMillis) - beforeFinish,
152-
recordingSeconds: recording.length / 32000,
153-
startMs,
109+
audio = converted;
110+
}
111+
const runs: {
112+
text: string;
113+
locale: string;
114+
finishMs: number;
115+
recordingSeconds: number;
116+
startMs: number;
117+
}[] = [];
118+
for (let run = 0; run < (model === "whisper-turbo" ? 4 : 3); run++) {
119+
if (model === "whisper-turbo" && run === 1)
120+
yield* settings.updateSettings({ dictation: { language: "en" } });
121+
if (model === "whisper-turbo" && run === 3)
122+
yield* settings.updateSettings({ dictation: { language: "nl" } });
123+
const recording =
124+
run !== 2
125+
? audio
126+
: Buffer.concat(
127+
Array.from({ length: 4 }, () => Buffer.concat([audio, Buffer.alloc(12800)])),
128+
);
129+
const beforeStart = yield* Clock.currentTimeMillis;
130+
const { sessionId } = yield* service.start;
131+
const startMs = (yield* Clock.currentTimeMillis) - beforeStart;
132+
let chunkCount = 0;
133+
for (let offset = 0; offset < recording.length; offset += 3200) {
134+
yield* service.append({
135+
sessionId,
136+
seq: chunkCount++,
137+
pcm: recording.subarray(offset, offset + 3200).toString("base64"),
154138
});
155-
assert.isTrue(result.text.length > 0);
156-
if (model === "whisper-turbo" && run === 3) assert.strictEqual(result.locale, "nl");
157139
}
158-
// This opt-in test reports the public fixture transcript; the product service never logs recordings.
159-
process.stdout.write(
160-
JSON.stringify(
161-
{
162-
model,
163-
availableParallelism: NodeOS.availableParallelism(),
164-
numThreads: Math.min(8, Math.max(1, NodeOS.availableParallelism())),
165-
runs,
166-
rssMB: Math.round(process.memoryUsage().rss / 1e6),
167-
},
168-
null,
169-
2,
170-
) + "\n",
171-
);
172-
}).pipe(Effect.provide(layer), Effect.scoped);
173-
}).pipe(
174-
Effect.provide(Layer.mergeAll(NodeServices.layer, FetchHttpClient.layer)),
175-
Effect.scoped,
176-
),
177-
{ timeout: 900_000 },
140+
const beforeFinish = yield* Clock.currentTimeMillis;
141+
const result = yield* service.finish({ sessionId, chunkCount });
142+
runs.push({
143+
text: result.text,
144+
locale: result.locale,
145+
finishMs: (yield* Clock.currentTimeMillis) - beforeFinish,
146+
recordingSeconds: recording.length / 32000,
147+
startMs,
148+
});
149+
assert.isTrue(result.text.length > 0);
150+
if (model === "whisper-turbo" && run === 3) assert.strictEqual(result.locale, "nl");
151+
}
152+
// This opt-in test reports the public fixture transcript; the product service never logs recordings.
153+
process.stdout.write(
154+
JSON.stringify(
155+
{
156+
model,
157+
availableParallelism: NodeOS.availableParallelism(),
158+
numThreads: Math.min(8, Math.max(1, NodeOS.availableParallelism())),
159+
runs,
160+
rssMB: Math.round(process.memoryUsage().rss / 1e6),
161+
},
162+
null,
163+
2,
164+
) + "\n",
165+
);
166+
}).pipe(Effect.provide(layer), Effect.scoped);
167+
}).pipe(Effect.provide(Layer.mergeAll(NodeServices.layer, FetchHttpClient.layer)), Effect.scoped),
178168
);

0 commit comments

Comments
 (0)