Skip to content

Commit 40d57b1

Browse files
feat(server): transcribe audio on a chosen environment
1 parent 567a105 commit 40d57b1

71 files changed

Lines changed: 6649 additions & 69 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎apps/mobile/src/features/voice-input/voiceInputPresentation.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@ export function resolveVoiceComposerPresentation(
5353
};
5454
}
5555
case "transcribing":
56+
case "post-processing":
5657
return {
5758
leadingAction: "cancel",
5859
trailingAction: "confirm",

‎apps/server/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@
7070
"saxes": "6.0.0",
7171
"stream-chain": "^4.2.5",
7272
"stream-json": "3.6.0",
73+
"transcribe-cpp": "0.4.1",
7374
"undici": "catalog:",
7475
"yaml": "catalog:",
7576
"yauzl": "^3.4.0",

‎apps/server/src/bin.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,11 @@ if (
3535
} else if (command === "acp-mcp-bridge" || command === "acp-mcp-call") {
3636
const { runAcpMcpCliFastPath } = await import("./mcp/AcpMcpStdioBridge.ts");
3737
await runAcpMcpCliFastPath(command, process.argv.slice(3));
38+
} else if (command === "__speech-worker") {
39+
const mode = process.argv[3];
40+
if (mode !== "discover" && mode !== "model") throw new Error("Unknown speech worker mode.");
41+
const { runNativeSpeechWorker } = await import("./speech/native.ts");
42+
await runNativeSpeechWorker(mode, process.argv[4]);
3843
} else {
3944
enableCliCompileCache();
4045
// Windows desktop launches without stdin; attach before reading, except on fast paths using pipes.

‎apps/server/src/binCli.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import { runDefaultServerCommand, serveCommand, startCommand } from "./cli/serve
2525
import { updateCommand } from "./cli/update.ts";
2626
import { uninstallCommand } from "./cli/uninstall.ts";
2727
import { serviceLauncherCommand } from "./cli/serviceLauncher.ts";
28+
import { speechWorkerCommand } from "./cli/speechWorker.ts";
2829
import { claudeHistoryCommand } from "./cli/claudeHistory.ts";
2930
import { sshHelperCommand } from "./cli/sshHelper.ts";
3031
import { serviceCommand } from "./cli/service.ts";
@@ -91,6 +92,7 @@ export const makeCli = ({ cloudEnabled = hasCloudPublicConfig } = {}) =>
9192
uninstallCommand,
9293
serviceLauncherCommand,
9394
claudeHistoryCommand,
95+
speechWorkerCommand,
9496
sshHelperCommand,
9597

9698
servicePreflightCommand,
Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
import * as Effect from "effect/Effect";
2+
import { Argument, Command } from "effect/cli";
3+
import { runNativeSpeechWorker } from "../speech/native.ts";
4+
5+
export const speechWorkerCommand = Command.make("__speech-worker", {
6+
mode: Argument.Literals("mode", ["discover", "model"]),
7+
moduleUrl: Argument.String("module-url").pipe(Argument.optional),
8+
}).pipe(
9+
Command.unlisted,
10+
Command.withHandler(({ mode, moduleUrl }) =>
11+
Effect.promise(() =>
12+
runNativeSpeechWorker(mode, moduleUrl._tag === "Some" ? moduleUrl.value : undefined),
13+
),
14+
),
15+
);

‎apps/server/src/environment/ServerEnvironment.test.ts‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,30 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("ServerEnvironmen
123123
}
124124
}),
125125
);
126+
it.effect.each([
127+
{ platform: "darwin", architecture: "arm64", supported: true },
128+
{ platform: "linux", architecture: "x64", supported: true },
129+
{ platform: "linux", architecture: "x64", libc: "musl", supported: false },
130+
{ platform: "win32", architecture: "arm64", supported: false },
131+
{ platform: "linux", architecture: "arm", supported: false },
132+
] as const)("advertises voice transcription on $platform-$architecture: $supported", (host) =>
133+
Effect.gen(function* () {
134+
const fileSystem = yield* FileSystem.FileSystem;
135+
const baseDir = yield* fileSystem.makeTempDirectoryScoped({
136+
prefix: "t3-server-environment-speech-test-",
137+
});
138+
const descriptor = yield* Effect.gen(function* () {
139+
const serverEnvironment = yield* ServerEnvironment.ServerEnvironment;
140+
return yield* serverEnvironment.getDescriptor;
141+
}).pipe(
142+
Effect.provide(layerServerEnvironment(baseDir)),
143+
Effect.provideService(HostProcess.Platform, host.platform),
144+
Effect.provideService(HostProcess.Architecture, host.architecture),
145+
Effect.provideService(HostProcess.LinuxLibc, "libc" in host ? host.libc : "gnu"),
146+
);
147+
expect(descriptor.capabilities.voiceTranscription).toBe(host.supported);
148+
}),
149+
);
126150
it.effect.each([
127151
{ name: "missing", content: undefined },
128152
{ name: "empty", content: "" },

‎apps/server/src/environment/ServerEnvironment.ts‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import { resolveServiceLauncherMode } from "../cloud/serviceLauncherClient.ts";
2222
import * as ServerConfig from "../config.ts";
2323
import { databaseMigrationHead } from "../persistence/databaseMigrationHead.ts";
2424
import * as ProcessRunner from "../processRunner.ts";
25+
import { isSpeechPlatformSupported } from "../speech/platform.ts";
2526
import { resolveServerEnvironmentLabel } from "./ServerEnvironmentLabel.ts";
2627
import { detectServerEnvironmentMachineKind } from "./ServerEnvironmentMachine.ts";
2728

@@ -189,6 +190,7 @@ export const make = Effect.gen(function* () {
189190
const identity = yield* ServerEnvironmentIdentity;
190191
const hostPlatform = yield* HostProcess.Platform;
191192
const hostArchitecture = yield* HostProcess.Architecture;
193+
const hostLibc = hostPlatform === "linux" ? yield* HostProcess.LinuxLibc : undefined;
192194
const environmentId = yield* identity.getEnvironmentId;
193195
const cwdBaseName = path.basename(serverConfig.cwd).trim();
194196
const label = yield* resolveServerEnvironmentLabel({ cwdBaseName });
@@ -269,6 +271,7 @@ export const make = Effect.gen(function* () {
269271
triage: true,
270272
oneTimeSchedules: true,
271273
npmReleaseLookup: true,
274+
voiceTranscription: isSpeechPlatformSupported(hostPlatform, hostArchitecture, hostLibc),
272275
...(serverSelfUpdate === null ? {} : { serverSelfUpdate }),
273276
...(serverInstallation === null ? {} : { serverInstallation }),
274277
...(serverSelfUpdate === "boot-service" || desktopAppUpdate

‎apps/server/src/git/GitManager.test.ts‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -290,6 +290,8 @@ function createTextGeneration(
290290
title: "Update workflow",
291291
}),
292292
generateReviewGuide: () => Effect.succeed({ stops: [] }),
293+
generateTranscriptionPostProcessing: () =>
294+
Effect.succeed({ transcription: "processed transcript" }),
293295
...overrides,
294296
};
295297

@@ -339,6 +341,8 @@ function createTextGeneration(
339341
),
340342
),
341343
generateReviewGuide: (input) => implementation.generateReviewGuide(input),
344+
generateTranscriptionPostProcessing: (input) =>
345+
implementation.generateTranscriptionPostProcessing(input),
342346
};
343347
}
344348

‎apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -259,6 +259,7 @@ it.effect("refreshes the failing instance snapshot after a committed usage_limit
259259
generatePrContent: () => Effect.die("unused"),
260260
generateThreadTitle: () => Effect.die("unused"),
261261
generateReviewGuide: () => Effect.die("unused"),
262+
generateTranscriptionPostProcessing: () => Effect.die("unused"),
262263
},
263264
snapshot: {
264265
refreshUsage: Deferred.succeed(refreshed, undefined).pipe(Effect.asVoid),

‎apps/server/src/provider/Drivers/CodexManagedProvider.ts‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -284,6 +284,11 @@ export const makeManagedCodexProvider = Effect.fn("makeManagedCodexProvider")(fu
284284
protect("generateThreadTitle", nativeGeneration.generateThreadTitle(value)),
285285
generateReviewGuide: (value) =>
286286
protect("generateReviewGuide", nativeGeneration.generateReviewGuide(value)),
287+
generateTranscriptionPostProcessing: (value) =>
288+
protect(
289+
"generateTranscriptionPostProcessing",
290+
nativeGeneration.generateTranscriptionPostProcessing(value),
291+
),
287292
};
288293
return {
289294
instanceId,

0 commit comments

Comments
 (0)