@@ -14,7 +14,10 @@ import type { PlatformError } from "effect/PlatformError";
1414import * as Schema from "effect/Schema" ;
1515import * as Stream from "effect/Stream" ;
1616
17+ import * as HostProcess from "@t3tools/shared/HostProcess" ;
18+
1719import * as ServerConfig from "../config.ts" ;
20+ import { expandHomePath } from "../os-jank.ts" ;
1821import type { SpeechModel } from "./model.ts" ;
1922
2023type ModelFile = Pick < SpeechModel , "id" | "filename" | "revision" | "size" | "sha256" > ;
@@ -44,13 +47,25 @@ export class SpeechModelStore extends Context.Service<
4447 readonly isReady : ( model : ModelFile ) => Effect . Effect < boolean > ;
4548 /** The installed file once its SHA-256 has been verified, hashing it on first use. */
4649 readonly installedPath : ( model : ModelFile ) => Effect . Effect < string | undefined > ;
50+ /** Links a copy the Hugging Face cache or Handy already holds; fetches only without one. */
4751 readonly download : (
4852 model : ModelFile ,
4953 progress ?: SpeechModelDownloadProgress ,
5054 ) => Effect . Effect <
5155 string ,
5256 SpeechModelDownloadError | PlatformError | HttpClientError . HttpClientError
5357 > ;
58+ /**
59+ * Installs catalog files already on this machine as symlinks, never copies.
60+ * `source` is a file or a folder to search; returns the ids it installed.
61+ */
62+ readonly link : (
63+ models : ReadonlyArray < ModelFile > ,
64+ source : string ,
65+ ) => Effect . Effect < ReadonlyArray < string > , PlatformError > ;
66+ /** The outside file a linked model points at. */
67+ readonly linkedFrom : ( model : ModelFile ) => Effect . Effect < string | undefined > ;
68+ /** Removes T3 Code's file or link; a linked file itself is left in place. */
5469 readonly remove : ( model : ModelFile ) => Effect . Effect < void , PlatformError > ;
5570 }
5671> ( ) ( "t3/speech/SpeechModelStore" ) { }
@@ -59,9 +74,14 @@ const DOWNLOAD_ATTEMPTS = 3;
5974const PARTIAL_FILENAME =
6075 / \. g g u f \. [ 0 - 9 a - f ] { 8 } - [ 0 - 9 a - f ] { 4 } - 4 [ 0 - 9 a - f ] { 3 } - [ 8 9 a b ] [ 0 - 9 a - f ] { 3 } - [ 0 - 9 a - f ] { 12 } \. p a r t $ / i;
6176
77+ const LINK_SEARCH_DEPTH = 5 ;
78+ const LINK_SEARCH_ENTRIES = 5_000 ;
79+
6280const make = Effect . gen ( function * ( ) {
6381 const fs = yield * FileSystem . FileSystem ;
6482 const path = yield * Path . Path ;
83+ const home = yield * HostProcess . HomeDirectory ;
84+ const platform = yield * HostProcess . Platform ;
6585 const http = yield * HttpClient . HttpClient ;
6686 const random = yield * SecureRandom . SecureRandom ;
6787 const config = yield * ServerConfig . ServerConfig ;
@@ -73,6 +93,28 @@ const make = Effect.gen(function* () {
7393 }
7494 }
7595 const modelPath = ( model : ModelFile ) => path . join ( directory , model . filename ) ;
96+ const appDataDirectories =
97+ platform === "darwin"
98+ ? [ path . join ( home , "Library" , "Application Support" ) ]
99+ : platform === "win32"
100+ ? [ path . join ( home , "AppData" , "Roaming" ) ]
101+ : [ path . join ( home , ".local" , "share" ) , path . join ( home , ".config" ) ] ;
102+ // Where the Hugging Face cache and Handy keep these same pinned files.
103+ const knownCopies = ( model : ModelFile ) => [
104+ path . join (
105+ home ,
106+ ".cache" ,
107+ "huggingface" ,
108+ "hub" ,
109+ `models--${ model . id . replace ( "/" , "--" ) } ` ,
110+ "snapshots" ,
111+ model . revision ,
112+ model . filename ,
113+ ) ,
114+ ...appDataDirectories . map ( ( appData ) =>
115+ path . join ( appData , "com.pais.handy" , "models" , model . filename ) ,
116+ ) ,
117+ ] ;
76118 // Files whose digest failed, by identity, so status polls stop calling them ready.
77119 const corrupt = new Map < string , string > ( ) ;
78120
@@ -179,12 +221,64 @@ const make = Effect.gen(function* () {
179221 return downloaded ;
180222 } ) ;
181223
224+ const filesUnder = Effect . fn ( "SpeechModelStore.filesUnder" ) ( function * ( source : string ) {
225+ const files : Array < { readonly file : string ; readonly size : number } > = [ ] ;
226+ const pending = [ { entry : source , depth : LINK_SEARCH_DEPTH } ] ;
227+ for ( let visited = 0 ; pending . length > 0 && visited < LINK_SEARCH_ENTRIES ; visited += 1 ) {
228+ const { entry, depth } = pending . pop ( ) ! ;
229+ const info = yield * statFile ( entry ) ;
230+ if ( info ?. type === "File" ) files . push ( { file : entry , size : Number ( info . size ) } ) ;
231+ if ( info ?. type !== "Directory" || depth === 0 ) continue ;
232+ for ( const name of yield * fs . readDirectory ( entry ) . pipe ( Effect . orElseSucceed ( ( ) => [ ] ) ) )
233+ pending . push ( { entry : path . join ( entry , name ) , depth : depth - 1 } ) ;
234+ }
235+ return files ;
236+ } ) ;
237+
238+ const realPath = ( file : string ) => fs . realPath ( file ) . pipe ( Effect . orElseSucceed ( ( ) => undefined ) ) ;
239+
240+ const link : SpeechModelStore [ "Service" ] [ "link" ] = Effect . fn ( "SpeechModelStore.link" ) (
241+ function * ( models , source ) {
242+ const root = path . resolve (
243+ yield * expandHomePath ( source . trim ( ) ) . pipe (
244+ Effect . provideService ( Path . Path , path ) ,
245+ Effect . provideService ( HostProcess . HomeDirectory , home ) ,
246+ ) ,
247+ ) ;
248+ yield * fs . makeDirectory ( directory , { recursive : true } ) ;
249+ const linked : string [ ] = [ ] ;
250+ for ( const { file, size } of yield * filesUnder ( root ) ) {
251+ for ( const model of models ) {
252+ if ( model . size !== size || linked . includes ( model . id ) ) continue ;
253+ const finalPath = modelPath ( model ) ;
254+ // Pointing at T3 Code's own model folder must not replace a file with a link to itself.
255+ if ( ( yield * realPath ( file ) ) === ( yield * realPath ( finalPath ) ) ) continue ;
256+ if ( yield * verifiedPath ( model , finalPath ) ) continue ;
257+ if ( ! ( yield * digestMatches ( model , file ) ) ) continue ;
258+ yield * fs . remove ( finalPath , { force : true } ) ;
259+ // Windows grants symlinks only to privileged accounts; a hard link also shares the bytes.
260+ yield * fs . symlink ( file , finalPath ) . pipe ( Effect . catch ( ( ) => fs . link ( file , finalPath ) ) ) ;
261+ const info = yield * fs . stat ( finalPath ) ;
262+ yield * fs
263+ . writeFileString ( `${ finalPath } .verified` , identity ( model , info ) , { mode : 0o600 } )
264+ . pipe ( Effect . ignore ) ;
265+ linked . push ( model . id ) ;
266+ }
267+ }
268+ return linked ;
269+ } ,
270+ ) ;
271+
182272 const download : SpeechModelStore [ "Service" ] [ "download" ] = Effect . fn ( "SpeechModelStore.download" ) (
183273 function * ( model , progress = { } ) {
184274 yield * Effect . annotateCurrentSpan ( { modelId : model . id } ) ;
185275 const finalPath = modelPath ( model ) ;
186276 yield * fs . makeDirectory ( directory , { recursive : true } ) ;
187277 if ( yield * verifiedPath ( model , finalPath , progress . onVerificationStart ) ) return finalPath ;
278+ for ( const copy of knownCopies ( model ) ) {
279+ const linked = yield * link ( [ model ] , copy ) . pipe ( Effect . orElseSucceed ( ( ) => [ ] ) ) ;
280+ if ( linked . length > 0 ) return finalPath ;
281+ }
188282 const partialPath = `${ finalPath } .${ yield * random . uuidV4 ( ) } .part` ;
189283 const url = `https://huggingface.co/${ model . id } /resolve/${ model . revision } /${ model . filename } ` ;
190284 return yield * Effect . gen ( function * ( ) {
@@ -228,6 +322,10 @@ const make = Effect.gen(function* () {
228322 ) ;
229323
230324 return SpeechModelStore . of ( {
325+ link,
326+ linkedFrom : Effect . fn ( "SpeechModelStore.linkedFrom" ) ( ( model ) =>
327+ fs . readLink ( modelPath ( model ) ) . pipe ( Effect . orElseSucceed ( ( ) => undefined ) ) ,
328+ ) ,
231329 isReady : Effect . fn ( "SpeechModelStore.isReady" ) ( ( model ) =>
232330 expectedFile ( model , modelPath ( model ) ) . pipe (
233331 Effect . map (
0 commit comments