@@ -162,6 +162,8 @@ const GIB = 1024 ** 3;
162162// Cloned backups still need headroom for trial writes and migrations to diverge.
163163const CLONED_BACKUP_MIN_HEADROOM_BYTES = GIB ;
164164const CLONED_BACKUP_HEADROOM_FRACTION = 0.1 ;
165+ // While the online backup holds its snapshot, the live WAL and its copy can each reach the server's 128 MiB gate.
166+ const ONLINE_BACKUP_WAL_HEADROOM_BYTES = 2 * 128 * 1024 ** 2 ;
165167
166168async function statDatabaseFiles ( dbPath : string ) : Promise < ReadonlyArray < NodeFS . Stats > > {
167169 const stats : Array < NodeFS . Stats > = [ ] ;
@@ -242,7 +244,7 @@ export async function databaseBackupBlocker(
242244 await removeAbandonedBackupCopies ( backupRoot ) ;
243245 const [ files , free ] = await Promise . all ( [ statDatabaseFiles ( dbPath ) , readFreeBytes ( backupRoot ) ] ) ;
244246 const size = files . reduce ( ( total , file ) => total + file . size , 0 ) ;
245- const fullCopyBytes = size * 1.05 ;
247+ const fullCopyBytes = size * 1.05 + ONLINE_BACKUP_WAL_HEADROOM_BYTES ;
246248 if ( free >= fullCopyBytes ) return undefined ;
247249 const clonedBytes = Math . max (
248250 CLONED_BACKUP_MIN_HEADROOM_BYTES ,
@@ -254,6 +256,83 @@ export async function databaseBackupBlocker(
254256 return `Not enough free disk space to back up the database before updating: ${ gb ( needed ) } GB needed, ${ gb ( free ) } GB free. Free up disk space and try again. The server was not stopped.` ;
255257}
256258
259+ // A child-process copy: closing any descriptor this process opens on the database drops the snapshot's POSIX locks.
260+ async function copyWithoutOpening ( source : string , destination : string ) : Promise < void > {
261+ // oxlint-disable-next-line t3code/no-global-process-runtime -- Standalone launcher has no Effect runtime.
262+ if ( process . platform === "win32" ) return NodeFSP . copyFile ( source , destination ) ;
263+ if ( ! ( await runCp ( [ source , destination ] ) ) ) throw new Error ( `cp ${ source } failed.` ) ;
264+ }
265+
266+ // Copies the main file while the server runs; an open read transaction stops SQLite resetting the WAL,
267+ // which `finish` copies after the stop and which overrides any page the copy caught mid-checkpoint.
268+ // `-shm` is left out: its backfill count would make SQLite trust those pages. Past the WAL size gate,
269+ // a write-heavy server's new writes wait, uncommitted, until the server stops.
270+ async function startOnlineDatabaseBackup (
271+ baseDir : string ,
272+ pending : PendingServiceUpdate ,
273+ probe : Partial < BackupSpaceProbe > = { } ,
274+ ) : Promise < ( ( ) => Promise < void > ) | undefined > {
275+ const backupDir = databaseBackupDir ( baseDir , pending . id ) ;
276+ if ( await pathExists ( backupDir ) ) return undefined ;
277+ const { canClone } = { ...filesystemBackupSpaceProbe , ...probe } ;
278+ if ( await canClone ( pending . dbPath , NodePath . dirname ( backupDir ) ) . catch ( ( ) => true ) )
279+ return undefined ;
280+ const NodeSqlite = await import ( "node:sqlite" ) . catch ( ( ) => undefined ) ;
281+ if ( NodeSqlite === undefined ) return undefined ;
282+
283+ const stagingDir = `${ backupDir } .staging` ;
284+ const discardStaging = ( ) =>
285+ NodeFSP . rm ( stagingDir , { recursive : true , force : true } ) . catch ( ( ) => undefined ) ;
286+ const fallBack = async ( cause : unknown ) => {
287+ await discardStaging ( ) ;
288+ process . stderr . write (
289+ `[service-launcher] Copying the database while the server runs failed; backing it up offline: ${ cause instanceof Error ? cause . message : String ( cause ) } \n` ,
290+ ) ;
291+ return undefined ;
292+ } ;
293+ let snapshot : InstanceType < typeof NodeSqlite . DatabaseSync > | undefined ;
294+ // SQLite keeps -wal beside the resolved file, not beside a symlink to it.
295+ let dbPath = pending . dbPath ;
296+ const release = ( ) => {
297+ try {
298+ snapshot ?. close ( ) ;
299+ } catch { }
300+ snapshot = undefined ;
301+ } ;
302+ try {
303+ await discardStaging ( ) ;
304+ await NodeFSP . mkdir ( stagingDir , { recursive : true , mode : 0o700 } ) ;
305+ dbPath = await NodeFSP . realpath ( pending . dbPath ) ;
306+ snapshot = new NodeSqlite . DatabaseSync ( dbPath , { readOnly : true } ) ;
307+ const mode = String ( snapshot . prepare ( "PRAGMA journal_mode" ) . get ( ) ?. journal_mode ) ;
308+ if ( mode . toLowerCase ( ) !== "wal" ) throw new Error ( `journal_mode is ${ mode } , not wal.` ) ;
309+ snapshot . exec ( "BEGIN" ) ;
310+ snapshot . prepare ( "SELECT count(*) FROM sqlite_schema" ) . get ( ) ;
311+ const mainCopy = databaseBackupFile ( stagingDir , "" ) ;
312+ await copyWithoutOpening ( dbPath , mainCopy ) ;
313+ await syncFile ( mainCopy ) ;
314+ } catch ( cause ) {
315+ release ( ) ;
316+ return fallBack ( cause ) ;
317+ }
318+ return async ( ) => {
319+ try {
320+ const wal = `${ dbPath } -wal` ;
321+ if ( ! ( await pathExists ( wal ) ) )
322+ throw new Error ( "the WAL disappeared while the snapshot was open." ) ;
323+ const walCopy = databaseBackupFile ( stagingDir , "-wal" ) ;
324+ await copyWithoutOpening ( wal , walCopy ) ;
325+ await syncFile ( walCopy ) ;
326+ release ( ) ;
327+ await NodeFSP . rename ( stagingDir , backupDir ) ;
328+ await syncDirectory ( NodePath . dirname ( backupDir ) ) ;
329+ } catch ( cause ) {
330+ release ( ) ;
331+ await fallBack ( cause ) ;
332+ }
333+ } ;
334+ }
335+
257336// Preserve completed backups; restarted trials may already have changed the live database.
258337async function backupDatabaseOnce ( baseDir : string , pending : PendingServiceUpdate ) : Promise < void > {
259338 const backupDir = databaseBackupDir ( baseDir , pending . id ) ;
@@ -330,8 +409,13 @@ async function restoreDatabaseBackup(
330409
331410 await markDatabaseRestorePending ( backupDir ) ;
332411 const directories = new Set ( [ NodePath . dirname ( pending . dbPath ) ] ) ;
412+ // SQLite keeps -wal and -shm beside the resolved database file, not beside a symlink to it.
413+ const mainTarget = await restoreTarget ( pending . dbPath ) ;
333414 for ( const suffix of DB_FILE_SUFFIXES ) {
334- const livePath = `${ pending . dbPath } ${ suffix } ` ;
415+ const livePath = suffix === "" ? pending . dbPath : `${ mainTarget } ${ suffix } ` ;
416+ if ( livePath !== `${ pending . dbPath } ${ suffix } ` ) {
417+ await NodeFSP . rm ( `${ pending . dbPath } ${ suffix } ` , { force : true } ) ;
418+ }
335419 const source = databaseBackupFile ( backupDir , suffix ) ;
336420 if ( await pathExists ( source ) ) {
337421 // Restore into the symlink target; cloning would replace the link.
@@ -708,8 +792,17 @@ export class Launcher {
708792 return ;
709793 }
710794 this . #timer = undefined ;
795+ // Anything thrown outside its own fallback still leaves the offline backup to run.
796+ const finishOnlineBackup = await startOnlineDatabaseBackup (
797+ this . #baseDir,
798+ pending ,
799+ this . #backupSpaceProbe,
800+ ) . catch ( ( ) => undefined ) ;
711801 this . #child = null ;
712802 await terminateChild ( child . process ) ;
803+ await finishOnlineBackup ?.( ) ;
804+ // A stop during a minutes-long copy should not start a trial; restart resumes the pending update.
805+ if ( this . #stopRequested) return ;
713806 await this . #startTrial( pending ) ;
714807 }
715808
0 commit comments