fix(sessions): sweep orphaned atomic-write temp files at gateway startup

Linux2010 · Linux2010 · commit 358f70148f0d · 2026-06-02T16:33:23.000Z
writeTextAtomic stages into <store>.<pid>.<uuid>.tmp then renames. If the process dies between write and rename (SIGKILL, OOM, hard shutdown), the temp is orphaned and accumulates on disk (#56827). Added sweepOrphanStoreTempsSync() that reclaims stale .tmp files older than 5 minutes (safe because live atomic writes rename within ms). Called once at gateway startup via server-startup-early.ts. Fixes #89520
diff --git a/src/config/sessions/disk-budget.test.ts b/src/config/sessions/disk-budget.test.ts
@@ -9,7 +9,11 @@ import {
   resolveTrajectoryPointerFilePath,
 } from "../../trajectory/paths.js";
 import { formatSessionArchiveTimestamp } from "./artifacts.js";
-import { enforceSessionDiskBudget, pruneUnreferencedSessionArtifacts } from "./disk-budget.js";
+import {
+  enforceSessionDiskBudget,
+  pruneUnreferencedSessionArtifacts,
+  sweepOrphanStoreTempsSync,
+} from "./disk-budget.js";
 import { saveSessionStore } from "./store.js";
 import type { SessionEntry } from "./types.js";
 
@@ -756,3 +760,63 @@ describe("pruneUnreferencedSessionArtifacts", () => {
     });
   });
 });
+
+describe("sweepOrphanStoreTempsSync", () => {
+  it("removes stale orphaned temp files", () => {
+    withTempDir(async (dir) => {
+      const storePath = path.join(dir, "sessions.json");
+      const staleTemp = path.join(
+        dir,
+        "sessions.json.12345.0f9c1a2b-3c4d-4e5f-8a9b-0c1d2e3f4a5b.tmp",
+      );
+      const freshTemp = path.join(
+        dir,
+        "sessions.json.12345.1a2b3c4d-5e6f-4a8b-9c0d-1e2f3a4b5c6d.tmp",
+      );
+
+      nodeFs.writeFileSync(staleTemp, "s".repeat(128), "utf-8");
+      nodeFs.writeFileSync(freshTemp, "f".repeat(128), "utf-8");
+
+      // Make stale temp older than 5 minutes
+      const old = new Date(Date.now() - 10 * 60 * 1000);
+      nodeFs.utimesSync(staleTemp, old, old);
+
+      const result = sweepOrphanStoreTempsSync(dir, "sessions.json");
+
+      expect(nodeFs.existsSync(staleTemp)).toBe(false);
+      expect(nodeFs.existsSync(freshTemp)).toBe(true);
+      expect(result.removedFiles).toBe(1);
+      expect(result.freedBytes).toBe(128);
+    });
+  });
+
+  it("returns zeros when no orphaned temps exist", () => {
+    withTempDir(async (dir) => {
+      const storePath = path.join(dir, "sessions.json");
+      nodeFs.writeFileSync(storePath, JSON.stringify({}), "utf-8");
+
+      const result = sweepOrphanStoreTempsSync(dir, "sessions.json");
+
+      expect(result.removedFiles).toBe(0);
+      expect(result.freedBytes).toBe(0);
+    });
+  });
+
+  it("skips non-matching temp files", () => {
+    withTempDir(async (dir) => {
+      const storePath = path.join(dir, "sessions.json");
+      const otherTemp = path.join(dir, "other.json.12345.0f9c1a2b-3c4d-4e5f-8a9b-0c1d2e3f4a5b.tmp");
+      nodeFs.writeFileSync(storePath, JSON.stringify({}), "utf-8");
+      nodeFs.writeFileSync(otherTemp, "x".repeat(64), "utf-8");
+
+      const old = new Date(Date.now() - 10 * 60 * 1000);
+      nodeFs.utimesSync(otherTemp, old, old);
+
+      const result = sweepOrphanStoreTempsSync(dir, "sessions.json");
+
+      // other.json.*.tmp should NOT be removed when sweeping for sessions.json
+      expect(nodeFs.existsSync(otherTemp)).toBe(true);
+      expect(result.removedFiles).toBe(0);
+    });
+  });
+});
diff --git a/src/config/sessions/disk-budget.ts b/src/config/sessions/disk-budget.ts
@@ -301,7 +301,65 @@ function isUnreferencedSessionArtifactFile(
 // An orphaned `sessions.json.<pid>.<uuid>.tmp` older than this is never a live
 // atomic write (those rename within milliseconds), so it is safe to reclaim
 // regardless of the general unreferenced-artifact age threshold (#56827).
-const SESSION_STORE_TEMP_STALE_MS = 5 * 60 * 1000;
+export const SESSION_STORE_TEMP_STALE_MS = 5 * 60 * 1000;
+
+/**
+ * Reclaim orphaned atomic-write temp files in a sessions directory.
+ *
+ * `writeTextAtomic` stages into `<store>.<pid>.<uuid>.tmp` then renames.
+ * If the process dies between write and rename, the temp is orphaned and
+ * accumulates on disk (#56827). This sweep removes temps older than the
+ * staleness window — safe because live atomic writes rename within ms.
+ *
+ * Call this once at gateway startup (or on first store load) to prevent
+ * unbounded disk waste from repeated crash-orphan cycles.
+ */
+export function sweepOrphanStoreTempsSync(
+  sessionsDir: string,
+  storeBasename: string,
+  options?: {
+    staleMs?: number;
+    log?: SessionDiskBudgetLogger;
+  },
+): { removedFiles: number; freedBytes: number } {
+  const staleMs = options?.staleMs ?? SESSION_STORE_TEMP_STALE_MS;
+  const log = options?.log ?? NOOP_LOGGER;
+  const cutoffMs = Date.now() - staleMs;
+  let removedFiles = 0;
+  let freedBytes = 0;
+
+  try {
+    const entries = fs.readdirSync(sessionsDir, { withFileTypes: true });
+    for (const entry of entries) {
+      if (!entry.isFile() || !isSessionStoreTempArtifactName(entry.name, storeBasename)) {
+        continue;
+      }
+      const filePath = path.join(sessionsDir, entry.name);
+      const stat = fs.statSync(filePath, { throwIfNoEntry: false });
+      if (!stat?.isFile() || stat.mtimeMs > cutoffMs) {
+        continue;
+      }
+      const size = stat.size;
+      fs.rmSync(filePath, { force: true });
+      removedFiles++;
+      freedBytes += size;
+    }
+  } catch {
+    // Directory may not exist yet (first run) — silently skip.
+    return { removedFiles: 0, freedBytes: 0 };
+  }
+
+  if (removedFiles > 0) {
+    log.info("swept orphaned atomic-write temp files", {
+      sessionsDir,
+      storeBasename,
+      removedFiles,
+      freedBytes,
+    });
+  }
+
+  return { removedFiles, freedBytes };
+}
 // Prompt blobs are written or mtime-refreshed before sessions.json points at
 // them. Treat fresh unreferenced blobs as in-flight so cleanup cannot strand a
 // durable promptRef that is about to be committed by another writer.
diff --git a/src/gateway/server-startup-early.ts b/src/gateway/server-startup-early.ts
@@ -126,6 +126,32 @@ export async function startGatewayEarlyRuntime(params: {
     taskRegistryMaintenance.startTaskRegistryMaintenance();
     getActiveTaskCount = () =>
       taskRegistryMaintenance.getInspectableActiveTaskRestartBlockers().length;
+
+    // Reclaim orphaned atomic-write temp files from previous crash cycles.
+    // `writeTextAtomic` stages into `<store>.<pid>.<uuid>.tmp` then renames;
+    // a crash between write and rename leaves the temp behind (#56827).
+    void measureStartup(params.startupTrace, "runtime.early.orphan-tmp-sweep", async () => {
+      const { sweepOrphanStoreTempsSync } = await import("../config/sessions/disk-budget.js");
+      const { resolveStorePath } = await import("../config/sessions/paths.js");
+      const { DEFAULT_AGENT_ID } = await import("../routing/session-key.js");
+      const path = await import("node:path");
+      const storePath = resolveStorePath(params.cfgAtStart.session?.store, {
+        agentId: DEFAULT_AGENT_ID,
+      });
+      const sessionsDir = path.dirname(storePath);
+      const storeBasename = path.basename(storePath);
+      const result = sweepOrphanStoreTempsSync(sessionsDir, storeBasename, {
+        log: {
+          info: (msg: string, ctx?: Record<string, unknown>) => params.log.info(msg),
+          warn: (msg: string, ctx?: Record<string, unknown>) => params.log.warn(msg),
+        },
+      });
+      if (result.removedFiles > 0) {
+        params.log.info(
+          `swept ${result.removedFiles} orphaned .tmp files (${(result.freedBytes / 1024).toFixed(1)} KB freed)`,
+        );
+      }
+    });
   }
 
   const skillsChangeUnsub = params.minimalTestGateway
