Skip to content

Instantly share code, notes, and snippets.

@JustYannicc
Created October 17, 2026 19:42
Show Gist options
  • Select an option

  • Save JustYannicc/64f4fe0566348a49d65a4db61ef34777 to your computer and use it in GitHub Desktop.

Select an option

Save JustYannicc/64f4fe0566348a49d65a4db61ef34777 to your computer and use it in GitHub Desktop.
Synthetic T3 Code turn-item history compaction reproduction

Turn-item history compaction reproduction

Run node measure-turn-item-history.mjs before and node measure-turn-item-history.mjs after from the T3 Code repository with its locked dependencies installed. The baseline loader uses upstream commit 13aa882c22d584de5388dfce3b3ca413387ba059; the after run uses the checked-out compactor. The measured fix is commit 66f8a3b6f3ab94c40e10d2a9723590aa680b6aaf, unchanged in candidate bcf0d667f8a4c6047bdce37774a4804d3f079fe3.

Each run creates a fresh temporary SQLite database, writes nine synthetic command/reasoning/todo items through the real event sink with 200 growing snapshots each, compacts, rebuilds the projections, and removes the fixture database. It never opens application state.

before.json and after.json were measured on macOS arm64, Node 24.18.0, at background priority. The dataset occupies about 28 MB. This is a small, warm fixture, not a simulation of a cold multi-gigabyte history.

The fixed pass keeps 18 eligible snapshots rather than 1,800, with 253,119 rather than 25,289,625 payload bytes. Full projection rebuild equality, unchanged global high-water, and a no-op second pass are asserted by the script. The main file size does not shrink during this measurement; deleted pages become reusable.

The existing compactor scans at most 50 rows per query and yields between pages. It has no timed duty-cycle cap. This retention change does not reduce incoming snapshot writes, alter streaming cadence, or change the six-hour maintenance schedule.

{
"mode": "after",
"node": "v24.18.0",
"items": 9,
"updatesPerItem": 200,
"before": {
"rows": 1800,
"payloadBytes": 25289625,
"allocatedBytes": 28151808,
"freeBytes": 24576
},
"after": {
"rows": 18,
"payloadBytes": 253119,
"allocatedBytes": 28151808,
"freeBytes": 26808320
},
"summary": {
"deletedEventCount": 1782,
"deletedReceiptCount": 0,
"reclaimableBytes": 26808320
},
"highWater": 1801,
"elapsedMs": 68.43104200000016,
"discoveryQueries": 37,
"discoveryTotalMs": 34.79287599999998,
"discoveryP95Ms": 1.0954580000002352,
"discoveryMaxMs": 11.719666000000416,
"deleteQueries": 36,
"deleteTotalMs": 22.75758400000086,
"deleteMaxMs": 1.1514589999997042,
"rebuildEqual": true,
"secondPassDeleted": 0,
"limitation": "50-row discovery pages with cooperative yields; no timed duty-cycle cap. Incoming snapshot writes and six-hour schedule are unchanged."
}
{
"mode": "before",
"node": "v24.18.0",
"items": 9,
"updatesPerItem": 200,
"before": {
"rows": 1800,
"payloadBytes": 25289625,
"allocatedBytes": 28151808,
"freeBytes": 24576
},
"after": {
"rows": 1800,
"payloadBytes": 25289625,
"allocatedBytes": 28151808,
"freeBytes": 24576
},
"summary": {
"deletedEventCount": 0,
"deletedReceiptCount": 0,
"reclaimableBytes": 24576
},
"highWater": 1801,
"elapsedMs": 14.673624999999902,
"discoveryQueries": 37,
"discoveryTotalMs": 9.47270700000081,
"discoveryP95Ms": 0.4124170000000049,
"discoveryMaxMs": 0.5140000000001237,
"deleteQueries": 0,
"deleteTotalMs": 0,
"deleteMaxMs": 0,
"rebuildEqual": true,
"secondPassDeleted": 0,
"limitation": "50-row discovery pages with cooperative yields; no timed duty-cycle cap. Incoming snapshot writes and six-hour schedule are unchanged."
}
import * as fs from 'node:fs';
import * as os from 'node:os';
import * as path from 'node:path';
import { createRequire } from 'node:module';
import { execFileSync } from 'node:child_process';
import { pathToFileURL } from 'node:url';
import assert from 'node:assert/strict';
const root = process.cwd();
const mode = process.argv[2];
assert.ok(mode === 'before' || mode === 'after');
const require = createRequire(path.join(root, 'apps/server/package.json'));
const pkg = (name) => import(pathToFileURL(require.resolve(name)).href);
const source = (name) => import(pathToFileURL(path.join(root, 'apps/server/src', name)).href);
const [Effect, Layer, DateTime, SqlClient, Tracer, NodeServices, Sqlite, EventStore, EventSink, ProjectionStore] = await Promise.all([
pkg('effect/Effect'), pkg('effect/Layer'), pkg('effect/DateTime'), pkg('effect/sql/SqlClient'),
pkg('effect/Tracer'), pkg('@effect/platform-node/NodeServices'), source('persistence/Sqlite.ts'),
source('orchestration-v2/EventStore.ts'), source('orchestration-v2/EventSink.ts'), source('orchestration-v2/ProjectionStore.ts'),
]);
const temp = fs.mkdtempSync(path.join(os.tmpdir(), 't3-turn-item-synthetic-'));
try {
let maintenancePath = path.join(root, 'apps/server/src/orchestration-v2/ProjectionMaintenance.ts');
if (mode === 'before') {
const original = execFileSync('git', ['show', '13aa882c22d584de5388dfce3b3ca413387ba059:apps/server/src/orchestration-v2/ProjectionMaintenance.ts'], { cwd: root, encoding: 'utf8' });
fs.symlinkSync(path.join(root, 'apps/server/node_modules'), path.join(temp, 'node_modules'));
maintenancePath = path.join(temp, 'ProjectionMaintenance.ts');
fs.writeFileSync(maintenancePath, original.replaceAll('"./EventStore.ts"', JSON.stringify(pathToFileURL(path.join(root, 'apps/server/src/orchestration-v2/EventStore.ts')).href)).replaceAll('"./ProjectionStore.ts"', JSON.stringify(pathToFileURL(path.join(root, 'apps/server/src/orchestration-v2/ProjectionStore.ts')).href)));
}
const Maintenance = await import(pathToFileURL(maintenancePath).href);
const database = Sqlite.layerFromPath(path.join(temp, 'synthetic.sqlite')).pipe(Layer.provide(NodeServices.layer));
const stores = Layer.mergeAll(database, EventStore.layer.pipe(Layer.provide(database)), ProjectionStore.layer.pipe(Layer.provide(database)));
const testLayer = Layer.mergeAll(stores, EventSink.layer.pipe(Layer.provide(stores)), Maintenance.layer.pipe(Layer.provide(stores)));
const result = await Effect.runPromise(Effect.gen(function* () {
const sql = yield* SqlClient.SqlClient;
const sink = yield* EventSink.EventSinkV2;
const store = yield* EventStore.EventStoreV2;
const projections = yield* ProjectionStore.ProjectionStoreV2;
const maintenance = yield* Maintenance.ProjectionMaintenanceV2;
const now = DateTime.makeUnsafe('2026-10-17T00:00:00Z');
const threadId = 'thread:synthetic';
const thread = {
id: threadId, createdBy: 'user', creationSource: 'web', projectId: 'project:synthetic', title: 'Synthetic',
providerInstanceId: 'codex', modelSelection: { instanceId: 'codex', model: 'gpt-5.4' },
runtimeMode: 'full-access', interactionMode: 'default', branch: null, worktreePath: null, activeProviderThreadId: null,
lineage: { parentThreadId: null, relationshipToParent: null, rootThreadId: threadId }, forkedFrom: null,
createdAt: now, updatedAt: now, archivedAt: null, settledOverride: null, settledAt: null, lastVisitedAt: null, deletedAt: null,
};
yield* sink.write({ events: [{ id: 'event:create', type: 'thread.created', threadId, occurredAt: now, payload: thread }] });
const types = ['command_execution', 'reasoning', 'todo_list'];
const updates = 200;
for (let step = 0; step < updates; step++) {
const events = types.flatMap((type) => Array.from({ length: 3 }, (_, item) => {
const text = 'synthetic output '.repeat((step + 1) * 8);
const payload = {
id: `item:${type}:${item}`, type, threadId, runId: null, nodeId: null, providerThreadId: null, providerTurnId: null,
nativeItemRef: null, parentItemId: null, ordinal: 1, status: step === updates - 1 ? 'completed' : 'running', title: null,
startedAt: now, completedAt: step === updates - 1 ? now : null, updatedAt: now,
...(type === 'command_execution' ? { input: 'synthetic', output: text, exitCode: 0 }
: type === 'reasoning' ? { text, streaming: step !== updates - 1 }
: { planId: `plan:${item}`, steps: [{ id: 'step', text: text.trim(), status: step === updates - 1 ? 'completed' : 'running' }] }),
};
return { id: `event:${type}:${item}:${step}`, type: 'turn-item.updated', threadId, occurredAt: now, payload };
}));
yield* sink.write({ events });
}
const stats = Effect.gen(function* () {
const rows = yield* sql`SELECT COUNT(*) AS rows, SUM(octet_length(payload_json)) AS payloadBytes FROM orchestration_events WHERE event_type = 'turn-item.updated'`;
const pages = yield* sql`PRAGMA page_count`;
const free = yield* sql`PRAGMA freelist_count`;
const size = yield* sql`PRAGMA page_size`;
return { ...rows[0], allocatedBytes: pages[0].page_count * size[0].page_size, freeBytes: free[0].freelist_count * size[0].page_size };
});
const before = yield* stats;
const projectionBefore = yield* projections.getThreadProjection(threadId);
const highWater = yield* store.latestApplicationSequence;
const scans = [];
const deletes = [];
const tracer = Tracer.make({ span(options) {
const span = new Tracer.NativeSpan(options);
const start = performance.now();
const end = span.end.bind(span);
span.end = (endTime, exit) => {
end(endTime, exit);
const query = span.attributes.get('db.query.text');
if (typeof query !== 'string') return;
if (query.includes('FROM orchestration_events AS event')) scans.push({ ms: performance.now() - start, query });
if (query.startsWith('DELETE FROM orchestration_events')) deletes.push(performance.now() - start);
};
return span;
}});
const started = performance.now();
const summary = yield* maintenance.compactEventStore.pipe(Effect.withTracer(tracer));
const elapsedMs = performance.now() - started;
const after = yield* stats;
assert.equal(yield* store.latestApplicationSequence, highWater);
assert.equal((yield* maintenance.rebuild).valid, true);
assert.deepEqual(yield* projections.getThreadProjection(threadId), projectionBefore);
const sorted = scans.map(x => x.ms).sort((a,b) => a-b);
assert.ok(scans.every(x => /ORDER BY event.sequence DESC\s+LIMIT \?/.test(x.query)));
const secondPass = yield* maintenance.compactEventStore;
assert.equal(secondPass.deletedEventCount, 0);
return { mode, node: process.version, items: 9, updatesPerItem: updates, before, after, summary, highWater,
elapsedMs, discoveryQueries: scans.length, discoveryTotalMs: sorted.reduce((a,b) => a+b,0),
discoveryP95Ms: sorted[Math.floor(sorted.length * .95)], discoveryMaxMs: sorted.at(-1),
deleteQueries: deletes.length, deleteTotalMs: deletes.reduce((a,b) => a+b,0), deleteMaxMs: Math.max(0,...deletes),
rebuildEqual: true, secondPassDeleted: secondPass.deletedEventCount,
limitation: '50-row discovery pages with cooperative yields; no timed duty-cycle cap. Incoming snapshot writes and six-hour schedule are unchanged.' };
}).pipe(Effect.provide(testLayer)));
console.log(JSON.stringify(result, null, 2));
} finally {
fs.rmSync(temp, { recursive: true, force: true });
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment