|
8 | 8 | type OrchestrationV2TurnItem, |
9 | 9 | MessageId, |
10 | 10 | ProjectId, |
| 11 | + RunId, |
11 | 12 | ProviderDriverKind, |
12 | 13 | ProviderInstanceId, |
13 | 14 | ThreadId, |
@@ -404,69 +405,76 @@ const countContent = (threadId: ThreadId) => |
404 | 405 | `, |
405 | 406 | ).pipe(Effect.map((rows) => rows[0])); |
406 | 407 |
|
407 | | -it.effect("purges content of threads deleted beyond the retained history", () => |
408 | | - Effect.gen(function* () { |
409 | | - const sink = yield* EventSink.EventSinkV2; |
410 | | - const projections = yield* ProjectionStore.ProjectionStoreV2; |
411 | | - const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2; |
412 | | - const now = yield* DateTime.now; |
413 | | - const seed = Effect.fnUntraced(function* (name: string) { |
414 | | - const threadId = ThreadId.make(`thread:purge-${name}`); |
415 | | - const thread = makeThread(threadId, now); |
416 | | - yield* sink.write({ |
417 | | - events: [{ ...threadCreated(thread, now), id: EventId.make(`event:purge:${name}:thread`) }], |
418 | | - }); |
419 | | - yield* sink.write({ |
420 | | - events: [ |
421 | | - itemEvent( |
422 | | - threadId, |
423 | | - now, |
424 | | - itemSpecs[0], |
425 | | - 0, |
426 | | - `event:purge:${name}:item`, |
427 | | - TurnItemId.make(`turn-item:purge:${name}`), |
428 | | - ), |
429 | | - ], |
430 | | - }); |
431 | | - yield* sink.write({ |
432 | | - events: [ |
433 | | - { |
434 | | - id: EventId.make(`event:purge:${name}:message`), |
435 | | - type: "message.updated", |
| 408 | +const purgeFixture = Effect.gen(function* () { |
| 409 | + const sink = yield* EventSink.EventSinkV2; |
| 410 | + const projections = yield* ProjectionStore.ProjectionStoreV2; |
| 411 | + const maintenance = yield* ProjectionMaintenance.ProjectionMaintenanceV2; |
| 412 | + const now = yield* DateTime.now; |
| 413 | + const seed = Effect.fnUntraced(function* ( |
| 414 | + name: string, |
| 415 | + forkedFrom: OrchestrationV2AppThread["forkedFrom"] = null, |
| 416 | + ) { |
| 417 | + const threadId = ThreadId.make(`thread:purge-${name}`); |
| 418 | + const thread = { ...makeThread(threadId, now), forkedFrom }; |
| 419 | + yield* sink.write({ |
| 420 | + events: [{ ...threadCreated(thread, now), id: EventId.make(`event:purge:${name}:thread`) }], |
| 421 | + }); |
| 422 | + yield* sink.write({ |
| 423 | + events: [ |
| 424 | + itemEvent( |
| 425 | + threadId, |
| 426 | + now, |
| 427 | + itemSpecs[0], |
| 428 | + 0, |
| 429 | + `event:purge:${name}:item`, |
| 430 | + TurnItemId.make(`turn-item:purge:${name}`), |
| 431 | + ), |
| 432 | + ], |
| 433 | + }); |
| 434 | + yield* sink.write({ |
| 435 | + events: [ |
| 436 | + { |
| 437 | + id: EventId.make(`event:purge:${name}:message`), |
| 438 | + type: "message.updated", |
| 439 | + threadId, |
| 440 | + occurredAt: now, |
| 441 | + payload: { |
| 442 | + id: MessageId.make(`message:purge:${name}`), |
436 | 443 | threadId, |
437 | | - occurredAt: now, |
438 | | - payload: { |
439 | | - id: MessageId.make(`message:purge:${name}`), |
440 | | - threadId, |
441 | | - runId: null, |
442 | | - nodeId: null, |
443 | | - createdBy: "agent", |
444 | | - creationSource: "provider", |
445 | | - role: "assistant", |
446 | | - text: `purgeable${threadId.replace(/\W/g, "")}`, |
447 | | - streaming: false, |
448 | | - attachments: [], |
449 | | - createdAt: now, |
450 | | - updatedAt: now, |
451 | | - }, |
| 444 | + runId: null, |
| 445 | + nodeId: null, |
| 446 | + createdBy: "agent", |
| 447 | + creationSource: "provider", |
| 448 | + role: "assistant", |
| 449 | + text: `purgeable${threadId.replace(/\W/g, "")}`, |
| 450 | + streaming: false, |
| 451 | + attachments: [], |
| 452 | + createdAt: now, |
| 453 | + updatedAt: now, |
452 | 454 | }, |
453 | | - ], |
454 | | - }); |
455 | | - return thread; |
| 455 | + }, |
| 456 | + ], |
456 | 457 | }); |
457 | | - const remove = (thread: OrchestrationV2AppThread, at: DateTime.Utc) => |
458 | | - sink.write({ |
459 | | - events: [ |
460 | | - { |
461 | | - id: EventId.make(`event:purge:${thread.id}:deleted`), |
462 | | - type: "thread.deleted", |
463 | | - threadId: thread.id, |
464 | | - occurredAt: at, |
465 | | - payload: { ...thread, deletedAt: at, updatedAt: at }, |
466 | | - }, |
467 | | - ], |
468 | | - }); |
| 458 | + return thread; |
| 459 | + }); |
| 460 | + const remove = (thread: OrchestrationV2AppThread, at: DateTime.Utc) => |
| 461 | + sink.write({ |
| 462 | + events: [ |
| 463 | + { |
| 464 | + id: EventId.make(`event:purge:${thread.id}:deleted`), |
| 465 | + type: "thread.deleted", |
| 466 | + threadId: thread.id, |
| 467 | + occurredAt: at, |
| 468 | + payload: { ...thread, deletedAt: at, updatedAt: at }, |
| 469 | + }, |
| 470 | + ], |
| 471 | + }); |
| 472 | + return { projections, maintenance, now, seed, remove }; |
| 473 | +}); |
469 | 474 |
|
| 475 | +it.effect("purges content of threads deleted beyond the retained history", () => |
| 476 | + Effect.gen(function* () { |
| 477 | + const { projections, maintenance, now, seed, remove } = yield* purgeFixture; |
470 | 478 | const old = yield* seed("old"); |
471 | 479 | const live = yield* seed("live"); |
472 | 480 | yield* remove(old, now); |
@@ -519,3 +527,56 @@ it.effect("lists every thread-scoped projection table or names why it is kept", |
519 | 527 | ); |
520 | 528 | }).pipe(Effect.provide(layerTest)), |
521 | 529 | ); |
| 530 | + |
| 531 | +it.effect("keeps a deleted thread that a live run-fork still inherits history from", () => |
| 532 | + Effect.gen(function* () { |
| 533 | + const { maintenance, seed, remove } = yield* purgeFixture; |
| 534 | + const root = yield* seed("root"); |
| 535 | + const middle = yield* seed("middle", { |
| 536 | + type: "run", |
| 537 | + threadId: root.id, |
| 538 | + runId: RunId.make("run:purge:root"), |
| 539 | + }); |
| 540 | + const fork = yield* seed("fork", { |
| 541 | + type: "run", |
| 542 | + threadId: middle.id, |
| 543 | + runId: RunId.make("run:purge:middle"), |
| 544 | + }); |
| 545 | + const unrelated = yield* seed("unrelated"); |
| 546 | + yield* remove(root, yield* DateTime.now); |
| 547 | + yield* remove(middle, yield* DateTime.now); |
| 548 | + yield* remove(unrelated, yield* DateTime.now); |
| 549 | + yield* TestClock.adjust("2 days"); |
| 550 | + |
| 551 | + yield* maintenance.pruneEventStore(); |
| 552 | + assert.equal((yield* countContent(root.id))?.items, 1); |
| 553 | + assert.equal((yield* countContent(middle.id))?.items, 1); |
| 554 | + assert.equal((yield* countContent(fork.id))?.items, 1); |
| 555 | + assert.equal((yield* countContent(unrelated.id))?.items, 0); |
| 556 | + }).pipe(Effect.provide(layerTest)), |
| 557 | +); |
| 558 | + |
| 559 | +it.effect("purges a very large thread in bounded batches", () => |
| 560 | + Effect.gen(function* () { |
| 561 | + const { maintenance, seed, remove } = yield* purgeFixture; |
| 562 | + const sql = yield* SqlClient.SqlClient; |
| 563 | + const big = yield* seed("big"); |
| 564 | + yield* sql` |
| 565 | + INSERT INTO orchestration_v2_turn_item_positions (thread_id, turn_item_id, ordinal) |
| 566 | + WITH RECURSIVE n(i) AS (SELECT 1 UNION ALL SELECT i + 1 FROM n WHERE i < 1200) |
| 567 | + SELECT ${big.id}, 'turn-item:purge:filler:' || i, i + 100 FROM n |
| 568 | + `; |
| 569 | + yield* remove(big, yield* DateTime.now); |
| 570 | + yield* TestClock.adjust("2 days"); |
| 571 | + |
| 572 | + let batches = 0; |
| 573 | + const result = yield* maintenance.pruneEventStore( |
| 574 | + Effect.sync(() => { |
| 575 | + batches += 1; |
| 576 | + }), |
| 577 | + ); |
| 578 | + assert.isAtLeast(result.purgedProjectionRowCount, 1200); |
| 579 | + assert.isAtLeast(batches, 3); |
| 580 | + assert.equal((yield* countContent(big.id))?.positions, 0); |
| 581 | + }).pipe(Effect.provide(layerTest)), |
| 582 | +); |
0 commit comments