|
1 | 1 | import { describe, expect, it } from "@effect/vitest"; |
2 | 2 | import { EventId, ProjectId, type OrchestrationV2ShellStreamItem } from "@t3tools/contracts"; |
3 | 3 | import * as NodeSqliteClient from "@t3tools/shared/nodeSqliteClient"; |
| 4 | +import * as Cause from "effect/Cause"; |
| 5 | +import * as Deferred from "effect/Deferred"; |
4 | 6 | import * as Effect from "effect/Effect"; |
| 7 | +import * as Exit from "effect/Exit"; |
| 8 | +import * as Fiber from "effect/Fiber"; |
5 | 9 | import * as Layer from "effect/Layer"; |
| 10 | +import * as Option from "effect/Option"; |
6 | 11 | import * as PubSub from "effect/PubSub"; |
| 12 | +import * as SqlClient from "effect/sql/SqlClient"; |
7 | 13 | import * as Stream from "effect/Stream"; |
8 | 14 |
|
9 | 15 | import * as ProjectStore from "./orchestration-v2/ProjectStore.ts"; |
10 | 16 | import { ORCHESTRATION_V2_PROJECTION_SCHEMA_VERSION } from "./orchestration-v2/ProjectionStore.ts"; |
11 | 17 | import * as ThreadManagementService from "./orchestration-v2/ThreadManagementService.ts"; |
12 | 18 | import * as OrchestrationEventStore from "./persistence/OrchestrationEventStore.ts"; |
| 19 | +import * as SqlitePersistence from "./persistence/Sqlite.ts"; |
13 | 20 | import * as ProjectEnrichmentService from "./project/ProjectEnrichmentService.ts"; |
14 | 21 | import * as ProjectService from "./project/ProjectService.ts"; |
15 | 22 | import { subscribeOrchestrationV2Shell } from "./ws.ts"; |
@@ -164,3 +171,337 @@ describe("shell subscription initial RPC chunks", () => { |
164 | 171 | }), |
165 | 172 | ); |
166 | 173 | }); |
| 174 | + |
| 175 | +const ReplayTestLayer = Layer.mergeAll(OrchestrationEventStore.layer, ProjectStore.layer).pipe( |
| 176 | + Layer.provideMerge(SqlitePersistence.layerMemory), |
| 177 | +); |
| 178 | + |
| 179 | +// Keep append, projection storage, raw paging, and replay-to-live recovery real. |
| 180 | +// Only the project-shell read is parked, like a busy projection service. |
| 181 | +const makeParkedReplay = Effect.fnUntraced(function* (unknownRows = 0) { |
| 182 | + const store = yield* OrchestrationEventStore.OrchestrationEventStore; |
| 183 | + const projects = yield* ProjectStore.ProjectStoreV2; |
| 184 | + const sql = yield* SqlClient.SqlClient; |
| 185 | + const changes = yield* Effect.acquireRelease( |
| 186 | + PubSub.unbounded<ProjectEnrichmentService.ProjectEnrichmentChange>(), |
| 187 | + PubSub.shutdown, |
| 188 | + ); |
| 189 | + const firstProjected = yield* Deferred.make<void>(); |
| 190 | + const projectionParked = yield* Deferred.make<void>(); |
| 191 | + const releaseProjection = yield* Deferred.make<void, ProjectService.ProjectOperationError>(); |
| 192 | + const projectionClosed = yield* Deferred.make<void>(); |
| 193 | + const replayClosed = yield* Deferred.make<void>(); |
| 194 | + const enrichmentClosed = yield* Deferred.make<void>(); |
| 195 | + const laterProject = { |
| 196 | + ...project, |
| 197 | + id: ProjectId.make("project-b"), |
| 198 | + workspaceRoot: "/workspace/b", |
| 199 | + }; |
| 200 | + const commit = Effect.fnUntraced(function* ( |
| 201 | + event: OrchestrationEventStore.UnsequencedProjectEvent, |
| 202 | + ) { |
| 203 | + const stored = yield* sql.withTransaction( |
| 204 | + Effect.gen(function* () { |
| 205 | + const stored = yield* store.appendProjectEvent(event); |
| 206 | + yield* projects.apply(stored); |
| 207 | + return stored; |
| 208 | + }), |
| 209 | + ); |
| 210 | + yield* store.publishCommitted([stored]); |
| 211 | + return stored; |
| 212 | + }); |
| 213 | + const eventFields = { |
| 214 | + aggregateKind: "project" as const, |
| 215 | + occurredAt: project.updatedAt, |
| 216 | + commandId: null, |
| 217 | + causationEventId: null, |
| 218 | + correlationId: null, |
| 219 | + metadata: {}, |
| 220 | + }; |
| 221 | + for (const shell of [project, laterProject]) { |
| 222 | + yield* commit({ |
| 223 | + ...eventFields, |
| 224 | + eventId: EventId.make(`created:${shell.id}`), |
| 225 | + aggregateId: shell.id, |
| 226 | + type: "project.created", |
| 227 | + payload: { ...shell, projectId: shell.id }, |
| 228 | + }); |
| 229 | + } |
| 230 | + const afterSequence = yield* store.latestApplicationSequence; |
| 231 | + // Newer-build rows cannot go through append. An entire unknown-only raw page |
| 232 | + // must not end replay before the known rows on the next page. |
| 233 | + if (unknownRows > 0) { |
| 234 | + yield* sql` |
| 235 | + WITH RECURSIVE history(n) AS ( |
| 236 | + SELECT 1 UNION ALL SELECT n + 1 FROM history WHERE n < ${unknownRows} |
| 237 | + ) |
| 238 | + INSERT INTO orchestration_events ( |
| 239 | + event_id, aggregate_kind, stream_id, stream_version, event_type, |
| 240 | + occurred_at, actor_kind, payload_json, metadata_json, application_event_version |
| 241 | + ) |
| 242 | + SELECT 'unknown:' || n, 'project', 'future-project', n, 'future.event', |
| 243 | + ${project.updatedAt}, 'server', 'not JSON', '{}', 2 FROM history |
| 244 | + `; |
| 245 | + } |
| 246 | + const replayEvents = []; |
| 247 | + for (const shell of [project, laterProject]) { |
| 248 | + replayEvents.push( |
| 249 | + yield* commit({ |
| 250 | + ...eventFields, |
| 251 | + eventId: EventId.make(`updated:${shell.id}`), |
| 252 | + aggregateId: shell.id, |
| 253 | + type: "project.meta-updated", |
| 254 | + payload: { projectId: shell.id, title: `Updated ${shell.id}`, updatedAt: shell.updatedAt }, |
| 255 | + }), |
| 256 | + ); |
| 257 | + } |
| 258 | + yield* sql` |
| 259 | + INSERT INTO orchestration_events ( |
| 260 | + event_id, aggregate_kind, stream_id, stream_version, event_type, |
| 261 | + occurred_at, actor_kind, payload_json, metadata_json, application_event_version |
| 262 | + ) VALUES ( |
| 263 | + 'unknown:tail', 'project', 'future-project', ${unknownRows + 1}, 'future.trailing', |
| 264 | + ${project.updatedAt}, 'server', 'not JSON', '{}', 2 |
| 265 | + ) |
| 266 | + `; |
| 267 | + const highWater = yield* store.latestApplicationSequence; |
| 268 | + expect(highWater - afterSequence).toBe(unknownRows + 3); |
| 269 | + const shells = yield* projects.listShells(); |
| 270 | + const liveAfterSequences: Array<number | undefined> = []; |
| 271 | + const dependencies = Layer.mergeAll( |
| 272 | + Layer.succeed(OrchestrationEventStore.OrchestrationEventStore, { |
| 273 | + ...store, |
| 274 | + readApplicationEvents: (input) => |
| 275 | + store |
| 276 | + .readApplicationEvents(input) |
| 277 | + .pipe(Stream.ensuring(Deferred.succeed(replayClosed, undefined))), |
| 278 | + streamProjectedApplicationEvents: (input) => |
| 279 | + Stream.suspend(() => { |
| 280 | + liveAfterSequences.push(input.afterSequence); |
| 281 | + return store.streamProjectedApplicationEvents(input); |
| 282 | + }), |
| 283 | + }), |
| 284 | + Layer.mock(ThreadManagementService.ThreadManagementService)({}), |
| 285 | + Layer.mock(ProjectService.ProjectService)({ |
| 286 | + getShell: (projectId) => |
| 287 | + Effect.gen(function* () { |
| 288 | + if (projectId === laterProject.id) { |
| 289 | + yield* Deferred.await(firstProjected); |
| 290 | + yield* Deferred.succeed(projectionParked, undefined); |
| 291 | + yield* Deferred.await(releaseProjection).pipe( |
| 292 | + Effect.ensuring(Deferred.succeed(projectionClosed, undefined)), |
| 293 | + ); |
| 294 | + } |
| 295 | + const shell = yield* projects.getShell(projectId).pipe( |
| 296 | + Effect.mapError( |
| 297 | + (cause) => |
| 298 | + new ProjectService.ProjectOperationError({ |
| 299 | + operation: "read-project", |
| 300 | + projectId, |
| 301 | + cause, |
| 302 | + }), |
| 303 | + ), |
| 304 | + ); |
| 305 | + if (projectId === project.id) yield* Deferred.succeed(firstProjected, undefined); |
| 306 | + return shell; |
| 307 | + }), |
| 308 | + }), |
| 309 | + Layer.mock(ProjectEnrichmentService.ProjectEnrichmentService)({ |
| 310 | + subscribeChanges: Effect.gen(function* () { |
| 311 | + yield* Effect.addFinalizer(() => Deferred.succeed(enrichmentClosed, undefined)); |
| 312 | + return yield* PubSub.subscribe(changes); |
| 313 | + }), |
| 314 | + getAvailable: () => |
| 315 | + Effect.succeed({ |
| 316 | + repositoryIdentity: null, |
| 317 | + faviconPath: null, |
| 318 | + repositoryIdentityResolved: true, |
| 319 | + }), |
| 320 | + }), |
| 321 | + ); |
| 322 | + const remove = (shell: typeof project) => |
| 323 | + commit({ |
| 324 | + ...eventFields, |
| 325 | + eventId: EventId.make(`deleted:${shell.id}`), |
| 326 | + aggregateId: shell.id, |
| 327 | + type: "project.deleted", |
| 328 | + payload: { projectId: shell.id, deletedAt: shell.updatedAt }, |
| 329 | + }); |
| 330 | + return { |
| 331 | + stream: Stream.unwrap( |
| 332 | + subscribeOrchestrationV2Shell({ afterSequence, requestCompletionMarker: true }), |
| 333 | + ).pipe(Stream.provide(dependencies)), |
| 334 | + highWater, |
| 335 | + shells, |
| 336 | + replayEvents, |
| 337 | + changes, |
| 338 | + laterProject, |
| 339 | + remove, |
| 340 | + projectionParked, |
| 341 | + releaseProjection, |
| 342 | + projectionClosed, |
| 343 | + replayClosed, |
| 344 | + enrichmentClosed, |
| 345 | + liveAfterSequences, |
| 346 | + }; |
| 347 | +}); |
| 348 | + |
| 349 | +describe("shell eager replay lifecycle", () => { |
| 350 | + it.live.each([0, 500])( |
| 351 | + "keeps racing live and enrichment updates after replay with %s unknown rows", |
| 352 | + (unknownRows) => |
| 353 | + Effect.gen(function* () { |
| 354 | + const fixture = yield* makeParkedReplay(unknownRows); |
| 355 | + const initialSent = yield* Deferred.make<void>(); |
| 356 | + const liveSent = yield* Deferred.make<void>(); |
| 357 | + const enrichmentSent = yield* Deferred.make<void>(); |
| 358 | + const sentinelSent = yield* Deferred.make<void>(); |
| 359 | + const chunks: Array<ReadonlyArray<OrchestrationV2ShellStreamItem>> = []; |
| 360 | + const consumer = yield* fixture.stream.pipe( |
| 361 | + Stream.runForEachArray((items) => |
| 362 | + Effect.gen(function* () { |
| 363 | + chunks.push([...items]); |
| 364 | + for (const item of items) { |
| 365 | + if (item.kind === "synchronized") yield* Deferred.succeed(initialSent, undefined); |
| 366 | + if (item.kind === "project.removed") { |
| 367 | + yield* Deferred.succeed( |
| 368 | + item.projectId === project.id ? liveSent : sentinelSent, |
| 369 | + undefined, |
| 370 | + ); |
| 371 | + } |
| 372 | + if ( |
| 373 | + item.kind === "snapshot" && |
| 374 | + item.snapshot.projects.some((shell) => shell.repositoryIdentity !== null) |
| 375 | + ) { |
| 376 | + yield* Deferred.succeed(enrichmentSent, undefined); |
| 377 | + } |
| 378 | + } |
| 379 | + }), |
| 380 | + ), |
| 381 | + Effect.scoped, |
| 382 | + Effect.forkChild, |
| 383 | + ); |
| 384 | + yield* Deferred.await(fixture.projectionParked); |
| 385 | + const newer = yield* fixture.remove(project); |
| 386 | + const identity = { |
| 387 | + canonicalKey: "github.com/test/replay", |
| 388 | + locator: { |
| 389 | + source: "git-remote" as const, |
| 390 | + remoteName: "origin", |
| 391 | + remoteUrl: "https://github.com/test/replay.git", |
| 392 | + }, |
| 393 | + }; |
| 394 | + yield* PubSub.publish(fixture.changes, { |
| 395 | + workspaceRoot: fixture.laterProject.workspaceRoot, |
| 396 | + repositoryIdentityResolved: true, |
| 397 | + enrichment: { |
| 398 | + repositoryIdentity: identity, |
| 399 | + faviconPath: null, |
| 400 | + repositoryIdentityResolved: true, |
| 401 | + }, |
| 402 | + }); |
| 403 | + expect(chunks).toEqual([]); |
| 404 | + expect(fixture.liveAfterSequences).toEqual([]); |
| 405 | + yield* Deferred.succeed(fixture.releaseProjection, undefined); |
| 406 | + yield* Deferred.await(initialSent); |
| 407 | + yield* Deferred.await(liveSent); |
| 408 | + yield* Deferred.await(enrichmentSent); |
| 409 | + // A later committed live event fences delivery before checking exact-once |
| 410 | + // recovery of the event published before the live subscription existed. |
| 411 | + const sentinel = yield* fixture.remove(fixture.laterProject); |
| 412 | + yield* Deferred.await(sentinelSent); |
| 413 | + yield* Fiber.interrupt(consumer); |
| 414 | + yield* Deferred.await(fixture.projectionClosed); |
| 415 | + yield* Deferred.await(fixture.replayClosed); |
| 416 | + yield* Deferred.await(fixture.enrichmentClosed); |
| 417 | + |
| 418 | + expect(fixture.liveAfterSequences).toEqual([fixture.highWater]); |
| 419 | + expect(chunks[0]).toEqual([ |
| 420 | + { |
| 421 | + kind: "snapshot", |
| 422 | + snapshot: { |
| 423 | + schemaVersion: ORCHESTRATION_V2_PROJECTION_SCHEMA_VERSION, |
| 424 | + snapshotSequence: fixture.highWater, |
| 425 | + projects: fixture.shells, |
| 426 | + threads: [], |
| 427 | + archivedThreads: [], |
| 428 | + }, |
| 429 | + resolvedRepositoryIdentityRoots: fixture.shells.map((shell) => shell.workspaceRoot), |
| 430 | + }, |
| 431 | + ...fixture.replayEvents.map((event, index) => ({ |
| 432 | + kind: "project.updated", |
| 433 | + sequence: event.sequence, |
| 434 | + project: fixture.shells[index], |
| 435 | + })), |
| 436 | + { kind: "synchronized" }, |
| 437 | + ]); |
| 438 | + const tail = chunks.slice(1).flat(); |
| 439 | + expect(tail.filter((item) => item.kind !== "snapshot")).toEqual([ |
| 440 | + { kind: "project.removed", sequence: newer.sequence, projectId: project.id }, |
| 441 | + { |
| 442 | + kind: "project.removed", |
| 443 | + sequence: sentinel.sequence, |
| 444 | + projectId: fixture.laterProject.id, |
| 445 | + }, |
| 446 | + ]); |
| 447 | + expect(tail.filter((item) => item.kind === "snapshot")).toEqual([ |
| 448 | + { |
| 449 | + kind: "snapshot", |
| 450 | + snapshot: { |
| 451 | + schemaVersion: ORCHESTRATION_V2_PROJECTION_SCHEMA_VERSION, |
| 452 | + snapshotSequence: newer.sequence, |
| 453 | + projects: [{ ...fixture.shells[1], repositoryIdentity: identity }], |
| 454 | + threads: [], |
| 455 | + archivedThreads: [], |
| 456 | + }, |
| 457 | + resolvedRepositoryIdentityRoots: [fixture.laterProject.workspaceRoot], |
| 458 | + }, |
| 459 | + ]); |
| 460 | + }).pipe(Effect.scoped, Effect.provide(ReplayTestLayer)), |
| 461 | + ); |
| 462 | + |
| 463 | + it.effect.each(["failure", "interruption"] as const)( |
| 464 | + "cleans up parked replay on %s without completion or live acquisition", |
| 465 | + (mode) => |
| 466 | + Effect.gen(function* () { |
| 467 | + const fixture = yield* makeParkedReplay(); |
| 468 | + const items: Array<OrchestrationV2ShellStreamItem> = []; |
| 469 | + const consumer = yield* fixture.stream.pipe( |
| 470 | + Stream.runForEach((item) => Effect.sync(() => items.push(item))), |
| 471 | + Effect.scoped, |
| 472 | + Effect.forkChild, |
| 473 | + ); |
| 474 | + yield* Deferred.await(fixture.projectionParked); |
| 475 | + expect(items).toEqual([]); |
| 476 | + expect(fixture.liveAfterSequences).toEqual([]); |
| 477 | + const failure = new ProjectService.ProjectOperationError({ |
| 478 | + operation: "read-project", |
| 479 | + projectId: fixture.laterProject.id, |
| 480 | + cause: new Error("projection unavailable"), |
| 481 | + }); |
| 482 | + if (mode === "failure") { |
| 483 | + yield* Deferred.fail(fixture.releaseProjection, failure); |
| 484 | + } else { |
| 485 | + yield* Fiber.interrupt(consumer); |
| 486 | + } |
| 487 | + const exit = yield* Fiber.await(consumer); |
| 488 | + expect(Exit.isFailure(exit)).toBe(true); |
| 489 | + if (Exit.isFailure(exit)) { |
| 490 | + if (mode === "failure") { |
| 491 | + expect(Option.getOrNull(Cause.findErrorOption(exit.cause))).toMatchObject({ |
| 492 | + _tag: "OrchestrationV2GetShellSnapshotError", |
| 493 | + message: "Failed to prepare the application shell stream", |
| 494 | + cause: failure, |
| 495 | + }); |
| 496 | + } else { |
| 497 | + expect(Cause.hasInterruptsOnly(exit.cause)).toBe(true); |
| 498 | + } |
| 499 | + } |
| 500 | + yield* Deferred.await(fixture.projectionClosed); |
| 501 | + yield* Deferred.await(fixture.replayClosed); |
| 502 | + yield* Deferred.await(fixture.enrichmentClosed); |
| 503 | + expect(items).toEqual([]); |
| 504 | + expect(fixture.liveAfterSequences).toEqual([]); |
| 505 | + }).pipe(Effect.scoped, Effect.provide(ReplayTestLayer)), |
| 506 | + ); |
| 507 | +}); |
0 commit comments