Skip to content

Commit d625e4d

Browse files
Merge main into fix/preview-annotation-keyboard
2 parents d3b7e8b + 99c692b commit d625e4d

83 files changed

Lines changed: 3589 additions & 740 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎apps/desktop/src/preview/Manager.test.ts‎

Lines changed: 194 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ import { DESKTOP_PREVIEW_RECORDING_CAPTURE_TRIGGER } from "@t3tools/contracts";
44
import type {
55
DesktopPreviewRecordingFrame,
66
DesktopPreviewRecordingInputEvent,
7+
PreviewAnnotationPayload,
8+
PreviewAnnotationRect,
79
} from "@t3tools/contracts";
810
import * as HostProcess from "@t3tools/shared/HostProcess";
911
import { parseKeybindingShortcut } from "@t3tools/shared/keybindings";
@@ -4359,68 +4361,208 @@ describe("PreviewManager", () => {
43594361
),
43604362
);
43614363

4362-
effectIt.effect("crops the picked area out of a full-page capture at device scale", () =>
4363-
withManager((manager) =>
4364-
Effect.gen(function* () {
4365-
let onPicked: ((event: unknown, ...args: unknown[]) => void) | undefined;
4366-
const crop = vi.fn((rect: Electron.Rectangle) => ({
4367-
getSize: () => ({ width: rect.width, height: rect.height }),
4368-
toDataURL: () => "data:image/png;base64,cropped",
4369-
}));
4370-
const capturePage = vi.fn(async () => ({
4371-
getSize: () => ({ width: 2000, height: 1200 }),
4372-
crop,
4373-
}));
4374-
fromId.mockReturnValue(
4375-
Object.assign(makeTestPreviewWebContents(capturePage as never), {
4376-
isFocused: () => true,
4377-
once: vi.fn(),
4378-
ipc: {
4379-
on: vi.fn((channel: string, listener: typeof onPicked) => {
4380-
if (channel === "preview:element-picked") onPicked = listener;
4381-
}),
4382-
off: vi.fn(),
4383-
removeListener: vi.fn(),
4364+
describe("annotation screenshots", () => {
4365+
const captureAnnotation = (
4366+
cropRect: PreviewAnnotationRect | null,
4367+
targets: Partial<Pick<PreviewAnnotationPayload, "elements" | "regions" | "strokes">> = {},
4368+
emptyImage?: "viewport" | "crop",
4369+
) =>
4370+
withManager((manager) =>
4371+
Effect.gen(function* () {
4372+
let onPicked: ((event: unknown, ...args: unknown[]) => void) | undefined;
4373+
const crop = vi.fn((rect: Electron.Rectangle) => ({
4374+
isEmpty: () => emptyImage === "crop",
4375+
getSize: () =>
4376+
emptyImage === "crop"
4377+
? { width: 0, height: 0 }
4378+
: { width: rect.width, height: rect.height },
4379+
toDataURL: () => (emptyImage === "crop" ? "data:," : "data:image/png;base64,cropped"),
4380+
}));
4381+
const capturePage = vi.fn(async () => ({
4382+
isEmpty: () => emptyImage === "viewport",
4383+
getSize: () =>
4384+
emptyImage === "viewport" ? { width: 0, height: 0 } : { width: 2000, height: 1200 },
4385+
toJPEG: () => Buffer.from("unused"),
4386+
toDataURL: () => (emptyImage === "viewport" ? "data:," : "data:image/png;base64,full"),
4387+
crop,
4388+
}));
4389+
fromId.mockReturnValue(
4390+
Object.assign(makeTestPreviewWebContents(capturePage), {
4391+
isFocused: () => true,
4392+
once: vi.fn(),
4393+
ipc: {
4394+
on: vi.fn((channel: string, listener: typeof onPicked) => {
4395+
if (channel === "preview:element-picked") onPicked = listener;
4396+
}),
4397+
off: vi.fn(),
4398+
removeListener: vi.fn(),
4399+
},
4400+
}),
4401+
);
4402+
4403+
yield* manager.createTab("tab_1");
4404+
yield* manager.registerWebview("tab_1", 42);
4405+
const pick = yield* manager.pickElement("tab_1").pipe(Effect.forkChild);
4406+
yield* Effect.yieldNow;
4407+
onPicked?.(
4408+
{},
4409+
{
4410+
id: "annotation_1",
4411+
pageUrl: "https://example.com",
4412+
pageTitle: "Example",
4413+
comment: "Tighten this spacing",
4414+
elements: [],
4415+
regions: [],
4416+
strokes: [],
4417+
...targets,
4418+
styleChanges: [],
4419+
screenshot: null,
4420+
createdAt: "2026-06-11T00:00:00.000Z",
43844421
},
4385-
}),
4386-
);
4422+
cropRect,
4423+
"attach",
4424+
2,
4425+
);
43874426

4388-
yield* manager.createTab("tab_1");
4389-
yield* manager.registerWebview("tab_1", 42);
4390-
const pick = yield* manager.pickElement("tab_1").pipe(Effect.forkChild);
4391-
yield* Effect.yieldNow;
4392-
onPicked?.(
4393-
{},
4394-
{
4395-
id: "annotation_1",
4396-
pageUrl: "https://example.com",
4397-
pageTitle: "Example",
4398-
comment: "Tighten this spacing",
4399-
elements: [],
4400-
regions: [{ id: "region_1", rect: { x: 5, y: 6, width: 20, height: 30 } }],
4401-
strokes: [],
4402-
styleChanges: [],
4403-
screenshot: null,
4404-
createdAt: "2026-06-11T00:00:00.000Z",
4405-
},
4406-
{ x: 5, y: 6, width: 20, height: 30 },
4407-
"attach",
4408-
2,
4409-
);
4427+
const result = yield* Fiber.join(pick);
4428+
expect(capturePage).toHaveBeenCalledExactlyOnceWith();
4429+
expect(result?.annotation.comment).toBe("Tighten this spacing");
4430+
expect(result?.submission).toBe("attach");
4431+
return { result, crop };
4432+
}),
4433+
);
44104434

4411-
const result = yield* Fiber.join(pick);
4412-
expect(capturePage).toHaveBeenCalledWith();
4413-
expect(crop).toHaveBeenCalledWith({ x: 10, y: 12, width: 40, height: 60 });
4435+
effectIt.effect("crops an on-screen region at device scale", () =>
4436+
Effect.gen(function* () {
4437+
const rect = { x: 5, y: 6, width: 20, height: 30 };
4438+
const { result, crop } = yield* captureAnnotation(rect, {
4439+
regions: [{ id: "region_1", rect }],
4440+
});
4441+
expect(crop).toHaveBeenCalledExactlyOnceWith({ x: 10, y: 12, width: 40, height: 60 });
44144442
expect(result?.screenshotFailed).toBeUndefined();
44154443
expect(result?.annotation.screenshot).toEqual({
44164444
dataUrl: "data:image/png;base64,cropped",
44174445
width: 40,
44184446
height: 60,
4419-
cropRect: { x: 5, y: 6, width: 20, height: 30 },
4447+
cropRect: rect,
44204448
});
44214449
}),
4422-
),
4423-
);
4450+
);
4451+
4452+
effectIt.effect("clips a partially visible selection to the captured viewport", () =>
4453+
Effect.gen(function* () {
4454+
const { result, crop } = yield* captureAnnotation(
4455+
{ x: -40, y: 570, width: 240, height: 110 },
4456+
{ regions: [{ id: "region_1", rect: { x: -20, y: 590, width: 200, height: 70 } }] },
4457+
);
4458+
expect(crop).toHaveBeenCalledExactlyOnceWith({ x: 0, y: 1140, width: 400, height: 60 });
4459+
expect(result?.screenshotFailed).toBeUndefined();
4460+
expect(result?.annotation.screenshot).toMatchObject({
4461+
dataUrl: "data:image/png;base64,cropped",
4462+
width: 400,
4463+
height: 60,
4464+
});
4465+
}),
4466+
);
4467+
4468+
effectIt.effect.each([
4469+
{
4470+
name: "an offscreen crop without individual target bounds",
4471+
cropRect: { x: 1010, y: 40, width: 100, height: 70 },
4472+
regions: [],
4473+
},
4474+
{
4475+
name: "padding overlapping the viewport while its target is offscreen",
4476+
cropRect: { x: 40, y: -100, width: 240, height: 110 },
4477+
regions: [{ id: "region_1", rect: { x: 60, y: -80, width: 200, height: 70 } }],
4478+
},
4479+
{
4480+
name: "a union spanning the viewport with both targets offscreen",
4481+
cropRect: { x: -230, y: 40, width: 1460, height: 110 },
4482+
regions: [
4483+
{ id: "region_1", rect: { x: -210, y: 60, width: 200, height: 70 } },
4484+
{ id: "region_2", rect: { x: 1010, y: 60, width: 200, height: 70 } },
4485+
],
4486+
},
4487+
{
4488+
name: "an offscreen target when crop bounds are missing",
4489+
cropRect: null,
4490+
regions: [{ id: "region_1", rect: { x: 60, y: -80, width: 200, height: 70 } }],
4491+
},
4492+
])("does not attach $name", ({ cropRect, regions }) =>
4493+
Effect.gen(function* () {
4494+
const { result, crop } = yield* captureAnnotation(cropRect, { regions });
4495+
expect(crop).not.toHaveBeenCalled();
4496+
expect(result?.annotation.screenshot).toBeNull();
4497+
expect(result?.screenshotFailed).toBe(true);
4498+
}),
4499+
);
4500+
4501+
effectIt.effect.each([
4502+
{
4503+
name: "zero-width SVG line on the left edge",
4504+
rect: { x: 0, y: 60, width: 0, height: 70 },
4505+
cropRect: { x: -20, y: 40, width: 40, height: 110 },
4506+
expectedCrop: { x: 0, y: 80, width: 40, height: 220 },
4507+
},
4508+
{
4509+
name: "zero-height SVG line on the top edge",
4510+
rect: { x: 60, y: 0, width: 70, height: 0 },
4511+
cropRect: { x: 40, y: -20, width: 110, height: 40 },
4512+
expectedCrop: { x: 80, y: 0, width: 220, height: 40 },
4513+
},
4514+
])("captures a stroked $name", ({ rect, cropRect, expectedCrop }) =>
4515+
Effect.gen(function* () {
4516+
const { result, crop } = yield* captureAnnotation(cropRect, {
4517+
elements: [
4518+
{
4519+
id: "element_1",
4520+
rect,
4521+
element: {
4522+
pageUrl: "https://example.com",
4523+
pageTitle: "Example",
4524+
tagName: "line",
4525+
selector: "svg > line",
4526+
htmlPreview: '<line stroke="black" stroke-width="2" />',
4527+
componentName: null,
4528+
source: null,
4529+
stack: [],
4530+
styles: "stroke: black; stroke-width: 2px;",
4531+
pickedAt: "2026-06-11T00:00:00.000Z",
4532+
},
4533+
},
4534+
],
4535+
});
4536+
expect(crop).toHaveBeenCalledExactlyOnceWith(expectedCrop);
4537+
expect(result?.screenshotFailed).toBeUndefined();
4538+
expect(result?.annotation.screenshot).toMatchObject({
4539+
dataUrl: "data:image/png;base64,cropped",
4540+
width: expectedCrop.width,
4541+
height: expectedCrop.height,
4542+
});
4543+
}),
4544+
);
4545+
4546+
effectIt.effect.each(["viewport", "crop"] as const)(
4547+
"reports an empty native %s image as a failed screenshot",
4548+
(emptyImage) =>
4549+
Effect.gen(function* () {
4550+
const rect = { x: 5, y: 6, width: 20, height: 30 };
4551+
const { result, crop } = yield* captureAnnotation(
4552+
rect,
4553+
{
4554+
regions: [{ id: "region_1", rect }],
4555+
},
4556+
emptyImage,
4557+
);
4558+
if (emptyImage === "viewport") expect(crop).not.toHaveBeenCalled();
4559+
else
4560+
expect(crop).toHaveBeenCalledExactlyOnceWith({ x: 10, y: 12, width: 40, height: 60 });
4561+
expect(result?.annotation.screenshot).toBeNull();
4562+
expect(result?.screenshotFailed).toBe(true);
4563+
}),
4564+
);
4565+
});
44244566

44254567
effectIt.effect("a stale capture from a replaced pick never touches the next pick", () =>
44264568
withManager((manager) =>

‎apps/desktop/src/preview/Manager.ts‎

Lines changed: 39 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -270,24 +270,35 @@ const normalizeCaptureRect = (value: unknown): PreviewAnnotationRect | null => {
270270
) {
271271
return null;
272272
}
273-
return {
274-
x: Math.max(0, Math.floor(x)),
275-
y: Math.max(0, Math.floor(y)),
276-
width: Math.max(1, Math.ceil(width)),
277-
height: Math.max(1, Math.ceil(height)),
278-
};
273+
return { x, y, width, height };
279274
};
280275

281-
/**
282-
* Crops a full-page capture to a picked annotation. `cropRect` is in CSS px;
283-
* the capture's pixels are CSS px × the guest's `devicePixelRatio`.
284-
*/
276+
// capturePage returns a 1x NativeImage containing device pixels, so crop() uses
277+
// CSS coordinates scaled by the guest's devicePixelRatio (including page zoom).
285278
const cropAnnotationScreenshot = (
286279
image: Electron.NativeImage,
287280
cropRect: PreviewAnnotationRect | null,
288281
devicePixelRatio: number,
282+
targetRects: ReadonlyArray<PreviewAnnotationRect>,
289283
): PreviewAnnotationPayload["screenshot"] => {
290284
const full = image.getSize();
285+
if (full.width <= 0 || full.height <= 0) return null;
286+
const scale = Number.isFinite(devicePixelRatio) && devicePixelRatio > 0 ? devicePixelRatio : 1;
287+
// Padding or a union of offscreen targets can overlap an empty viewport area.
288+
// A zero-width/height SVG line can still paint at the top or left edge.
289+
if (
290+
targetRects.length > 0 &&
291+
!targetRects.some(
292+
(rect) =>
293+
(rect.width > 0 || rect.height > 0) &&
294+
rect.x * scale < full.width &&
295+
rect.y * scale < full.height &&
296+
(rect.width === 0 ? rect.x >= 0 : rect.x + rect.width > 0) &&
297+
(rect.height === 0 ? rect.y >= 0 : rect.y + rect.height > 0),
298+
)
299+
) {
300+
return null;
301+
}
291302
if (!cropRect) {
292303
return {
293304
dataUrl: image.toDataURL(),
@@ -296,16 +307,21 @@ const cropAnnotationScreenshot = (
296307
cropRect: { x: 0, y: 0, width: full.width, height: full.height },
297308
};
298309
}
299-
const scale = Number.isFinite(devicePixelRatio) && devicePixelRatio > 0 ? devicePixelRatio : 1;
300-
const x = Math.min(full.width - 1, Math.max(0, Math.floor(cropRect.x * scale)));
301-
const y = Math.min(full.height - 1, Math.max(0, Math.floor(cropRect.y * scale)));
310+
const x = Math.max(0, Math.floor(cropRect.x * scale));
311+
const y = Math.max(0, Math.floor(cropRect.y * scale));
312+
const right = Math.min(full.width, Math.ceil((cropRect.x + cropRect.width) * scale));
313+
const bottom = Math.min(full.height, Math.ceil((cropRect.y + cropRect.height) * scale));
314+
// A target scrolled out of view has no pixels in this capture. Clamping its
315+
// origin onto the edge would attach an unrelated strip as a successful crop.
316+
if (right <= x || bottom <= y) return null;
302317
const cropped = image.crop({
303318
x,
304319
y,
305-
width: Math.max(1, Math.min(full.width - x, Math.ceil(cropRect.width * scale))),
306-
height: Math.max(1, Math.min(full.height - y, Math.ceil(cropRect.height * scale))),
320+
width: right - x,
321+
height: bottom - y,
307322
});
308323
const size = cropped.getSize();
324+
if (size.width <= 0 || size.height <= 0) return null;
309325
return { dataUrl: cropped.toDataURL(), width: size.width, height: size.height, cropRect };
310326
};
311327

@@ -2220,13 +2236,20 @@ const makeNativeOperations = Effect.fn("PreviewManager.makeOperations")(function
22202236
const submission =
22212237
args[2] === "send" && annotationSendEnabled.get(tabId) === true ? "send" : "attach";
22222238
const devicePixelRatio = typeof args[3] === "number" ? args[3] : 1;
2239+
const targetRects = [
2240+
...payload.elements.map((target) => target.rect),
2241+
...payload.regions.map((target) => target.rect),
2242+
...payload.strokes.map((target) => target.bounds),
2243+
];
22232244
runFork(
22242245
capturePageWithRetry(
22252246
{ operation: "captureAnnotationScreenshot", tabId, webContentsId: wc.id },
22262247
tabId,
22272248
wc,
22282249
).pipe(
2229-
Effect.map((image) => cropAnnotationScreenshot(image, cropRect, devicePixelRatio)),
2250+
Effect.map((image) =>
2251+
cropAnnotationScreenshot(image, cropRect, devicePixelRatio, targetRects),
2252+
),
22302253
Effect.tapError((error) =>
22312254
Effect.logWarning("preview annotation screenshot failed").pipe(
22322255
Effect.annotateLogs({ tabId, webContentsId: wc.id, error: error.message }),

‎apps/desktop/src/preview/PickPreload.ts‎

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -258,15 +258,12 @@ function unionRects(
258258
const top = Math.min(...rects.map((rect) => rect.y));
259259
const right = Math.max(...rects.map((rect) => rect.x + rect.width));
260260
const bottom = Math.max(...rects.map((rect) => rect.y + rect.height));
261-
const x = Math.max(0, left - padding);
262-
const y = Math.max(0, top - padding);
263-
const maxWidth = Math.max(1, window.innerWidth - x);
264-
const maxHeight = Math.max(1, window.innerHeight - y);
261+
// Main clips to the captured viewport; clipping here would lose offscreen bounds.
265262
return {
266-
x,
267-
y,
268-
width: Math.min(maxWidth, right - left + padding * 2),
269-
height: Math.min(maxHeight, bottom - top + padding * 2),
263+
x: left - padding,
264+
y: top - padding,
265+
width: right - left + padding * 2,
266+
height: bottom - top + padding * 2,
270267
};
271268
}
272269

@@ -1447,7 +1444,7 @@ function startAnnotation(sendEnabled: boolean): void {
14471444
annotation,
14481445
screenshotRect,
14491446
submission === "send" && !sendEnabled ? "attach" : submission,
1450-
// Main crops a full-page capture, whose pixels are CSS px × this.
1447+
// Main crops a viewport capture, whose pixels are CSS px × this.
14511448
window.devicePixelRatio,
14521449
);
14531450
})

0 commit comments

Comments
 (0)