Skip to content

Commit 68892c0

Browse files
committed
fix(shared): preserve surrogate pairs in truncated CSV cells
1 parent 32ecd71 commit 68892c0

2 files changed

Lines changed: 43 additions & 2 deletions

File tree

‎packages/shared/src/delimitedPreview.test.ts‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,39 @@ describe("delimited file previews", () => {
5959
expect(table.rows[0]?.[0]).toHaveLength(2000);
6060
expect(parseDelimitedPreview('a,"unfinished', ",").truncated).toBe(true);
6161
});
62+
describe.each([",", "\t"] as const)("Unicode cells with delimiter %j", (delimiter) => {
63+
it.each([false, true])("does not split surrogate pairs in quoted=%s cells", (quoted) => {
64+
const prefix = "x".repeat(1999);
65+
const value = `${prefix}😀tail`;
66+
const input = `${quoted ? `"${value}"` : value}${delimiter}next\nlast`;
67+
expect(parseDelimitedPreview(input, delimiter)).toEqual({
68+
rows: [[prefix, "next"], ["last"]],
69+
truncated: true,
70+
});
71+
});
72+
it("keeps pairs that fit and truncates later characters", () => {
73+
const value = `${"x".repeat(1998)}😀`;
74+
expect(parseDelimitedPreview(value, delimiter)).toEqual({
75+
rows: [[value]],
76+
truncated: false,
77+
});
78+
expect(parseDelimitedPreview(`${value}tail`, delimiter)).toEqual({
79+
rows: [[value]],
80+
truncated: true,
81+
});
82+
});
83+
it("keeps escaped quotes and BMP text at the same limit", () => {
84+
const prefix = "x".repeat(1999);
85+
expect(parseDelimitedPreview(`"${prefix}""tail"`, delimiter)).toEqual({
86+
rows: [[`${prefix}"`]],
87+
truncated: true,
88+
});
89+
expect(parseDelimitedPreview(`${prefix}éextra`, delimiter)).toEqual({
90+
rows: [[`${prefix}é`]],
91+
truncated: true,
92+
});
93+
});
94+
});
6295
it("reads worker pages beyond the old row, column and cell limits", () => {
6396
const text = Array.from({ length: 128 }, (_, row) =>
6497
Array.from({ length: 40 }, (_, column) => `${row}:${column}`).join(","),

‎packages/shared/src/delimitedPreview.ts‎

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,14 @@ export function parseDelimitedPreview(
2424
let truncated = false;
2525
let rowStart = text.charCodeAt(0) === 0xfeff ? 1 : 0;
2626
const endCell = () => {
27+
if (cell.length > limits.maxCellLength) {
28+
let end = limits.maxCellLength;
29+
const before = cell.charCodeAt(end - 1);
30+
const after = cell.charCodeAt(end);
31+
if (before >= 0xd800 && before <= 0xdbff && after >= 0xdc00 && after <= 0xdfff) end--;
32+
cell = cell.slice(0, end);
33+
truncated = true;
34+
}
2735
if (row.length < limits.maxColumns) row.push(cell);
2836
else truncated = true;
2937
cell = "";
@@ -32,7 +40,7 @@ export function parseDelimitedPreview(
3240
const char = text[index];
3341
if (char === '"') {
3442
if (quoted && text[index + 1] === '"') {
35-
if (cell.length < limits.maxCellLength) cell += '"';
43+
if (cell.length <= limits.maxCellLength) cell += '"';
3644
else truncated = true;
3745
index++;
3846
continue;
@@ -52,7 +60,7 @@ export function parseDelimitedPreview(
5260
if (rows.length === limits.maxRows)
5361
return { rows, truncated: truncated || index < text.length - 1 };
5462
}
55-
} else if (cell.length < limits.maxCellLength) cell += char;
63+
} else if (cell.length <= limits.maxCellLength) cell += char;
5664
else truncated = true;
5765
}
5866
if (rowStart < text.length) {

0 commit comments

Comments
 (0)