Skip to content

Commit aefc7f0

Browse files
mcollinaaduh95
mcollina
authored and
aduh95
committed
buffer: add Buffer.stringLength()
Add `Buffer.stringLength(input[, encoding])`, the counterpart of `Buffer.byteLength()`: it returns the number of UTF-16 code units that `buf.toString(encoding)` would produce, without decoding. For UTF-8 the count is computed with simdutf. Invalid input is counted with the same maximal-subpart replacement that the decoder applies, so the result always matches `toString().length`. The other encodings are computed from `byteLength` alone. This lets code that accumulates streamed input check the result against `buffer.constants.MAX_STRING_LENGTH` and size its memory budget before decoding. Refs: #66062 Signed-off-by: Matteo Collina <[email protected]> PR-URL: #66064 Reviewed-By: Paolo Insogna <[email protected]> Reviewed-By: Gürgün Dayıoğlu <[email protected]>
1 parent d12cc45 commit aefc7f0

5 files changed

Lines changed: 340 additions & 0 deletions

File tree

‎benchmark/buffers/buffer-stringlength.js‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
'use strict';
2+
3+
const common = require('../common.js');
4+
const { Buffer } = require('node:buffer');
5+
const assert = require('node:assert');
6+
7+
const bench = common.createBenchmark(main, {
8+
n: [1e6],
9+
encoding: ['utf8', 'latin1', 'base64'],
10+
len: [32, 4096, 1048576],
11+
input: ['ascii', 'multibyte', 'invalid'],
12+
});
13+
14+
function main({ n, encoding, len, input }) {
15+
let buf;
16+
if (input === 'ascii') {
17+
buf = Buffer.alloc(len, 'a');
18+
} else {
19+
buf = Buffer.alloc(len - (len % 3), '€');
20+
if (input === 'invalid') buf = Buffer.concat([buf, Buffer.from([0xE2, 0x82])]);
21+
}
22+
const expected = buf.toString(encoding).length;
23+
bench.start();
24+
for (let i = 0; i < n; ++i) {
25+
assert.strictEqual(Buffer.stringLength(buf, encoding), expected);
26+
}
27+
bench.end(n);
28+
}

‎doc/api/buffer.md‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1001,6 +1001,59 @@ console.log(`${str}: ${str.length} characters, ` +
10011001
When `string` is a {Buffer|DataView|TypedArray|ArrayBuffer|SharedArrayBuffer},
10021002
the byte length as reported by `.byteLength` is returned.
10031003

1004+
### Static method: `Buffer.stringLength(input[, encoding])`
1005+
1006+
<!-- YAML
1007+
added: REPLACEME
1008+
-->
1009+
1010+
* `input` {Buffer | ArrayBuffer | TypedArray} The bytes that would be decoded.
1011+
* `encoding` {string} The character encoding `input` would be decoded with.
1012+
**Default:** `'utf8'`.
1013+
* Returns: {integer}
1014+
1015+
Returns the length, in UTF-16 code units, of the string that
1016+
`buf.toString(encoding)` would produce for the same bytes, without decoding
1017+
them. This is the counterpart of [`Buffer.byteLength()`][], which returns the
1018+
number of bytes a string would encode to.
1019+
1020+
For `'utf8'`, invalid byte sequences are counted as they would be decoded:
1021+
each maximal invalid subsequence becomes one `U+FFFD` replacement character.
1022+
For every other encoding the result is computed from `input.byteLength` alone.
1023+
1024+
A detached `ArrayBuffer`, or a `TypedArray` backed by one, is treated as empty.
1025+
1026+
The result is not capped: compare it with
1027+
[`buffer.constants.MAX_STRING_LENGTH`][] before decoding to know whether the
1028+
decode can succeed at all. A string of `n` code units occupies between `n` and
1029+
`2 * n` bytes of memory.
1030+
1031+
```mjs
1032+
import { Buffer, constants } from 'node:buffer';
1033+
1034+
const buf = Buffer.from('€ 100', 'utf8');
1035+
1036+
console.log(Buffer.stringLength(buf));
1037+
// Prints: 5
1038+
console.log(Buffer.stringLength(buf, 'hex'));
1039+
// Prints: 14
1040+
console.log(Buffer.stringLength(buf) <= constants.MAX_STRING_LENGTH);
1041+
// Prints: true
1042+
```
1043+
1044+
```cjs
1045+
const { Buffer, constants } = require('node:buffer');
1046+
1047+
const buf = Buffer.from('€ 100', 'utf8');
1048+
1049+
console.log(Buffer.stringLength(buf));
1050+
// Prints: 5
1051+
console.log(Buffer.stringLength(buf, 'hex'));
1052+
// Prints: 14
1053+
console.log(Buffer.stringLength(buf) <= constants.MAX_STRING_LENGTH);
1054+
// Prints: true
1055+
```
1056+
10041057
### Static method: `Buffer.compare(buf1, buf2)`
10051058

10061059
<!-- YAML
@@ -5629,6 +5682,7 @@ introducing security vulnerabilities into an application.
56295682
[`Buffer.alloc()`]: #static-method-bufferallocsize-fill-encoding
56305683
[`Buffer.allocUnsafe()`]: #static-method-bufferallocunsafesize
56315684
[`Buffer.allocUnsafeSlow()`]: #static-method-bufferallocunsafeslowsize
5685+
[`Buffer.byteLength()`]: #static-method-bufferbytelengthstring-encoding
56325686
[`Buffer.concat()`]: #static-method-bufferconcatlist-totallength
56335687
[`Buffer.copyBytesFrom()`]: #static-method-buffercopybytesfromview-offset-length
56345688
[`Buffer.from(array)`]: #static-method-bufferfromarray

‎lib/buffer.js‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ const {
2626
ArrayBufferIsView,
2727
ArrayIsArray,
2828
ArrayPrototypeForEach,
29+
MathCeil,
2930
MathFloor,
3031
MathMin,
3132
MathTrunc,
@@ -63,6 +64,7 @@ const {
6364
fill: bindingFill,
6465
isAscii: bindingIsAscii,
6566
isUtf8: bindingIsUtf8,
67+
stringLengthUtf8: bindingStringLengthUtf8,
6668
indexOfBuffer,
6769
indexOfNumber,
6870
indexOfString,
@@ -885,6 +887,7 @@ function byteLength(string, encoding) {
885887
}
886888

887889
Buffer.byteLength = byteLength;
890+
Buffer.stringLength = stringLength;
888891

889892
// For backwards compatibility.
890893
ObjectDefineProperty(Buffer.prototype, 'parent', {
@@ -1442,6 +1445,32 @@ function isAscii(input) {
14421445
throw new ERR_INVALID_ARG_TYPE('input', ['ArrayBuffer', 'Buffer', 'TypedArray'], input);
14431446
}
14441447

1448+
function stringLength(input, encoding = 'utf8') {
1449+
if (!isTypedArray(input) && !isAnyArrayBuffer(input)) {
1450+
throw new ERR_INVALID_ARG_TYPE('input', ['ArrayBuffer', 'Buffer', 'TypedArray'], input);
1451+
}
1452+
validateString(encoding, 'encoding');
1453+
const ops = getEncodingOps(encoding);
1454+
if (ops === undefined) {
1455+
throw new ERR_UNKNOWN_ENCODING(encoding);
1456+
}
1457+
const length = input.byteLength;
1458+
switch (ops.encodingVal) {
1459+
case encodingsMap.utf8:
1460+
return length === 0 ? 0 : bindingStringLengthUtf8(input);
1461+
case encodingsMap.utf16le:
1462+
return MathFloor(length / 2);
1463+
case encodingsMap.hex:
1464+
return length * 2;
1465+
case encodingsMap.base64:
1466+
return MathCeil(length / 3) * 4;
1467+
case encodingsMap.base64url:
1468+
return MathCeil(length * 4 / 3);
1469+
default: // latin1, ascii
1470+
return length;
1471+
}
1472+
}
1473+
14451474
module.exports = {
14461475
Buffer,
14471476
SlowBuffer: deprecate(

‎src/node_buffer.cc‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1355,6 +1355,94 @@ static bool FastIsAscii(Local<Value> receiver,
13551355

13561356
static CFunction fast_is_ascii(CFunction::Make(FastIsAscii));
13571357

1358+
// Number of UTF-16 code units produced by decoding [p, end) as UTF-8 with
1359+
// WHATWG "maximal subpart" U+FFFD replacement, matching the fallback that
1360+
// StringBytes::Encode takes for invalid input (v8::String::NewFromUtf8).
1361+
static size_t Utf16LengthFromInvalidUtf8(const uint8_t* p, const uint8_t* end) {
1362+
size_t units = 0;
1363+
while (p < end) {
1364+
const uint8_t lead = *p;
1365+
if (lead < 0x80) {
1366+
p++;
1367+
units++;
1368+
continue;
1369+
}
1370+
size_t len;
1371+
uint8_t lo = 0x80;
1372+
uint8_t hi = 0xBF;
1373+
if (lead >= 0xC2 && lead <= 0xDF) {
1374+
len = 2;
1375+
} else if (lead >= 0xE0 && lead <= 0xEF) {
1376+
len = 3;
1377+
if (lead == 0xE0) lo = 0xA0;
1378+
if (lead == 0xED) hi = 0x9F;
1379+
} else if (lead >= 0xF0 && lead <= 0xF4) {
1380+
len = 4;
1381+
if (lead == 0xF0) lo = 0x90;
1382+
if (lead == 0xF4) hi = 0x8F;
1383+
} else {
1384+
// Invalid lead byte: one replacement character.
1385+
p++;
1386+
units++;
1387+
continue;
1388+
}
1389+
size_t i = 1;
1390+
for (; i < len && p + i < end; i++) {
1391+
const uint8_t c = p[i];
1392+
if (i == 1 ? (c < lo || c > hi) : (c < 0x80 || c > 0xBF)) break;
1393+
}
1394+
if (i == len) {
1395+
p += len;
1396+
units += (len == 4) ? 2 : 1;
1397+
} else {
1398+
// The lead byte plus the valid continuation bytes seen so far form the
1399+
// maximal subpart and become one replacement character; the byte that
1400+
// failed is decoded again on the next iteration.
1401+
p += i;
1402+
units++;
1403+
}
1404+
}
1405+
return units;
1406+
}
1407+
1408+
static double StringLengthUtf8Impl(Local<Value> value) {
1409+
ArrayBufferViewContents<uint8_t> abv(value);
1410+
const uint8_t* data = abv.data();
1411+
const size_t length = abv.length();
1412+
if (length == 0) return 0;
1413+
const simdutf::result r = simdutf::validate_utf8_with_errors(
1414+
reinterpret_cast<const char*>(data), length);
1415+
if (r.error == simdutf::error_code::SUCCESS) {
1416+
return static_cast<double>(simdutf::utf16_length_from_utf8(
1417+
reinterpret_cast<const char*>(data), length));
1418+
}
1419+
// r.count is the offset of the first invalid sequence; everything before it
1420+
// is valid UTF-8.
1421+
const size_t valid = simdutf::utf16_length_from_utf8(
1422+
reinterpret_cast<const char*>(data), r.count);
1423+
return static_cast<double>(
1424+
valid + Utf16LengthFromInvalidUtf8(data + r.count, data + length));
1425+
}
1426+
1427+
static void StringLengthUtf8(const FunctionCallbackInfo<Value>& args) {
1428+
CHECK_EQ(args.Length(), 1);
1429+
CHECK(args[0]->IsTypedArray() || args[0]->IsArrayBuffer() ||
1430+
args[0]->IsSharedArrayBuffer());
1431+
1432+
args.GetReturnValue().Set(StringLengthUtf8Impl(args[0]));
1433+
}
1434+
1435+
static double FastStringLengthUtf8(Local<Value> receiver,
1436+
Local<Value> value,
1437+
// NOLINTNEXTLINE(runtime/references)
1438+
FastApiCallbackOptions& options) {
1439+
TRACK_V8_FAST_API_CALL("buffer.stringLengthUtf8");
1440+
HandleScope scope(options.isolate);
1441+
return StringLengthUtf8Impl(value);
1442+
}
1443+
1444+
static CFunction fast_string_length_utf8(CFunction::Make(FastStringLengthUtf8));
1445+
13581446
void SetBufferPrototype(const FunctionCallbackInfo<Value>& args) {
13591447
Realm* realm = Realm::GetCurrent(args);
13601448

@@ -1738,6 +1826,11 @@ void Initialize(Local<Object> target,
17381826
SetFastMethodNoSideEffect(context, target, "isUtf8", IsUtf8, &fast_is_utf8);
17391827
SetFastMethodNoSideEffect(
17401828
context, target, "isAscii", IsAscii, &fast_is_ascii);
1829+
SetFastMethodNoSideEffect(context,
1830+
target,
1831+
"stringLengthUtf8",
1832+
StringLengthUtf8,
1833+
&fast_string_length_utf8);
17411834

17421835
target
17431836
->Set(context,
@@ -1813,6 +1906,8 @@ void RegisterExternalReferences(ExternalReferenceRegistry* registry) {
18131906
registry->Register(fast_is_utf8);
18141907
registry->Register(IsAscii);
18151908
registry->Register(fast_is_ascii);
1909+
registry->Register(StringLengthUtf8);
1910+
registry->Register(fast_string_length_utf8);
18161911

18171912
registry->Register(StringSlice<ASCII>);
18181913
registry->Register(StringSlice<BASE64>);

0 commit comments

Comments
 (0)