Skip to content

Commit 8dcf14f

Browse files
eliau2005v8-scoped@luci-project-accounts.iam.gserviceaccount.com
authored andcommitted
[parser] Create debug magic comment values as source substrings
The values of the //# sourceURL, sourceMappingURL and debugId magic comments were copied into LiteralBuffers while scanning and then internalized. An inline source map (sourceMappingURL=data:...) can be tens of MiB, so this allocated a string of that size on the heap in a non-throwing allocation, e.g. aborting a Node.js worker with a small old-generation limit even though the (external) source fits. The scanner now only records the source locations of these values, and the values are created as substrings of the script source: - On the main thread, when parsing finishes. - For background and streaming compiles, where the source is not available while parsing, the locations are kept in the BackgroundCompileTask and the substrings are created in FinalizeScript after the source is set. Locations outside of the source provided by the embedder are ignored. The semantics are unchanged: the last comment wins, trailing garbage or an empty value removes the value, and a source map URL provided through the API takes precedence. Since these values are now usually SlicedStrings of the script source, the code serializer serializes all sliced strings as sequential strings (one-byte if their contents are, even if their parent is two-byte). The script source is an attached object, and off-thread deserialization attaches a placeholder for it, so a serialized slice of it would not be valid. Consuming a code cache still materializes these values as sequential strings, e.g. on Node.js's NODE_COMPILE_CACHE path, so a huge cached value can still be large on the heap there. Keeping them as substrings of the source when deserializing requires a code cache format change and is left for a follow-up. This also removes the LiteralBuffer copies of the values. [email protected], [email protected] Bug: 566467883 Change-Id: I07d8d0c36f56c7aee40793b1db272657832ab36f Refs: nodejs/node#64155 Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/8472768 Reviewed-by: Leszek Swirski <[email protected]> Reviewed-by: Patrick Thier <[email protected]> Auto-Submit: eliau elkouby (‫אליהו אלקובי‬‎) <[email protected]> Commit-Queue: Patrick Thier <[email protected]> Cr-Commit-Position: refs/heads/main@{#110612}
1 parent 118296c commit 8dcf14f

16 files changed

Lines changed: 730 additions & 108 deletions

‎src/codegen/compiler.cc‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2047,6 +2047,9 @@ void BackgroundCompileTask::Run(
20472047
parser.ParseOnBackground(isolate, &info, script_, start_position_,
20482048
end_position_, function_literal_id_);
20492049
parser.UpdateStatistics(script_, &use_counts_, &total_preparse_skipped_);
2050+
if (toplevel_script_compilation) {
2051+
debug_magic_comment_locations_ = parser.debug_magic_comment_locations();
2052+
}
20502053

20512054
TRACE_EVENT(TRACE_DISABLED_BY_DEFAULT("v8.compile"),
20522055
"V8.CompileCodeBackground");
@@ -2790,8 +2793,14 @@ MaybeHandle<SharedFunctionInfo> BackgroundCompileTask::FinalizeScript(
27902793
{
27912794
DisallowGarbageCollection no_gc;
27922795
SetScriptFieldsFromDetails(isolate, *script, script_details, &no_gc);
2793-
LOG(isolate, ScriptDetails(*script));
27942796
}
2797+
// After SetScriptFieldsFromDetails, so that a source map URL provided
2798+
// through the API takes precedence. The locations come from the streamed
2799+
// source; the embedder is expected to pass the same source here, but an
2800+
// out-of-bounds location is ignored.
2801+
Parser::SetDebugMagicCommentFields(isolate, script, source,
2802+
debug_magic_comment_locations_);
2803+
LOG(isolate, ScriptDetails(*script));
27952804
}
27962805

27972806
ReportStatistics(isolate);

‎src/codegen/compiler.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
#include "src/objects/debug-objects.h"
2525
#include "src/parsing/parse-info.h"
2626
#include "src/parsing/pending-compilation-error-handler.h"
27+
#include "src/parsing/scanner.h" // For Scanner::DebugMagicCommentLocations.
2728
#include "src/snapshot/code-serializer.h"
2829
#include "src/utils/allocation.h"
2930
#include "src/zone/zone.h"
@@ -652,6 +653,9 @@ class V8_EXPORT_PRIVATE BackgroundCompileTask {
652653
DeferredFinalizationJobDataList jobs_to_retry_finalization_on_main_thread_;
653654
base::SmallVector<v8::Isolate::UseCounterFeature, 8> use_counts_;
654655
int total_preparse_skipped_ = 0;
656+
// The source is not available on the background thread, so the debug magic
657+
// comment values are created from these locations when finalizing.
658+
Scanner::DebugMagicCommentLocations debug_magic_comment_locations_;
655659

656660
// Single function data for top-level function compilation.
657661
MaybeIndirectHandle<SharedFunctionInfo> input_shared_info_;

‎src/parsing/literal-buffer.cc‎

Lines changed: 0 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -6,29 +6,11 @@
66

77
#include "src/base/strings.h"
88
#include "src/base/unique-array.h"
9-
#include "src/execution/isolate.h"
10-
#include "src/execution/local-isolate.h"
11-
#include "src/heap/factory.h"
12-
#include "src/objects/objects.h"
13-
#include "src/objects/string.h"
149
#include "src/utils/memcopy.h"
1510

1611
namespace v8 {
1712
namespace internal {
1813

19-
template <typename IsolateT>
20-
DirectHandle<String> LiteralBuffer::Internalize(IsolateT* isolate) const {
21-
if (is_one_byte()) {
22-
return isolate->factory()->InternalizeString(one_byte_literal());
23-
}
24-
return isolate->factory()->InternalizeString(two_byte_literal());
25-
}
26-
27-
template DirectHandle<String> LiteralBuffer::Internalize(
28-
Isolate* isolate) const;
29-
template DirectHandle<String> LiteralBuffer::Internalize(
30-
LocalIsolate* isolate) const;
31-
3214
size_t LiteralBuffer::NewCapacity(size_t min_capacity) {
3315
return min_capacity < (kMaxGrowth / (kGrowthFactor - 1))
3416
? min_capacity * kGrowthFactor

‎src/parsing/literal-buffer.h‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -99,9 +99,6 @@ class LiteralBuffer final {
9999
is_one_byte_ = true;
100100
}
101101

102-
template <typename IsolateT>
103-
DirectHandle<String> Internalize(IsolateT* isolate) const;
104-
105102
private:
106103
static constexpr size_t kInitialCapacity = 256;
107104
static constexpr size_t kGrowthFactor = 4;

‎src/parsing/parser.cc‎

Lines changed: 30 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
#include "src/common/message-template.h"
2121
#include "src/common/scoped-modification.h"
2222
#include "src/compiler-dispatcher/lazy-compile-dispatcher.h"
23+
#include "src/heap/factory-inl.h"
2324
#include "src/heap/local-heap-inl.h"
2425
#include "src/heap/parked-scope.h"
2526
#include "src/logging/counters.h"
@@ -3567,32 +3568,47 @@ void Parser::InsertSloppyBlockFunctionVarBindings(DeclarationScope* scope) {
35673568
// ----------------------------------------------------------------------------
35683569
// Parser support
35693570

3570-
template <typename IsolateT>
3571-
void Parser::HandleDebugMagicComments(IsolateT* isolate,
3571+
void Parser::HandleDebugMagicComments(Isolate* isolate,
35723572
DirectHandle<Script> script) {
3573-
DirectHandle<String> source_url = scanner_.SourceUrl(isolate);
3573+
// On the main thread, the scanned source is the script source.
3574+
DirectHandle<String> source(Cast<String>(script->source()), isolate);
3575+
SetDebugMagicCommentFields(isolate, script, source,
3576+
scanner_.debug_magic_comment_locations());
3577+
}
3578+
3579+
// static
3580+
void Parser::SetDebugMagicCommentFields(
3581+
Isolate* isolate, DirectHandle<Script> script, DirectHandle<String> source,
3582+
const Scanner::DebugMagicCommentLocations& locations) {
3583+
// The values can be arbitrarily large (e.g. inline source maps), so they are
3584+
// substrings of the source rather than copies.
3585+
auto value = [&](Scanner::Location location) -> DirectHandle<String> {
3586+
if (!location.IsValid() || location.length() == 0 ||
3587+
static_cast<uint32_t>(location.end_pos) > source->length()) {
3588+
return {};
3589+
}
3590+
return isolate->factory()->NewSubString(source, location.beg_pos,
3591+
location.end_pos);
3592+
};
3593+
DirectHandle<String> source_url = value(locations.source_url);
35743594
if (!source_url.is_null()) {
35753595
script->set_source_url(*source_url);
35763596
}
3577-
DirectHandle<String> source_mapping_url = scanner_.SourceMappingUrl(isolate);
35783597
// The API can provide a source map URL and the API should take precedence.
35793598
// Let's make sure we do not override the API with the magic comment.
3580-
if (!source_mapping_url.is_null() &&
3581-
IsUndefined(script->source_mapping_url())) {
3582-
script->set_source_mapping_url(*source_mapping_url);
3599+
if (IsUndefined(script->source_mapping_url())) {
3600+
DirectHandle<String> source_mapping_url =
3601+
value(locations.source_mapping_url);
3602+
if (!source_mapping_url.is_null()) {
3603+
script->set_source_mapping_url(*source_mapping_url);
3604+
}
35833605
}
3584-
3585-
DirectHandle<String> debug_id = scanner_.DebugId(isolate);
3606+
DirectHandle<String> debug_id = value(locations.debug_id);
35863607
if (!debug_id.is_null()) {
35873608
script->set_debug_id(*debug_id);
35883609
}
35893610
}
35903611

3591-
template void Parser::HandleDebugMagicComments(Isolate* isolate,
3592-
DirectHandle<Script> script);
3593-
template void Parser::HandleDebugMagicComments(LocalIsolate* isolate,
3594-
DirectHandle<Script> script);
3595-
35963612
void Parser::UpdateStatistics(Isolate* isolate, DirectHandle<Script> script) {
35973613
CHECK_NOT_NULL(isolate);
35983614

@@ -3695,9 +3711,6 @@ void Parser::ParseOnBackground(LocalIsolate* isolate, ParseInfo* info,
36953711
MaybeProcessSourceRanges(info, result, stack_limit_);
36963712
});
36973713
// We need to unpark by now though, to be able to internalize.
3698-
if (flags().is_toplevel()) {
3699-
HandleDebugMagicComments(isolate, script);
3700-
}
37013714
if (result == nullptr) return;
37023715
PostProcessParseResult(isolate, info, result);
37033716
}

‎src/parsing/parser.h‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -169,8 +169,18 @@ class V8_EXPORT_PRIVATE Parser : public NON_EXPORTED_BASE(ParserBase<Parser>) {
169169
DirectHandle<Script> script,
170170
base::SmallVector<v8::Isolate::UseCounterFeature, 8>* use_counters,
171171
int* preparse_skipped);
172-
template <typename IsolateT>
173-
void HandleDebugMagicComments(IsolateT* isolate, DirectHandle<Script> script);
172+
void HandleDebugMagicComments(Isolate* isolate, DirectHandle<Script> script);
173+
const Scanner::DebugMagicCommentLocations& debug_magic_comment_locations()
174+
const {
175+
return scanner_.debug_magic_comment_locations();
176+
}
177+
// Sets the debug magic comment fields of {script} to substrings of {source}
178+
// at the given locations. Locations that are empty, invalid or out of bounds
179+
// are ignored.
180+
static void SetDebugMagicCommentFields(
181+
Isolate* isolate, DirectHandle<Script> script,
182+
DirectHandle<String> source,
183+
const Scanner::DebugMagicCommentLocations& locations);
174184

175185
private:
176186
friend class ParserBase<Parser>;

‎src/parsing/scanner.cc‎

Lines changed: 16 additions & 47 deletions
Original file line numberDiff line numberDiff line change
@@ -293,12 +293,15 @@ void Scanner::TryToParseMagicComment(base::uc32 hash_or_at_sign) {
293293
}
294294
if (!name.is_one_byte()) return;
295295
base::Vector<const uint8_t> name_literal = name.one_byte_literal();
296-
LiteralBuffer* value;
296+
// The debug magic comment values are recorded as source locations, so that
297+
// they can be created as substrings of the source instead of being copied.
298+
Location* value_location = nullptr;
299+
bool is_compile_hints = false;
297300
LiteralBuffer per_function_compile_hints_value;
298301
if (name_literal == base::StaticOneByteVector("sourceURL")) {
299-
value = &source_url_;
302+
value_location = &debug_magic_comment_locations_.source_url;
300303
} else if (name_literal == base::StaticOneByteVector("sourceMappingURL")) {
301-
value = &source_mapping_url_;
304+
value_location = &debug_magic_comment_locations_.source_mapping_url;
302305
DCHECK(hash_or_at_sign == '#' || hash_or_at_sign == '@');
303306
saw_source_mapping_url_magic_comment_at_sign_ = hash_or_at_sign == '@';
304307
} else if (!saw_non_comment_ &&
@@ -309,38 +312,41 @@ void Scanner::TryToParseMagicComment(base::uc32 hash_or_at_sign) {
309312
} else if (name_literal ==
310313
base::StaticOneByteVector("functionsCalledOnLoad") &&
311314
hash_or_at_sign == '#') {
312-
value = &per_function_compile_hints_value;
315+
is_compile_hints = true;
313316
} else if (name_literal == base::StaticOneByteVector("debugId") &&
314317
hash_or_at_sign == '#') {
315-
value = &debug_id_;
318+
value_location = &debug_magic_comment_locations_.debug_id;
316319
} else {
317320
return;
318321
}
319322
if (c0_ != '=') {
320323
return;
321324
}
322-
value->Start();
325+
if (is_compile_hints) per_function_compile_hints_value.Start();
323326
Advance();
324327
while (IsWhiteSpace(c0_)) {
325328
Advance();
326329
}
330+
int value_beg_pos = source_pos();
327331
while (c0_ != kEndOfInput && !unibrow::IsLineTerminator(c0_)) {
328332
if (IsWhiteSpace(c0_)) {
329333
break;
330334
}
331-
value->AddChar(c0_);
335+
if (is_compile_hints) per_function_compile_hints_value.AddChar(c0_);
332336
Advance();
333337
}
338+
Location value_location_in_source(value_beg_pos, source_pos());
334339
// Allow whitespace at the end.
335340
while (c0_ != kEndOfInput && !unibrow::IsLineTerminator(c0_)) {
336341
if (!IsWhiteSpace(c0_)) {
337-
value->Start();
342+
if (is_compile_hints) per_function_compile_hints_value.Start();
343+
value_location_in_source = Location::invalid();
338344
break;
339345
}
340346
Advance();
341347
}
342-
if (value == &per_function_compile_hints_value &&
343-
per_function_compile_hints_value.is_one_byte()) {
348+
if (value_location != nullptr) *value_location = value_location_in_source;
349+
if (is_compile_hints && per_function_compile_hints_value.is_one_byte()) {
344350
base::Vector<const uint8_t> value_literal =
345351
per_function_compile_hints_value.one_byte_literal();
346352
per_function_compile_hint_positions_.clear();
@@ -689,43 +695,6 @@ Token::Value Scanner::ScanTemplateSpan() {
689695
return result;
690696
}
691697

692-
template <typename IsolateT>
693-
DirectHandle<String> Scanner::SourceUrl(IsolateT* isolate) const {
694-
DirectHandle<String> tmp;
695-
if (source_url_.length() > 0) {
696-
tmp = source_url_.Internalize(isolate);
697-
}
698-
return tmp;
699-
}
700-
701-
template DirectHandle<String> Scanner::SourceUrl(Isolate* isolate) const;
702-
template DirectHandle<String> Scanner::SourceUrl(LocalIsolate* isolate) const;
703-
704-
template <typename IsolateT>
705-
DirectHandle<String> Scanner::SourceMappingUrl(IsolateT* isolate) const {
706-
DirectHandle<String> tmp;
707-
if (source_mapping_url_.length() > 0) {
708-
tmp = source_mapping_url_.Internalize(isolate);
709-
}
710-
return tmp;
711-
}
712-
713-
template DirectHandle<String> Scanner::SourceMappingUrl(Isolate* isolate) const;
714-
template DirectHandle<String> Scanner::SourceMappingUrl(
715-
LocalIsolate* isolate) const;
716-
717-
template <typename IsolateT>
718-
DirectHandle<String> Scanner::DebugId(IsolateT* isolate) const {
719-
DirectHandle<String> tmp;
720-
if (debug_id_.length() > 0) {
721-
tmp = debug_id_.Internalize(isolate);
722-
}
723-
return tmp;
724-
}
725-
726-
template DirectHandle<String> Scanner::DebugId(Isolate* isolate) const;
727-
template DirectHandle<String> Scanner::DebugId(LocalIsolate* isolate) const;
728-
729698
bool Scanner::ScanDigitsWithNumericSeparators(bool (*predicate)(base::uc32 ch),
730699
bool is_check_first_digit) {
731700
// we must have at least one digit after 'x'/'b'/'o'

‎src/parsing/scanner.h‎

Lines changed: 13 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -464,12 +464,18 @@ class V8_EXPORT_PRIVATE Scanner {
464464
return ScanTemplateSpan();
465465
}
466466

467-
template <typename IsolateT>
468-
DirectHandle<String> SourceUrl(IsolateT* isolate) const;
469-
template <typename IsolateT>
470-
DirectHandle<String> SourceMappingUrl(IsolateT* isolate) const;
471-
template <typename IsolateT>
472-
DirectHandle<String> DebugId(IsolateT* isolate) const;
467+
// Source locations of the values of the debug magic comments. A location is
468+
// invalid if there was no such comment, or if the last one was malformed; it
469+
// is empty if the last one had an empty value. In both cases there is no
470+
// value.
471+
struct DebugMagicCommentLocations {
472+
Location source_url = Location::invalid();
473+
Location source_mapping_url = Location::invalid();
474+
Location debug_id = Location::invalid();
475+
};
476+
const DebugMagicCommentLocations& debug_magic_comment_locations() const {
477+
return debug_magic_comment_locations_;
478+
}
473479

474480
bool SawSourceMappingUrlMagicCommentAtSign() const {
475481
return saw_source_mapping_url_magic_comment_at_sign_;
@@ -807,10 +813,7 @@ class V8_EXPORT_PRIVATE Scanner {
807813
// Whether this scanner encountered an HTML comment.
808814
bool found_html_comment_;
809815

810-
// Values parsed from magic comments.
811-
LiteralBuffer source_url_;
812-
LiteralBuffer source_mapping_url_;
813-
LiteralBuffer debug_id_;
816+
DebugMagicCommentLocations debug_magic_comment_locations_;
814817
bool saw_source_mapping_url_magic_comment_at_sign_ = false;
815818
bool saw_magic_comment_compile_hints_all_ = false;
816819
bool saw_non_comment_ = false;

‎src/snapshot/code-serializer.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -250,6 +250,12 @@ class CodeSerializer : public Serializer {
250250
sink_.Put(kSynchronize, "EndOfUntrustedSection");
251251
Pad();
252252
}
253+
// Slices usually refer to the script source (e.g. debug magic comment
254+
// values), which is an attached object: off-thread deserialization attaches
255+
// a placeholder instead of the source.
256+
bool SerializeSlicedStringsAsSequentialStrings() const override {
257+
return true;
258+
}
253259

254260
DISALLOW_GARBAGE_COLLECTION(no_gc_)
255261
SourceHash source_hash_;

0 commit comments

Comments
 (0)