Skip to content

Commit 75bbce8

Browse files
mihailikmihailik
authored
Proof of concept of incremental pausable loading
Loading a large project in incremental fashion (here -- all source files in TypeScript repo) See this comment for context: microsoft#37713 (comment) Batch size tuning and a threshold to switch into partial file loading is at the start of this `demo.js` script, as well as some other pieces. On my very humble laptop the majority of incremental loads take ~400ms, with occasional 1-2s chunks. The biggest struggle seems to be the `checker.ts`, assuming that's because it's one huge scope rather than a set of smaller ones. Also if you enable `'.js'` extension, the big build outputs (`tsc.js`, `typescript.js` etc.) lead to even bigger delays, up to 20s on my laptop. In general JavaScript seems to take more effort to process than TypeScript, at least with these examples. Note there are 2 switches (`false` by default) to query syntactic/semantic errors in each file as chunks are injected. They slow everything a lot. In practice, the vast majority of errors during project loading are superficial, due to files missing yet. A more realistic check is to request a completion in the middle of the loaded file.
1 parent 0018b8f commit 75bbce8

File tree

1 file changed

+332
-0
lines changed

1 file changed

+332
-0
lines changed

‎demo.js

+332
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,332 @@
1+
// @ts-check
2+
3+
var largeSizeThreshold = 60 * 1000;
4+
var batchSize = largeSizeThreshold / 2;
5+
var contextCodeQuoteLength = 35;
6+
var parseExtensions =
7+
['.ts', '.tsx', '.d.ts'];
8+
// ['.ts', '.tsx', '.d.ts', '.js', '.json'];
9+
10+
var requestSyntacticDiagnosticOnEachStep = false;
11+
var requestSemanticDiagnosticOnEachStep = false;
12+
13+
logTimed('Loading TypeScript library...');
14+
var ts = require('./lib/typescript');
15+
logTimed('...at ' + require.resolve('./lib/typescript'));
16+
17+
var projectRoot = process.argv.length > 2 ? ts.sys.resolvePath(process.argv[2]) :
18+
ts.sys.resolvePath(require.resolve('./lib/typescript') + '../../..');
19+
20+
logTimed('Project root at ' + projectRoot);
21+
22+
var settings = ts.getDefaultCompilerOptions();
23+
settings.allowJs = true;
24+
settings.checkJs = true;
25+
settings.resolveJsonModule = true;
26+
27+
/** @typedef {{
28+
* version?: number,
29+
* text?: string,
30+
* } & import('./lib/typescript').IScriptSnapshot} VScriptSnapshot */
31+
/** @type {{ [absoluteFilePath: string]: VScriptSnapshot }} */
32+
var scripts = {};
33+
34+
logTimed('Creating LanguageServiceHost...');
35+
/** @type {import('./lib/typescript').LanguageServiceHost} */
36+
var lsHost = {
37+
getCompilationSettings: function () { return settings; },
38+
getScriptFileNames: function () { return Object.keys(scripts); },
39+
getScriptVersion: function (fileName) { return scripts[fileName] && String(scripts[fileName].version || ''); },
40+
getScriptSnapshot: function (fileName) { return scripts[fileName]; },
41+
getCurrentDirectory: function () { return projectRoot; },
42+
getDefaultLibFileName: function (options) {
43+
const name = ts.getDefaultLibFileName(options);
44+
return name;
45+
}
46+
};
47+
48+
logTimed('Creating LanguageService...');
49+
var langService = ts.createLanguageService(lsHost);
50+
51+
logTimed('Enumerating directory...');
52+
var allFiles = ts.sys.readDirectory(
53+
projectRoot,
54+
parseExtensions);
55+
logTimed('...' + allFiles.length + ' found.');
56+
57+
58+
logTimed('Loading...');
59+
var lastFileLoadReport = Date.now();
60+
var previousTimes;
61+
for (var indexOfFile = 0; indexOfFile < allFiles.length; indexOfFile++) {
62+
previousTimes = loadNextFile(indexOfFile, previousTimes);
63+
}
64+
65+
/** @typedef {{
66+
* indexOfFile: number,
67+
* fileName: string,
68+
* fileLoadStart: number,
69+
* fileLoadEnd?: number,
70+
* size?: number,
71+
* text?: string,
72+
* snapshot?: VScriptSnapshot,
73+
* syntDiag?: import('./lib/typescript').Diagnostic[],
74+
* semDiag1?: import('./lib/typescript').Diagnostic[],
75+
* semDiag2?: import('./lib/typescript').Diagnostic[],
76+
* complets?: import('./lib/typescript').WithMetadata<import('./lib/typescript').CompletionInfo>,
77+
* toString(): string
78+
* }} FileDesc */
79+
80+
/**
81+
* @param {number} indexOfFile
82+
* @param {FileDesc} previousTimes
83+
* @returns {FileDesc}
84+
*/
85+
function loadNextFile(indexOfFile, previousTimes) {
86+
var fileName = allFiles[indexOfFile];
87+
88+
/** @type {FileDesc} */
89+
var times = {
90+
indexOfFile: indexOfFile,
91+
fileName: fileName,
92+
fileLoadStart: Date.now(),
93+
toString: timeToString
94+
};
95+
96+
// if previous didn't report AND next load will be large, show that context
97+
times.size = ts.sys.getFileSize(fileName);
98+
var anticipateLargeFile = times.size > largeSizeThreshold;
99+
if (anticipateLargeFile) {
100+
if (previousTimes)
101+
logTimed(previousTimes.toString());
102+
103+
loadLargeFile(times);
104+
return;
105+
}
106+
else {
107+
loadSmallFile(times);
108+
109+
if (times.fileLoadEnd - lastFileLoadReport > 200 ||
110+
times.fileLoadEnd - times.fileLoadStart > 600) {
111+
logTimed(times.toString());
112+
return;
113+
}
114+
115+
return times;
116+
}
117+
}
118+
119+
/**
120+
* @param {FileDesc} fileDesc
121+
*/
122+
function loadSmallFile(fileDesc) {
123+
fileDesc.text = ts.sys.readFile(fileDesc.fileName);
124+
recordFileTiming(fileDesc, 'read');
125+
126+
/** @type {VScriptSnapshot} */
127+
fileDesc.snapshot = ts.ScriptSnapshot.fromString(fileDesc.text);
128+
129+
fileDesc.snapshot.version = 0;
130+
scripts[fileDesc.fileName] = fileDesc.snapshot;
131+
132+
revalidateFile(fileDesc);
133+
}
134+
135+
/**
136+
* @param {FileDesc} fileDesc
137+
*/
138+
function revalidateFile(fileDesc) {
139+
if (requestSyntacticDiagnosticOnEachStep) {
140+
fileDesc.syntDiag = langService.getSyntacticDiagnostics(fileDesc.fileName);
141+
recordFileTiming(fileDesc, 'syntx');
142+
}
143+
144+
if (requestSemanticDiagnosticOnEachStep) {
145+
fileDesc.semDiag1 = langService.getSemanticDiagnostics(fileDesc.fileName);
146+
recordFileTiming(fileDesc, 'sem1');
147+
148+
fileDesc.semDiag2 = langService.getSemanticDiagnostics(fileDesc.fileName);
149+
recordFileTiming(fileDesc, 'sem2');
150+
}
151+
152+
var completionsPos =
153+
((fileDesc.snapshot && fileDesc.snapshot.text ? fileDesc.snapshot.text.length : fileDesc.text.length) / 2) | 0;
154+
155+
fileDesc.complets = langService.getCompletionsAtPosition(fileDesc.fileName, completionsPos, {});
156+
recordFileTiming(fileDesc, 'comp');
157+
}
158+
159+
/**
160+
* @param {FileDesc} fileDesc
161+
*/
162+
function loadLargeFile(fileDesc) {
163+
fileDesc.text = ts.sys.readFile(fileDesc.fileName);
164+
recordFileTiming(fileDesc, 'read');
165+
166+
logTimed(
167+
fileDescHeadToString(fileDesc) + ' ' +
168+
formatSizeWithBlue(Math.round(fileDesc.size / 1000), 'K'));
169+
170+
while (true) {
171+
var addStart = fileDesc.snapshot ? fileDesc.snapshot.text.length : 0;
172+
173+
var addEnd = findBestChunkEnd(addStart, fileDesc.text);
174+
175+
var addChunk = fileDesc.text.slice(addStart, addEnd);
176+
scripts[fileDesc.fileName] = fileDesc.snapshot = partialSnapshot(addChunk, fileDesc.snapshot);
177+
178+
revalidateFile(fileDesc);
179+
180+
var contextText = tryPrintContext(fileDesc.fileName, addStart, addChunk);
181+
182+
logTimed(
183+
' +' + formatSizeWithBlue(Math.round((addEnd - addStart) / 1000), 'K') +
184+
(addStart >= fileDesc.text.length ? '/end' : '') + ' ...' +
185+
timeTailToString(fileDesc) +
186+
(contextText ? '\n' + contextText : ''));
187+
188+
if (addEnd >= fileDesc.text.length)
189+
break;
190+
}
191+
}
192+
193+
function formatSizeWithBlue(size, suffix) {
194+
var sizeStr = String(size);
195+
return '\x1b[36m' + sizeStr.slice(0, Math.max(0, sizeStr.length - 3)) +
196+
'\x1b[34m' + sizeStr.slice(-3) + (suffix ? '\x1b[36m' + suffix : '') + '\x1b[0m';
197+
}
198+
199+
200+
/**
201+
* @param {string} fileName
202+
* @param {number} addStart
203+
* @param {string} addChunk
204+
*/
205+
function tryPrintContext(fileName, addStart, addChunk) {
206+
var firstLineMatch = /^\s*[\s\S][\s\S][\s\S]\s*\S[^\n\r]*[\n\r]/.exec(addChunk);
207+
var lastLineMatch = /[\n\r]*[^\n\r]*\s*[\s\S][\s\S][\s\S]\s*$/.exec(addChunk);
208+
if (firstLineMatch && lastLineMatch) {
209+
var firstLine = firstLineMatch[0].replace(/^\s+/, '').replace(/\s+$/, '').replace(/[\r\n]/g, ' ');
210+
var lastLine = lastLineMatch[0].replace(/^\s+/, '').replace(/\s+$/, '').replace(/[\r\n]/g, ' ');
211+
if (firstLine.length > contextCodeQuoteLength)
212+
firstLine = firstLine.slice(0, contextCodeQuoteLength);
213+
if (lastLine.length > contextCodeQuoteLength)
214+
lastLine = lastLine.slice(-contextCodeQuoteLength);
215+
if (firstLine.length && lastLine.length) {
216+
var prog = langService.getProgram();
217+
var file = prog && prog.getSourceFile(fileName);
218+
var firstLineStart = addStart + addChunk.indexOf(firstLine.charAt(0));
219+
var firstLineNum = file && file.getLineAndCharacterOfPosition(firstLineStart).line + 1;
220+
var lastLineEnd = addStart + addChunk.lastIndexOf(lastLine.charAt(lastLine.length - 1));
221+
var lastLineNum = file && file.getLineAndCharacterOfPosition(lastLineEnd).line + 1;
222+
return (
223+
' ' + (firstLineNum ? 'L' + firstLineNum + ' ' : '') + ' \x1b[90m' + firstLine + '\x1b[0m ... ' +
224+
'\x1b[90m' + lastLine + '\x1b[0m' +
225+
(lastLineNum ? ' L' + lastLineNum +
226+
'\x1b[90m+' + formatSizeWithBlue(lastLineNum - firstLineNum) + '\x1b[0m ' :
227+
'')
228+
);
229+
}
230+
}
231+
}
232+
233+
/**
234+
* @param {number} chunkStart
235+
* @param {string | string[]} text
236+
*/
237+
function findBestChunkEnd(chunkStart, text) {
238+
var chunkEnd = chunkStart + Math.min(batchSize, (text.length - chunkStart) / 2);
239+
// closing bracket at the start of the line is probably a safe breaking point
240+
// (except look for a next newline after, for the sake of IIFE)
241+
var bracketMatch = text.indexOf('\n}', chunkEnd);
242+
var newLineAfterBracket = bracketMatch < 0 ? -1 : text.indexOf('\n', bracketMatch + 2);
243+
var chunkEnd = newLineAfterBracket >= 0 ? newLineAfterBracket : text.length;
244+
// sometimes the safe chunk is just too large, go unsafe
245+
if (chunkEnd - chunkStart > batchSize * 4) {
246+
if (bracketMatch > 0 && bracketMatch - chunkStart < batchSize * 4) {
247+
chunkEnd = bracketMatch + 1;
248+
}
249+
else {
250+
chunkEnd = text.indexOf('}', chunkStart + batchSize);
251+
if (chunkEnd < 0 || chunkEnd - chunkStart > batchSize * 4)
252+
chunkEnd = chunkStart + batchSize;
253+
}
254+
}
255+
return chunkEnd;
256+
}
257+
258+
function partialSnapshot(text, prevSnapshot) {
259+
var updated = {
260+
text: prevSnapshot ? prevSnapshot.text + text : text,
261+
version: prevSnapshot ? prevSnapshot.version +1 : 0,
262+
getText: partialSnapshot_getText,
263+
getLength: partialSnapshot_getLength,
264+
getChangeRange: partialSnapshot_getChangeRange
265+
};
266+
267+
return updated;
268+
}
269+
270+
function partialSnapshot_getText(start, end) {
271+
return this.text.slice(start, end);
272+
}
273+
274+
function partialSnapshot_getLength() {
275+
return this.text.length;
276+
}
277+
278+
function partialSnapshot_getChangeRange(oldSnapshot) {
279+
return {
280+
span: { start: oldSnapshot.text.length, length: 0 },
281+
newLength: this.text.length - oldSnapshot.text.length
282+
};
283+
}
284+
285+
286+
function fileDescHeadToString(fileDesc) {
287+
var shortFileName = fileDesc.fileName.slice(projectRoot.length);
288+
return (
289+
fileDesc.indexOfFile + ') ' +
290+
shortFileName +
291+
' read:' + fileDesc.read
292+
);
293+
}
294+
295+
function timeTailToString(fileDesc) {
296+
return (
297+
(requestSyntacticDiagnosticOnEachStep ?
298+
' syntx:' + fileDesc.syntx : '') +
299+
(requestSemanticDiagnosticOnEachStep ?
300+
' sem1:' + fileDesc.sem1 + '/' + fileDesc.sem2 : '') +
301+
' comp:' + fileDesc.comp +
302+
(fileDesc.complets && fileDesc.complets.entries && fileDesc.complets.entries.length &&
303+
'\x1b[32m~' + fileDesc.complets.entries.length + '*' +
304+
fileDesc.complets.entries[Math.min(2, fileDesc.complets.entries.length - 1)].name +
305+
'\x1b[0m' || '')
306+
);
307+
}
308+
309+
function timeToString() {
310+
return fileDescHeadToString(this) + timeTailToString(this);
311+
}
312+
313+
function recordFileTiming(outcome, name) {
314+
var now = Date.now();
315+
outcome[name] = now - (outcome.fileLoadEnd || outcome.fileLoadStart);
316+
outcome.fileLoadEnd = now;
317+
}
318+
319+
function logTimed() {
320+
var now = Date.now();
321+
var passedMs = (now - /** @type {*} */(logTimed).lastWrite);
322+
var passed = (' ' + (passedMs >= 0 ? String(passedMs) : 'start')).slice(-6);
323+
if (passedMs > 400) passed = passed;
324+
else if (passedMs >= 0) passed = '\x1b[90m' + passed + '\x1b[0m';
325+
/** @type {*} */(logTimed).lastWrite = now;
326+
327+
var args = [passed];
328+
for (var i = 0; i < arguments.length; i++) {
329+
args.push(arguments[i]);
330+
}
331+
console.log.apply(console, args);
332+
}

0 commit comments

Comments
 (0)