Skip to content

Commit fa28c05

Browse files
rossilor95rossilor95
authored
test(wpt): mark timed out tests as 'failed' (#2644)
1 parent d5ad466 commit fa28c05

1 file changed

Lines changed: 86 additions & 65 deletions

File tree

‎test/wpt/runner/runner.mjs‎

Lines changed: 86 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -58,11 +58,13 @@ export class WPTRunner extends EventEmitter {
5858
#reportPath
5959

6060
#stats = {
61-
completed: 0,
62-
failed: 0,
63-
success: 0,
61+
completedTests: 0,
62+
failedTests: 0,
63+
passedTests: 0,
6464
expectedFailures: 0,
65-
skipped: 0
65+
failedFiles: 0,
66+
passedFiles: 0,
67+
skippedFiles: 0
6668
}
6769

6870
constructor (folder, url, { appendReport = false, reportPath } = {}) {
@@ -158,7 +160,7 @@ export class WPTRunner extends EventEmitter {
158160
const status = resolveStatusPath(test, this.#status)
159161

160162
if (status.file.skip || status.topLevel.skip) {
161-
this.#stats.skipped += 1
163+
this.#stats.skippedFiles += 1
162164

163165
console.log(colors(`[${finishedFiles}/${total}] SKIPPED - ${test}`, 'yellow'))
164166
console.log('='.repeat(96))
@@ -187,19 +189,19 @@ export class WPTRunner extends EventEmitter {
187189
}
188190
})
189191

190-
let result, report
192+
const fileUrl = new URL(`/${this.#folderName}${test.slice(this.#folderPath.length)}`, 'http://wpt')
193+
fileUrl.pathname = fileUrl.pathname.replace(/\.js$/, '.html')
194+
fileUrl.search = variant
195+
const result = {
196+
test: fileUrl.href.slice(fileUrl.origin.length),
197+
subtests: [],
198+
status: ''
199+
}
200+
201+
let report
191202
if (this.#appendReport) {
192203
report = JSON.parse(readFileSync(this.#reportPath))
193-
194-
const fileUrl = new URL(`/${this.#folderName}${test.slice(this.#folderPath.length)}`, 'http://wpt')
195-
fileUrl.pathname = fileUrl.pathname.replace(/\.js$/, '.html')
196-
fileUrl.search = variant
197-
198-
result = {
199-
test: fileUrl.href.slice(fileUrl.origin.length),
200-
subtests: [],
201-
status: 'OK'
202-
}
204+
result.status = 'OK'
203205
report.results.push(result)
204206
}
205207

@@ -214,8 +216,8 @@ export class WPTRunner extends EventEmitter {
214216
this.handleTestCompletion(worker)
215217
} else if (message.type === 'error') {
216218
this.#uncaughtExceptions.push({ error: message.error, test })
217-
this.#stats.failed += 1
218-
this.#stats.success -= 1
219+
this.#stats.failedTests += 1
220+
this.#stats.passedTests -= 1
219221
}
220222
})
221223

@@ -224,14 +226,27 @@ export class WPTRunner extends EventEmitter {
224226
signal: AbortSignal.timeout(timeout)
225227
})
226228

227-
console.log(colors(`[${finishedFiles}/${total}] PASSED - ${test}`, 'green'))
229+
if (result.subtests.some((subtest) => subtest?.isExpectedFailure === false)) {
230+
this.#stats.failedFiles += 1
231+
console.log(colors(`[${finishedFiles}/${total}] FAILED - ${test}`, 'red'))
232+
} else {
233+
this.#stats.passedFiles += 1
234+
console.log(colors(`[${finishedFiles}/${total}] PASSED - ${test}`, 'green'))
235+
}
236+
228237
if (variant) console.log('Variant:', variant)
229-
console.log(`Test took ${(performance.now() - start).toFixed(2)}ms`)
238+
console.log(`File took ${(performance.now() - start).toFixed(2)}ms`)
230239
console.log('='.repeat(96))
231240
} catch (e) {
232-
console.log(`${test} timed out after ${timeout}ms`)
241+
// If the worker is terminated by the timeout signal, the test is marked as failed
242+
this.#stats.failedFiles += 1
243+
console.log(colors(`[${finishedFiles}/${total}] FAILED - ${test}`, 'red'))
244+
245+
if (variant) console.log('Variant:', variant)
246+
console.log(`File timed out after ${timeout}ms`)
247+
console.log('='.repeat(96))
233248
} finally {
234-
if (result?.subtests.length > 0) {
249+
if (this.#appendReport && result?.subtests.length > 0) {
235250
writeFileSync(this.#reportPath, JSON.stringify(report))
236251
}
237252

@@ -248,50 +263,49 @@ export class WPTRunner extends EventEmitter {
248263
* Called after a test has succeeded or failed.
249264
*/
250265
handleIndividualTestCompletion (message, status, path, meta, wptResult) {
251-
const { file, topLevel } = status
252-
253-
if (message.type === 'result') {
254-
this.#stats.completed += 1
266+
this.#stats.completedTests += 1
255267

256-
if (message.result.status === 1) {
257-
let expectedFailure = false
258-
this.#stats.failed += 1
268+
const { file, topLevel } = status
269+
const isFailure = message.result.status === 1
259270

260-
wptResult?.subtests.push({
261-
status: 'FAIL',
262-
name: sanitizeUnpairedSurrogates(message.result.name),
263-
message: sanitizeUnpairedSurrogates(message.result.message)
264-
})
271+
const testResult = {
272+
status: isFailure ? 'FAIL' : 'PASS',
273+
name: sanitizeUnpairedSurrogates(message.result.name)
274+
}
265275

266-
const name = normalizeName(message.result.name)
267-
268-
if (file.flaky?.includes(name)) {
269-
expectedFailure = true
270-
this.#stats.expectedFailures += 1
271-
} else if (file.allowUnexpectedFailures || topLevel.allowUnexpectedFailures || file.fail?.includes(name)) {
272-
if (!file.allowUnexpectedFailures && !topLevel.allowUnexpectedFailures) {
273-
if (Array.isArray(file.fail)) {
274-
this.#statusOutput[path] ??= []
275-
this.#statusOutput[path].push(name)
276-
}
276+
if (isFailure) {
277+
let isExpectedFailure = false
278+
this.#stats.failedTests += 1
279+
280+
const name = normalizeName(message.result.name)
281+
const sanitizedMessage = sanitizeUnpairedSurrogates(message.result.message)
282+
283+
if (file.flaky?.includes(name)) {
284+
isExpectedFailure = true
285+
this.#stats.expectedFailures += 1
286+
wptResult?.subtests.push({ ...testResult, message: sanitizedMessage, isExpectedFailure })
287+
} else if (file.allowUnexpectedFailures || topLevel.allowUnexpectedFailures || file.fail?.includes(name)) {
288+
if (!file.allowUnexpectedFailures && !topLevel.allowUnexpectedFailures) {
289+
if (Array.isArray(file.fail)) {
290+
this.#statusOutput[path] ??= []
291+
this.#statusOutput[path].push(name)
277292
}
278-
279-
expectedFailure = true
280-
this.#stats.expectedFailures += 1
281-
} else {
282-
process.exitCode = 1
283-
console.error(message.result)
284-
}
285-
if (!expectedFailure) {
286-
process._rawDebug(`Failed test: ${path}`)
287293
}
294+
295+
isExpectedFailure = true
296+
this.#stats.expectedFailures += 1
297+
wptResult?.subtests.push({ ...testResult, message: sanitizedMessage, isExpectedFailure })
288298
} else {
289-
wptResult?.subtests.push({
290-
status: 'PASS',
291-
name: sanitizeUnpairedSurrogates(message.result.name)
292-
})
293-
this.#stats.success += 1
299+
wptResult?.subtests.push({ ...testResult, message: sanitizedMessage, isExpectedFailure })
300+
process.exitCode = 1
301+
console.error(message.result)
302+
}
303+
if (!isExpectedFailure) {
304+
process._rawDebug(`Failed test: ${path}`)
294305
}
306+
} else {
307+
this.#stats.passedTests += 1
308+
wptResult?.subtests.push(testResult)
295309
}
296310
}
297311

@@ -313,16 +327,23 @@ export class WPTRunner extends EventEmitter {
313327
}
314328

315329
this.emit('completion')
316-
const { completed, failed, success, expectedFailures, skipped } = this.#stats
330+
331+
const { passedFiles, failedFiles, skippedFiles } = this.#stats
332+
console.log(
333+
`File results for folder [${this.#folderName}]: ` +
334+
`completed: ${this.#files.length}, passed: ${passedFiles}, failed: ${failedFiles}, ` +
335+
`skipped: ${skippedFiles}`
336+
)
337+
338+
const { completedTests, failedTests, passedTests, expectedFailures } = this.#stats
317339
console.log(
318-
`[${this.#folderName}]: ` +
319-
`completed: ${completed}, failed: ${failed}, success: ${success}, ` +
340+
`Test results for folder [${this.#folderName}]: ` +
341+
`completed: ${completedTests}, failed: ${failedTests}, passed: ${passedTests}, ` +
320342
`expected failures: ${expectedFailures}, ` +
321-
`unexpected failures: ${failed - expectedFailures}, ` +
322-
`skipped: ${skipped}`
343+
`unexpected failures: ${failedTests - expectedFailures}`
323344
)
324345

325-
process.exit(failed - expectedFailures ? 1 : process.exitCode)
346+
process.exit(failedTests - expectedFailures ? 1 : process.exitCode)
326347
}
327348

328349
addInitScript (code) {

0 commit comments

Comments
 (0)