Skip to content

Commit 77a9432

Browse files
hi-ogawaclaude
hi-ogawa
and
claude
authored
fix: show attempts in test error logs of retried tests (#11696)
Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent 3667cf8 commit 77a9432

4 files changed

Lines changed: 153 additions & 15 deletions

File tree

‎packages/vitest/src/node/reporters/base.ts‎

Lines changed: 31 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ import type {
2222
} from './reported-tasks'
2323
import { readFileSync } from 'node:fs'
2424
import { performance } from 'node:perf_hooks'
25-
import { toArray } from '@vitest/utils/helpers'
25+
import { ordinal, toArray } from '@vitest/utils/helpers'
2626
import { parseStacktrace } from '@vitest/utils/source-map'
2727
import { relative } from 'pathe'
2828
import c from 'tinyrainbow'
@@ -1501,22 +1501,27 @@ export abstract class BaseReporter implements Reporter {
15011501

15021502
for (const { error, sources } of errorsQueue) {
15031503
const tasks = sources.map(({ task }) => task)
1504-
const attemptsByTask = new Map<Task, Set<TaskResultAttempt | undefined>>()
1504+
const attemptsByTask = new Map<Task, (TaskResultAttempt | undefined)[]>()
15051505
for (const { task, attempt } of sources) {
1506-
attemptsByTask.set(task, (attemptsByTask.get(task) ?? new Set()).add(attempt))
1506+
attemptsByTask.set(task, [...(attemptsByTask.get(task) ?? []), attempt])
15071507
}
15081508
const screenshotPaths = [...attemptsByTask].flatMap(([task, attempts]) =>
1509-
[...attempts].flatMap((attempt) =>
1509+
[...new Set(attempts)].flatMap((attempt) =>
15101510
attempt ? getFailureScreenshotPaths(task, attempt) : [],
15111511
),
15121512
)
1513-
for (const task of tasks) {
1513+
for (const [task, attempts] of attemptsByTask) {
15141514
const filepath = (task as File)?.filepath || ''
15151515
const projectName = (task as File)?.projectName || task.file?.projectName || ''
15161516
const project = this.ctx.projects.find((p) => p.name === projectName)
15171517

15181518
let name = this.getFullName(task, separator)
15191519

1520+
const attemptLabel = formatErrorAttempts(task, attempts)
1521+
if (attemptLabel) {
1522+
name += c.dim(` (${attemptLabel})`)
1523+
}
1524+
15201525
if (filepath) {
15211526
name += c.dim(` [ ${this.relative(filepath)} ]`)
15221527
}
@@ -1556,6 +1561,27 @@ export abstract class BaseReporter implements Reporter {
15561561
}
15571562
}
15581563

1564+
function formatErrorAttempts(task: Task, attempts: (TaskResultAttempt | undefined)[]): string {
1565+
const showAttempt = (task.result?.attempts?.length ?? 0) > 1
1566+
return [...Map.groupBy(attempts, (attempt) => attempt)]
1567+
.map(([attempt, occurrences]) => {
1568+
const label = showAttempt && attempt ? formatAttempt(attempt) : ''
1569+
const count = occurrences.length > 1 ? `${occurrences.length} times` : ''
1570+
return count && label ? `${count} in ${label}` : count || label
1571+
})
1572+
.filter(Boolean)
1573+
.join(', ')
1574+
}
1575+
1576+
function formatAttempt({ repeatIndex, retryIndex }: TaskResultAttempt): string {
1577+
const repeat = repeatIndex ? `${ordinal(repeatIndex)} repeat` : ''
1578+
const retry = retryIndex ? `${ordinal(retryIndex)} retry` : ''
1579+
if (repeat && retry) {
1580+
return `${retry} of ${repeat}`
1581+
}
1582+
return repeat || retry || 'initial run'
1583+
}
1584+
15591585
function getErrorsWithAttempt(task: Task): { error: TestError; attempt?: TaskResultAttempt }[] {
15601586
const attempts = task.result?.attempts
15611587
if (!attempts?.length) {

‎test/browser/specs/failure-screenshot.test.ts‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -139,19 +139,19 @@ describe('failure screenshots', () => {
139139
})
140140

141141
expect(extractFailureLines(stderr, root)).toMatchInlineSnapshot(`
142-
" FAIL |chromium| basic.test.ts > repeated
142+
" FAIL |chromium| basic.test.ts > repeated (initial run)
143143
AssertionError: expected 1 to be 10 // Object.is equality
144144
Failure screenshot:
145145
- <root>/.vitest/attachments/failure-screenshots/basic.test.ts/repeated-0-0.png
146-
FAIL |chromium| basic.test.ts > repeated
146+
FAIL |chromium| basic.test.ts > repeated (1st retry)
147147
AssertionError: expected 2 to be 10 // Object.is equality
148148
Failure screenshot:
149149
- <root>/.vitest/attachments/failure-screenshots/basic.test.ts/repeated-0-1.png
150-
FAIL |chromium| basic.test.ts > repeated
150+
FAIL |chromium| basic.test.ts > repeated (1st repeat)
151151
AssertionError: expected 3 to be 10 // Object.is equality
152152
Failure screenshot:
153153
- <root>/.vitest/attachments/failure-screenshots/basic.test.ts/repeated-1-0.png
154-
FAIL |chromium| basic.test.ts > repeated
154+
FAIL |chromium| basic.test.ts > repeated (1st retry of 1st repeat)
155155
AssertionError: expected 4 to be 10 // Object.is equality
156156
Failure screenshot:
157157
- <root>/.vitest/attachments/failure-screenshots/basic.test.ts/repeated-1-1.png"
@@ -174,14 +174,12 @@ describe('failure screenshots', () => {
174174

175175
// screenshots are taken per attempt, so each soft error lists the same ones
176176
expect(extractFailureLines(stderr, root)).toMatchInlineSnapshot(`
177-
" FAIL |chromium| basic.test.ts > soft
178-
FAIL |chromium| basic.test.ts > soft
177+
" FAIL |chromium| basic.test.ts > soft (initial run, 1st retry)
179178
AssertionError: expected 1 to be 2 // Object.is equality
180179
Failure screenshots:
181180
- <root>/.vitest/attachments/failure-screenshots/basic.test.ts/soft-0-0.png
182181
- <root>/.vitest/attachments/failure-screenshots/basic.test.ts/soft-0-1.png
183-
FAIL |chromium| basic.test.ts > soft
184-
FAIL |chromium| basic.test.ts > soft
182+
FAIL |chromium| basic.test.ts > soft (initial run, 1st retry)
185183
AssertionError: expected 2 to be 3 // Object.is equality
186184
Failure screenshots:
187185
- <root>/.vitest/attachments/failure-screenshots/basic.test.ts/soft-0-0.png

‎test/e2e/test/retry.test.ts‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -346,3 +346,118 @@ test('syntax errors remain failures after successful repeats', async () => {
346346
}
347347
`)
348348
})
349+
350+
describe('failure summary header', () => {
351+
function extractFailureHeaders(stderr: string) {
352+
return stderr
353+
.split('\n')
354+
.filter((line) => /^ FAIL |Error: /.test(line))
355+
.join('\n')
356+
}
357+
358+
test('prints one header for an error repeated on every attempt', async () => {
359+
const { stderr } = await runInlineTests({
360+
'basic.test.js': `
361+
import { expect, test } from 'vitest'
362+
363+
test('retry', { retry: 2 }, () => {
364+
expect(1).toBe(2)
365+
})
366+
367+
test('repeats', { repeats: 1 }, () => {
368+
expect(1).toBe(2)
369+
})
370+
371+
test('retry-repeats', { retry: 1, repeats: 1 }, () => {
372+
expect(1).toBe(2)
373+
})
374+
`,
375+
})
376+
377+
expect(extractFailureHeaders(stderr)).toMatchInlineSnapshot(`
378+
" FAIL basic.test.js > retry (initial run, 1st retry, 2nd retry)
379+
AssertionError: expected 1 to be 2 // Object.is equality
380+
FAIL basic.test.js > repeats (initial run, 1st repeat)
381+
AssertionError: expected 1 to be 2 // Object.is equality
382+
FAIL basic.test.js > retry-repeats (initial run, 1st retry, 1st repeat, 1st retry of 1st repeat)
383+
AssertionError: expected 1 to be 2 // Object.is equality"
384+
`)
385+
})
386+
387+
test('labels each header with the attempt of its error', async () => {
388+
const { stderr } = await runInlineTests({
389+
'basic.test.js': `
390+
import { expect, test } from 'vitest'
391+
392+
let attempt = 0
393+
test('fails differently', { retry: 3 }, () => {
394+
attempt++
395+
expect(attempt).toBe(10)
396+
})
397+
`,
398+
})
399+
400+
expect(extractFailureHeaders(stderr)).toMatchInlineSnapshot(`
401+
" FAIL basic.test.js > fails differently (initial run)
402+
AssertionError: expected 1 to be 10 // Object.is equality
403+
FAIL basic.test.js > fails differently (1st retry)
404+
AssertionError: expected 2 to be 10 // Object.is equality
405+
FAIL basic.test.js > fails differently (2nd retry)
406+
AssertionError: expected 3 to be 10 // Object.is equality
407+
FAIL basic.test.js > fails differently (3rd retry)
408+
AssertionError: expected 4 to be 10 // Object.is equality"
409+
`)
410+
})
411+
412+
test('labels only the repeats that failed', async () => {
413+
const { stderr } = await runInlineTests({
414+
'basic.test.js': `
415+
import { expect, test } from 'vitest'
416+
417+
let run = 0
418+
test('flaky repeats', { repeats: 3 }, () => {
419+
run++
420+
expect(run % 2).toBe(0)
421+
})
422+
`,
423+
})
424+
425+
expect(extractFailureHeaders(stderr)).toMatchInlineSnapshot(`
426+
" FAIL basic.test.js > flaky repeats (initial run, 2nd repeat)
427+
AssertionError: expected 1 to be +0 // Object.is equality"
428+
`)
429+
})
430+
431+
test('prints headers for soft errors repeated within an attempt', async () => {
432+
const { stderr } = await runInlineTests({
433+
'basic.test.js': `
434+
import { expect, test } from 'vitest'
435+
436+
test('soft-no-retry', () => {
437+
for (let i = 0; i < 2; i++) {
438+
expect.soft(1).toBe(2)
439+
}
440+
expect.soft(2).toBe(3)
441+
})
442+
443+
test('soft-retry', { retry: 1 }, () => {
444+
for (let i = 0; i < 2; i++) {
445+
expect.soft(1).toBe(2)
446+
}
447+
expect.soft(2).toBe(3)
448+
})
449+
`,
450+
})
451+
452+
expect(extractFailureHeaders(stderr)).toMatchInlineSnapshot(`
453+
" FAIL basic.test.js > soft-no-retry (2 times)
454+
AssertionError: expected 1 to be 2 // Object.is equality
455+
FAIL basic.test.js > soft-no-retry
456+
AssertionError: expected 2 to be 3 // Object.is equality
457+
FAIL basic.test.js > soft-retry (2 times in initial run, 2 times in 1st retry)
458+
AssertionError: expected 1 to be 2 // Object.is equality
459+
FAIL basic.test.js > soft-retry (initial run, 1st retry)
460+
AssertionError: expected 2 to be 3 // Object.is equality"
461+
`)
462+
})
463+
})

‎test/e2e/test/scoped-fixtures.test.ts‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2783,8 +2783,7 @@ describe('builder pattern API with automatic type inference', () => {
27832783
"
27842784
⎯⎯⎯⎯⎯⎯⎯ Failed Tests 1 ⎯⎯⎯⎯⎯⎯⎯
27852785
2786-
FAIL basic.test.ts > test runs but cleanup fails
2787-
FAIL basic.test.ts > test runs but cleanup fails
2786+
FAIL basic.test.ts > test runs but cleanup fails (2 times)
27882787
Error: cleanup error
27892788
❯ test.js:10:13
27902789
8| onCleanup(() => {

0 commit comments

Comments
 (0)