Skip to content

Commit d6db09f

Browse files
soltonigiriSimenB
andauthored
[jest-circus, jest-snapshot] Keep snapshot state and counts correct when a test retries (#16344)
Co-authored-by: Simen Bekkhus <[email protected]>
1 parent 1c21e32 commit d6db09f

11 files changed

Lines changed: 777 additions & 45 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
- `[jest-circus]` Clear `currentlyRunningTest` after skipped and todo tests ([#16342](https://github.com/jestjs/jest/pull/16342))
2020
- `[jest-circus]` Prevent late `done()` callbacks from affecting later test or hook invocations ([#16343](https://github.com/jestjs/jest/pull/16343))
2121
- `[jest-circus, jest-jasmine2, jest-message-util]` Serialize the inner errors of an `AggregateError` into `failureMessages`, `retryReasons` and `unhandledErrors`, so `--json` output and reporter annotations include them ([#16316](https://github.com/jestjs/jest/pull/16316))
22+
- `[jest-circus, jest-snapshot]` Keep snapshot state and counts correct when a test retries ([#16344](https://github.com/jestjs/jest/pull/16344))
2223
- `[@jest/create-cache-key-function]` Include the caller support flags in the generated key, so a transformer that emits ESM or CJS based on them no longer shares one cache entry between the two ([#16331](https://github.com/jestjs/jest/pull/16331))
2324
- `[@jest/create-cache-key-function]` Include the stringified project config in the generated key, so editing a transformer's own settings invalidates what it cached ([#16331](https://github.com/jestjs/jest/pull/16331))
2425
- `[@jest/transform]` Include the caller support flags in a transform's cache key, so a file transformed both as ESM and as CJS no longer serves one shape's output for the other ([#16331](https://github.com/jestjs/jest/pull/16331))

‎e2e/__tests__/toMatchInlineSnapshotWithRetries.test.ts‎

Lines changed: 47 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
*/
77

88
import * as path from 'path';
9+
import * as fs from 'graceful-fs';
910
import {skipSuiteOnJasmine} from '@jest/test-utils';
1011
import {cleanup, makeTemplate, writeFiles} from '../Utils';
1112
import runJest from '../runJest';
@@ -118,7 +119,52 @@ test('works when multiple tests have snapshots but only one of them failed multi
118119
[filename]: template(['index', '4' /* retries */]),
119120
});
120121
const {stderr, exitCode} = runJest(DIR, ['-w=1', '--ci=false', filename]);
121-
expect(stderr).toMatch('Snapshots: 1 passed, 1 total');
122+
expect(stderr).toMatch('Snapshots: 2 passed, 2 total');
122123
expect(exitCode).toBe(0);
123124
}
124125
});
126+
127+
test('preserves added and updated snapshots from colliding full names', () => {
128+
const filename = 'preserve-unrelated-inline-snapshots.test.js';
129+
const template = makeTemplate(`
130+
let attempt = 0;
131+
jest.retryTimes(2);
132+
133+
describe('a', () => {
134+
test('b c', () => {
135+
expect('before retry').toMatchInlineSnapshot(\`"outdated"\`);
136+
expect('new snapshot').toMatchInlineSnapshot();
137+
});
138+
});
139+
140+
describe('a b', () => {
141+
test('c', () => {
142+
attempt += 1;
143+
expect('retry ' + attempt).toMatchInlineSnapshot();
144+
expect(attempt).toBe(3);
145+
});
146+
});
147+
`);
148+
149+
writeFiles(TESTS_DIR, {[filename]: template([])});
150+
const {stderr, exitCode} = runJest(DIR, [
151+
'-w=1',
152+
'--ci=false',
153+
'-u',
154+
filename,
155+
]);
156+
const testContents = fs.readFileSync(path.join(TESTS_DIR, filename), 'utf8');
157+
158+
expect(stderr).not.toContain(
159+
'Multiple inline snapshots for the same call are not supported.',
160+
);
161+
expect(stderr).toMatch('2 snapshots written from 1 test suite.');
162+
expect(stderr).toMatch('1 snapshot updated from 1 test suite.');
163+
expect(testContents).toContain('toMatchInlineSnapshot(`"before retry"`)');
164+
expect(testContents).toContain('toMatchInlineSnapshot(`"new snapshot"`)');
165+
expect(testContents).toContain('toMatchInlineSnapshot(`"retry 3"`)');
166+
expect(testContents).not.toContain('`"retry 1"`');
167+
expect(testContents).not.toContain('`"retry 2"`');
168+
expect(testContents.match(/toMatchInlineSnapshot\(`/g)).toHaveLength(3);
169+
expect(exitCode).toBe(0);
170+
});

‎e2e/__tests__/toMatchSnapshotWithRetries.test.ts‎

Lines changed: 198 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
*/
77

88
import * as path from 'path';
9+
import * as fs from 'graceful-fs';
910
import {skipSuiteOnJasmine} from '@jest/test-utils';
1011
import {cleanup, makeTemplate, writeFiles} from '../Utils';
1112
import runJest from '../runJest';
@@ -58,6 +59,202 @@ test('works with a single snapshot', () => {
5859
}
5960
});
6061

62+
test('keeps snapshot counts from other tests when a test retries', () => {
63+
const filename = 'counts-across-retries.test.js';
64+
const template = makeTemplate(`
65+
test('passing snapshot', () => expect('foo').toMatchSnapshot());
66+
67+
describe('with retries', () => {
68+
let index = 0;
69+
afterEach(() => {
70+
index += 1;
71+
});
72+
jest.retryTimes(4);
73+
test('flaky snapshot', () => expect(index).toMatchSnapshot());
74+
});
75+
`);
76+
77+
writeFiles(TESTS_DIR, {
78+
[filename]: template([]),
79+
[`__snapshots__/${filename}.snap`]: `// Jest Snapshot v1, https://goo.gl/fbAQLP
80+
81+
exports[\`with retries flaky snapshot 1\`] = \`3\`;
82+
`,
83+
});
84+
85+
const {stderr, exitCode} = runJest(DIR, ['-w=1', '--ci=false', filename]);
86+
87+
expect(exitCode).toBe(0);
88+
expect(stderr).toMatch('1 snapshot written from 1 test suite.');
89+
expect(stderr).toMatch('Snapshots: 1 written, 1 passed, 2 total');
90+
});
91+
92+
test('reports a snapshot added on a discarded attempt as written', () => {
93+
const filename = 'written-on-retry.test.js';
94+
const template = makeTemplate(`
95+
let index = 0;
96+
afterEach(() => {
97+
index += 1;
98+
});
99+
jest.retryTimes(1);
100+
test('existing', () => expect('old').toMatchSnapshot());
101+
test('adds a new snapshot then fails once', () => {
102+
expect('brand new').toMatchSnapshot();
103+
expect(index).toBe(2);
104+
});
105+
`);
106+
107+
writeFiles(TESTS_DIR, {
108+
[filename]: template([]),
109+
[`__snapshots__/${filename}.snap`]: `// Jest Snapshot v1, https://goo.gl/fbAQLP
110+
111+
exports[\`existing 1\`] = \`"old"\`;
112+
`,
113+
});
114+
115+
const {stderr, exitCode} = runJest(DIR, ['-w=1', '--ci=false', filename]);
116+
117+
expect(exitCode).toBe(0);
118+
expect(stderr).toMatch('1 snapshot written from 1 test suite.');
119+
expect(stderr).toMatch('Snapshots: 1 written, 1 passed, 2 total');
120+
});
121+
122+
test('still reports a snapshot only reached on a discarded attempt as obsolete', () => {
123+
const filename = 'obsolete-across-retries.test.js';
124+
const template = makeTemplate(`
125+
let attempt = 0;
126+
jest.retryTimes(1);
127+
test('varies', () => {
128+
attempt += 1;
129+
expect('a').toMatchSnapshot();
130+
if (attempt === 1) {
131+
expect('b').toMatchSnapshot();
132+
throw new Error('fail once');
133+
}
134+
});
135+
`);
136+
137+
writeFiles(TESTS_DIR, {
138+
[filename]: template([]),
139+
[`__snapshots__/${filename}.snap`]: `// Jest Snapshot v1, https://goo.gl/fbAQLP
140+
141+
exports[\`varies 1\`] = \`"a"\`;
142+
143+
exports[\`varies 2\`] = \`"b"\`;
144+
`,
145+
});
146+
147+
const {stderr, exitCode} = runJest(DIR, ['-w=1', '--ci=false', filename]);
148+
149+
// Matches a run without retries: an obsolete snapshot is reported and fails
150+
// the run, even though every test passed.
151+
expect(exitCode).toBe(1);
152+
expect(stderr).toMatch('Snapshots: 1 obsolete, 1 passed, 1 total');
153+
});
154+
155+
test('keeps counters for colliding full names across an immediate retry', () => {
156+
const filename = 'counters-across-immediate-retries.test.js';
157+
const template = makeTemplate(`
158+
let attempt = 0;
159+
jest.retryTimes(1, {retryImmediately: true});
160+
161+
describe('a', () => {
162+
test('b c', () => {
163+
expect('first').toMatchSnapshot();
164+
});
165+
});
166+
167+
test('flaky', () => {
168+
attempt += 1;
169+
expect(attempt).toBe(2);
170+
});
171+
172+
describe('a b', () => {
173+
test('c', () => {
174+
expect('second').toMatchSnapshot();
175+
});
176+
});
177+
`);
178+
179+
writeFiles(TESTS_DIR, {[filename]: template([])});
180+
181+
const {stderr, exitCode} = runJest(DIR, ['-w=1', '--ci=false', filename]);
182+
const snapshot = fs.readFileSync(
183+
path.join(TESTS_DIR, '__snapshots__', `${filename}.snap`),
184+
'utf8',
185+
);
186+
187+
// The two full names collide, so the second test's key must keep counting
188+
// from the first test's, even though the flaky test was cleared in between.
189+
expect(exitCode).toBe(0);
190+
expect(stderr).toMatch('Snapshots: 2 written, 2 total');
191+
expect(snapshot).toContain('exports[`a b c 1`] = `"first"`;');
192+
expect(snapshot).toContain('exports[`a b c 2`] = `"second"`;');
193+
});
194+
195+
test('scopes retry cleanup to the test, not its enclosing hooks', () => {
196+
const filename = 'hook-snapshots-on-retry.test.js';
197+
const template = makeTemplate(`
198+
let attempt = 0;
199+
jest.retryTimes(1);
200+
201+
describe('suite', () => {
202+
beforeAll(() => {
203+
expect('from beforeAll').toMatchSnapshot();
204+
});
205+
beforeEach(() => {
206+
expect('from beforeEach').toMatchSnapshot();
207+
});
208+
test('flaky', () => {
209+
attempt += 1;
210+
expect('from test').toMatchSnapshot();
211+
expect(attempt).toBe(2);
212+
});
213+
});
214+
`);
215+
216+
writeFiles(TESTS_DIR, {[filename]: template([])});
217+
218+
const {stderr, exitCode} = runJest(DIR, ['-w=1', '--ci=false', filename]);
219+
220+
// `beforeAll` runs outside the test, so its snapshot is written once and
221+
// survives the retry. The `beforeEach` and test snapshots belong to the test,
222+
// so the discarded attempt's copies are rolled back and written once each.
223+
expect(exitCode).toBe(0);
224+
expect(stderr).toMatch('3 snapshots written from 1 test suite.');
225+
expect(stderr).toMatch('Snapshots: 3 written, 3 total');
226+
});
227+
228+
test('keeps concurrent tests independent when one of them retries', () => {
229+
const filename = 'concurrent-snapshots-on-retry.test.js';
230+
const template = makeTemplate(`
231+
let attempt = 0;
232+
jest.retryTimes(1);
233+
234+
test.concurrent('steady', async () => {
235+
expect('steady value').toMatchSnapshot();
236+
});
237+
test.concurrent('flaky', async () => {
238+
attempt += 1;
239+
expect('flaky value').toMatchSnapshot();
240+
expect(attempt).toBe(2);
241+
});
242+
`);
243+
244+
writeFiles(TESTS_DIR, {[filename]: template([])});
245+
246+
const {stderr, exitCode} = runJest(DIR, [
247+
'-w=1',
248+
'--maxConcurrency=1',
249+
'--ci=false',
250+
filename,
251+
]);
252+
253+
expect(exitCode).toBe(0);
254+
expect(stderr).toMatch('2 snapshots written from 1 test suite.');
255+
expect(stderr).toMatch('Snapshots: 2 written, 2 total');
256+
});
257+
61258
test('works when multiple tests have snapshots but only one of them failed multiple times', () => {
62259
const filename = 'basic-support.test.js';
63260
const template = makeTemplate(`
@@ -96,7 +293,7 @@ test('works when multiple tests have snapshots but only one of them failed multi
96293
[filename]: template(['index', '4' /* retries */]),
97294
});
98295
const {stderr, exitCode} = runJest(DIR, ['-w=1', '--ci=false', filename]);
99-
expect(stderr).toMatch('Snapshots: 1 passed, 1 total');
296+
expect(stderr).toMatch('Snapshots: 2 passed, 2 total');
100297
expect(exitCode).toBe(0);
101298
}
102299
});

‎packages/expect/src/types.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,12 @@ export interface MatcherUtils {
6565
export interface MatcherState {
6666
assertionCalls: number;
6767
currentConcurrentTestName?: () => string | undefined;
68+
/**
69+
* Stable identity of the currently running test: the same object across the
70+
* retries of one test, a different object for every other test, even when
71+
* full test names collide. `undefined` outside a test.
72+
*/
73+
currentTestIdentity?: () => object | undefined;
6874
currentTestName?: string;
6975
error?: Error;
7076
expand?: boolean;
Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
/**
2+
* Copyright (c) Meta Platforms, Inc. and affiliates.
3+
*
4+
* This source code is licensed under the MIT license found in the
5+
* LICENSE file in the root directory of this source tree.
6+
*/
7+
8+
import {afterEach, beforeEach, expect, test} from '@jest/globals';
9+
import {jestExpect} from '@jest/expect';
10+
import type {Circus, Global} from '@jest/types';
11+
import {test as circusTest} from '../index';
12+
import run from '../run';
13+
import {getState, resetState} from '../state';
14+
import {RETRY_TIMES} from '../types';
15+
16+
let originalRetryTimes: unknown;
17+
18+
beforeEach(() => {
19+
resetState();
20+
originalRetryTimes = (globalThis as Global.Global)[RETRY_TIMES];
21+
(globalThis as Global.Global)[RETRY_TIMES] = '1';
22+
});
23+
24+
afterEach(() => {
25+
if (originalRetryTimes === undefined) {
26+
Reflect.deleteProperty(globalThis, RETRY_TIMES);
27+
} else {
28+
(globalThis as Global.Global)[RETRY_TIMES] = originalRetryTimes;
29+
}
30+
});
31+
32+
test('expect state exposes the same test entry across retries', async () => {
33+
const observedNames: Array<string | undefined> = [];
34+
const observedIdentities: Array<object | undefined> = [];
35+
let attempts = 0;
36+
37+
circusTest('retried test', () => {
38+
attempts++;
39+
const expectState = jestExpect.getState();
40+
observedNames.push(expectState.currentConcurrentTestName?.());
41+
observedIdentities.push(expectState.currentTestIdentity?.());
42+
if (attempts === 1) {
43+
throw new Error('retry');
44+
}
45+
});
46+
47+
const testEntry = getState().rootDescribeBlock
48+
.children[0] as Circus.TestEntry;
49+
const result = await run();
50+
51+
expect(result.testResults).toHaveLength(1);
52+
expect(result.testResults[0].status).toBe('done');
53+
expect(attempts).toBe(2);
54+
expect(observedNames).toEqual(['retried test', 'retried test']);
55+
expect(observedIdentities).toEqual([testEntry, testEntry]);
56+
expect(jestExpect.getState().currentConcurrentTestName?.()).toBeUndefined();
57+
expect(jestExpect.getState().currentTestIdentity?.()).toBeUndefined();
58+
});

‎packages/jest-circus/src/legacy-code-todo-rewrite/jestAdapterInit.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -357,7 +357,7 @@ const handleSnapshotStateAfterRetry =
357357
switch (event.name) {
358358
case 'test_retry': {
359359
// Clear any snapshot data that occurred in previous test run
360-
snapshotState.clear();
360+
snapshotState.clear(event.test);
361361
}
362362
}
363363
};

0 commit comments

Comments
 (0)