diff --git a/doc/api/test.md b/doc/api/test.md index a2c98388fa4f..c48648baf170 100644 --- a/doc/api/test.md +++ b/doc/api/test.md @@ -160,6 +160,8 @@ in this example, there are two run attempts, with two tests defined in `test.js` the first test succeeded on the first attempt, and the second test succeeded on the second attempt. When the `--test-rerun-failures` option is used, the test runner will only run tests that have not yet passed. +A `todo` test that fails counts as not yet passed and is rerun, even though its failure does not +cause the test run to fail. ```bash node --test-rerun-failures /path/to/state/file diff --git a/lib/internal/test_runner/reporter/rerun.js b/lib/internal/test_runner/reporter/rerun.js index 3f9ae102fea3..ba2a1fe39141 100644 --- a/lib/internal/test_runner/reporter/rerun.js +++ b/lib/internal/test_runner/reporter/rerun.js @@ -21,7 +21,13 @@ function reportReruns(previousRuns, globalOptions) { function startTest(data) { const originalSuite = currentSuite; - currentSuite = { __proto__: null, data, parent: currentSuite, children: [] }; + currentSuite = { + __proto__: null, + data, + parent: currentSuite, + children: [], + hasFailedDescendant: false, + }; if (originalSuite?.children) { ArrayPrototypePush(originalSuite.children, currentSuite); } @@ -57,7 +63,11 @@ function reportReruns(previousRuns, globalOptions) { } else { disambiguator[baseIdentifier] = 1; } - if (type === 'test:pass') { + if (type === 'test:fail') { + for (let t = currentTest.parent; t !== null; t = t.parent) { + t.hasFailedDescendant = true; + } + } else if (!currentTest.hasFailedDescendant) { const children = ArrayPrototypeMap(currentTest.children, (child) => child.data); obj[identifier] = { __proto__: null, diff --git a/test/fixtures/test-runner/rerun-todo.js b/test/fixtures/test-runner/rerun-todo.js new file mode 100644 index 000000000000..c8b9fbb99a93 --- /dev/null +++ b/test/fixtures/test-runner/rerun-todo.js @@ -0,0 +1,21 @@ +const { test, describe } = require('node:test'); + +test('top-level todo fails on first attempt', { todo: true }, ({ attempt }) => { + if (attempt < 1) { + throw new Error('This todo test is expected to fail on the first attempt'); + } +}); + +describe('suite with failing todo', () => { + test('nested todo fails on first attempt', { todo: true }, ({ attempt }) => { + if (attempt < 1) { + throw new Error('This todo test is expected to fail on the first attempt'); + } + }); + + test('ok', ({ attempt }) => { + if (attempt > 0) { + throw new Error('Test should not rerun once it has passed'); + } + }); +}); diff --git a/test/parallel/test-runner-test-rerun-failures.js b/test/parallel/test-runner-test-rerun-failures.js index 0af3722fad70..585406ec919c 100644 --- a/test/parallel/test-runner-test-rerun-failures.js +++ b/test/parallel/test-runner-test-rerun-failures.js @@ -283,3 +283,73 @@ test('using `run` api', async () => { await setTimeout(common.platformTimeout(10)); // Wait for the stream to finish processing assert.deepStrictEqual(await getStateFile(), expectedStateFile); }); + +test('failing todo test is rerun even when its parent suite passed', async () => { + const fixturePath = fixtures.path('test-runner', 'rerun-todo.js'); + const fixtureKey = relative(process.cwd(), fixturePath).replaceAll('\\', '/'); + const topLevelTodo = `${fixtureKey}:3:1`; + const suite = `${fixtureKey}:9:1`; + const nestedTodo = `${fixtureKey}:10:3`; + const ok = `${fixtureKey}:16:3`; + + async function runAttempt() { + const results = { __proto__: null, passed: [], failed: [], summaries: [] }; + const stream = run({ files: [fixturePath], rerunFailuresFilePath: stateFile }); + for await (const { type, data } of stream) { + const { name, todo, details } = data; + if (type === 'test:pass') { + results.passed.push({ name, todo, attempt: details.attempt, passed_on_attempt: details.passed_on_attempt }); + } else if (type === 'test:fail') { + results.failed.push({ name, todo, attempt: details.attempt }); + } else if (type === 'test:summary') { + results.summaries.push(data); + } + } + await setTimeout(common.platformTimeout(10)); // Wait for the stream to finish processing + return results; + } + + let results = await runAttempt(); + assert.deepStrictEqual(results.failed, [ + { name: 'top-level todo fails on first attempt', todo: true, attempt: 0 }, + { name: 'nested todo fails on first attempt', todo: true, attempt: 0 }, + ]); + assert.deepStrictEqual(results.passed, [ + { name: 'ok', todo: undefined, attempt: 0, passed_on_attempt: undefined }, + { name: 'suite with failing todo', todo: undefined, attempt: 0, passed_on_attempt: undefined }, + ]); + for (const { success, counts } of results.summaries) { + assert.strictEqual(success, true); + assert.strictEqual(counts.failed, 0); + assert.strictEqual(counts.passed, 1); + assert.strictEqual(counts.todo, 2); + } + let state = await getStateFile(); + assert.strictEqual(state.length, 1); + assert.deepStrictEqual(state[0], { + [ok]: { passed_on_attempt: 0, name: 'ok' }, + }); + + results = await runAttempt(); + assert.deepStrictEqual(results.failed, []); + assert.deepStrictEqual(results.passed, [ + { name: 'top-level todo fails on first attempt', todo: true, attempt: 1, passed_on_attempt: undefined }, + { name: 'nested todo fails on first attempt', todo: true, attempt: 1, passed_on_attempt: undefined }, + { name: 'ok', todo: undefined, attempt: 1, passed_on_attempt: 0 }, + { name: 'suite with failing todo', todo: undefined, attempt: 1, passed_on_attempt: undefined }, + ]); + for (const { success, counts } of results.summaries) { + assert.strictEqual(success, true); + assert.strictEqual(counts.failed, 0); + assert.strictEqual(counts.passed, 1); + assert.strictEqual(counts.todo, 2); + } + state = await getStateFile(); + assert.strictEqual(state.length, 2); + assert.deepStrictEqual(state[1], { + [topLevelTodo]: { passed_on_attempt: 1, name: 'top-level todo fails on first attempt' }, + [nestedTodo]: { passed_on_attempt: 1, name: 'nested todo fails on first attempt' }, + [ok]: { passed_on_attempt: 0, name: 'ok' }, + [suite]: { passed_on_attempt: 1, name: 'suite with failing todo' }, + }); +});