From 454ccc0482015186a0afad71338a43c9d9b8143c Mon Sep 17 00:00:00 2001 From: sangwook Date: Sun, 31 May 2026 00:02:04 +0900 Subject: [PATCH 1/3] test_runner: add flaky option to retry on failure Add a `flaky` option that re-runs a failing test until it passes, intended for tests with unavoidable nondeterminism. Setting `flaky: true` retries up to 20 times; `flaky: ` sets an explicit retry budget. The option is accepted on tests and suites and via the it.flaky/test.flaky/describe.flaky/suite.flaky shorthands; a test-case value overrides an inherited suite value (nearest wins), and `flaky: false` opts a test out. Only the final attempt is observable: intermediate failures emit no test:fail, no per-test diagnostics, and nothing on the node.test error channel. Each result carries a new `retryCount` field on the test:pass, test:fail, and test:complete events (the number of retries performed, `undefined` for non-flaky tests), reporters print a `# FLAKY` directive, and the run summary gains a `flaky` counter. beforeEach/afterEach re-run on every attempt while before/after run once, so per-attempt state is reset and retries do not leak state. An externally aborted test and an expectFailure are not retried. A flaky test whose timeout is exhausted is reported as a failure rather than cancelled. Co-authored-by: vespa7 <98526766+vespa7@users.noreply.github.com> Signed-off-by: sangwook --- doc/api/test.md | 65 ++++ lib/internal/test_runner/harness.js | 3 +- lib/internal/test_runner/reporter/dot.js | 6 +- lib/internal/test_runner/reporter/junit.js | 10 + lib/internal/test_runner/reporter/tap.js | 13 +- lib/internal/test_runner/reporter/utils.js | 7 +- lib/internal/test_runner/runner.js | 8 +- lib/internal/test_runner/test.js | 303 ++++++++++----- lib/internal/test_runner/tests_stream.js | 4 + lib/internal/test_runner/utils.js | 5 + .../fixtures/test-runner/flaky/api-surface.js | 42 ++ .../test-runner/flaky/assert-before-plan.js | 7 + .../test-runner/flaky/channel-silence.js | 10 + test/fixtures/test-runner/flaky/counter.js | 5 + .../flaky/default-twenty-exhaust.js | 8 + .../test-runner/flaky/default-twenty.js | 8 + .../test-runner/flaky/eventually-passes.js | 9 + test/fixtures/test-runner/flaky/exhausts.js | 5 + .../test-runner/flaky/expect-failure.js | 8 + .../test-runner/flaky/external-abort.js | 11 + .../test-runner/flaky/flaky-subtest.js | 11 + test/fixtures/test-runner/flaky/hooks.js | 17 + .../fixtures/test-runner/flaky/inheritance.js | 20 + .../test-runner/flaky/non-throw-retry.js | 12 + .../test-runner/flaky/run-parent-summary.js | 10 + .../flaky/shorthand-equals-options.js | 7 + .../test-runner/flaky/timeout-exhaust.js | 11 + test/fixtures/test-runner/output/flaky.js | 40 ++ .../test-runner/output/flaky.snapshot | 61 +++ test/parallel/test-runner-flaky.js | 360 ++++++++++++++++++ test/test-runner/test-output-flaky.mjs | 12 + 31 files changed, 1006 insertions(+), 92 deletions(-) create mode 100644 test/fixtures/test-runner/flaky/api-surface.js create mode 100644 test/fixtures/test-runner/flaky/assert-before-plan.js create mode 100644 test/fixtures/test-runner/flaky/channel-silence.js create mode 100644 test/fixtures/test-runner/flaky/counter.js create mode 100644 test/fixtures/test-runner/flaky/default-twenty-exhaust.js create mode 100644 test/fixtures/test-runner/flaky/default-twenty.js create mode 100644 test/fixtures/test-runner/flaky/eventually-passes.js create mode 100644 test/fixtures/test-runner/flaky/exhausts.js create mode 100644 test/fixtures/test-runner/flaky/expect-failure.js create mode 100644 test/fixtures/test-runner/flaky/external-abort.js create mode 100644 test/fixtures/test-runner/flaky/flaky-subtest.js create mode 100644 test/fixtures/test-runner/flaky/hooks.js create mode 100644 test/fixtures/test-runner/flaky/inheritance.js create mode 100644 test/fixtures/test-runner/flaky/non-throw-retry.js create mode 100644 test/fixtures/test-runner/flaky/run-parent-summary.js create mode 100644 test/fixtures/test-runner/flaky/shorthand-equals-options.js create mode 100644 test/fixtures/test-runner/flaky/timeout-exhaust.js create mode 100644 test/fixtures/test-runner/output/flaky.js create mode 100644 test/fixtures/test-runner/output/flaky.snapshot create mode 100644 test/parallel/test-runner-flaky.js create mode 100644 test/test-runner/test-output-flaky.mjs diff --git a/doc/api/test.md b/doc/api/test.md index 2d38f136f28332..7ff55cb2319829 100644 --- a/doc/api/test.md +++ b/doc/api/test.md @@ -1868,6 +1868,16 @@ added: Shorthand for marking a suite as `TODO`. This is the same as [`suite([name], { todo: true }[, fn])`][suite options]. +## `suite.flaky([name][, options][, fn])` + + + +Shorthand for marking a suite as flaky. This is the same as +[`suite([name], { flaky: true }[, fn])`][suite options]. + ## `suite.only([name][, options][, fn])` + +Shorthand for marking a test as flaky, +same as [`test([name], { flaky: true }[, fn])`][it options]. + ## `describe([name][, options][, fn])` Alias for [`suite()`][]. @@ -2027,6 +2055,16 @@ added: Shorthand for marking a suite as `only`. This is the same as [`describe([name], { only: true }[, fn])`][describe options]. +## `describe.flaky([name][, options][, fn])` + + + +Shorthand for marking a suite as flaky. This is the same as +[`describe([name], { flaky: true }[, fn])`][describe options]. + ## `it([name][, options][, fn])` + +Shorthand for marking a test as flaky, +same as [`it([name], { flaky: true }[, fn])`][it options]. + ## `before([fn][, options])`