test_runner: test runner bail

Author: marco-ippolito

doc/api/cli.md

@@ -1441,6 +1441,16 @@ Starts the Node.js command line test runner. This flag cannot be combined with
 See the documentation on [running tests from the command line][]
 for more details.
 
+### `--test-bail`
+
+<!-- YAML
+added:
+  - REPLACEME
+-->
+
+Specifies the bailout behavior of the test runner when running tests.
+See the documentation on [test bailout][] for more details.
+
 ### `--test-name-pattern`
 
 <!-- YAML
@@ -2643,6 +2653,7 @@ done
 [security warning]: #warning-binding-inspector-to-a-public-ipport-combination-is-insecure
 [semi-space]: https://www.memorymanagement.org/glossary/s.html#semi.space
 [single executable application]: single-executable-applications.md
+[test bailout]: test.md#test-bail
 [test reporters]: test.md#test-reporters
 [timezone IDs]: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
 [tracking issue for user-land snapshots]: https://github.com/nodejs/node/issues/44014

doc/api/test.md

@@ -602,6 +602,23 @@ test('mocks setTimeout to be executed synchronously without having to actually w
 });
 ```
 
+## Test bail
+
+<!-- YAML
+added:
+  - REPLACEME
+-->
+
+```bash
+node --test --test-bail=1
+```
+
+An integer value representing the number of test failures after
+which the test execution should stop.
+If set to `0`, the test runner will not bail out,
+and all tests will run regardless of failures.
+**Default:** `0`.
+
 ## Test reporters
 
 <!-- YAML

lib/internal/main/test_runner.js

@@ -57,7 +57,13 @@ if (shardOption) {
   };
 }
 
-run({ concurrency, inspectPort, watch: getOptionValue('--watch'), setup: setupTestReporters, shard })
-.once('test:fail', () => {
+run({
+  concurrency,
+  inspectPort,
+  watch: getOptionValue('--watch'),
+  setup: setupTestReporters,
+  shard,
+  bail: getOptionValue('--test-bail'),
+}).once('test:fail', () => {
   process.exitCode = kGenericUserError;
 });

lib/internal/test_runner/harness.js

@@ -179,6 +179,7 @@ function setup(root) {
       topLevel: 0,
       suites: 0,
     },
+    bailAmount: globalOptions.bail,
     shouldColorizeTestFiles: false,
   };
   root.startTime = hrtime();

lib/internal/test_runner/runner.js

@@ -136,6 +136,10 @@ class FileTest extends Test {
   #rawBufferSize = 0;
   #reportedChildren = 0;
   failedSubtests = false;
+
+  #bailAmount = this.root.harness.bailAmount;
+  #failedTestsCount = 0;
+
   #skipReporting() {
     return this.#reportedChildren > 0 && (!this.error || this.error.failureType === kSubtestsFailed);
   }
@@ -179,6 +183,11 @@ class FileTest extends Test {
     if (item.type !== 'test:pass' && item.type !== 'test:fail') {
       return;
     }
+
+    if (item.type === 'test:fail') {
+      this.#failedTestsCount++;
+    }
+
     this.#reportedChildren++;
     if (item.data.nesting === 0 && item.type === 'test:fail') {
       this.failedSubtests = true;
@@ -191,6 +200,9 @@ class FileTest extends Test {
     }
   }
   addToReport(item) {
+    if (this.#bailAmount && this.#failedTestsCount > this.#bailAmount) {
+      return;
+    }
     this.#accumulateReportItem(item);
     if (!this.isClearToSend()) {
       ArrayPrototypePush(this.#reportBuffer, item);
@@ -425,7 +437,7 @@ function run(options) {
     options = kEmptyObject;
   }
   let { testNamePatterns, shard } = options;
-  const { concurrency, timeout, signal, files, inspectPort, watch, setup } = options;
+  const { concurrency, timeout, signal, files, inspectPort, watch, setup, bail } = options;
 
   if (files != null) {
     validateArray(files, 'options.files');
@@ -469,6 +481,10 @@ function run(options) {
     });
   }
 
+  if (bail != null) {
+    validateInteger(bail, 'options.bail', 0);
+  }
+
   const root = createTestTree({ concurrency, timeout, signal });
   let testFiles = files ?? createTestFileList();
 

lib/internal/test_runner/utils.js

@@ -179,6 +179,7 @@ function parseCommandLine() {
   }
 
   const isTestRunner = getOptionValue('--test');
+  const bail = getOptionValue('--test-bail');
   const coverage = getOptionValue('--experimental-test-coverage');
   const isChildProcess = process.env.NODE_TEST_CONTEXT === 'child';
   const isChildProcessV8 = process.env.NODE_TEST_CONTEXT === 'child-v8';
@@ -230,6 +231,7 @@ function parseCommandLine() {
   globalTestOptions = {
     __proto__: null,
     isTestRunner,
+    bail,
     coverage,
     testOnlyFlag,
     testNamePatterns,

src/node_options.cc

@@ -567,6 +567,9 @@ EnvironmentOptionsParser::EnvironmentOptionsParser() {
             "profile generated with --heap-prof. (default: 512 * 1024)",
             &EnvironmentOptions::heap_prof_interval);
 #endif  // HAVE_INSPECTOR
+  AddOption("--test-bail",
+            "stop test execution when given number of tests have failed",
+            &EnvironmentOptions::test_bail);
   AddOption("--max-http-header-size",
             "set the maximum size of HTTP headers (default: 16384 (16KB))",
             &EnvironmentOptions::max_http_header_size,

src/node_options.h

@@ -159,6 +159,7 @@ class EnvironmentOptions : public Options {
   std::string redirect_warnings;
   std::string diagnostic_dir;
   bool test_runner = false;
+  uint64_t test_bail = 0;
   bool test_runner_coverage = false;
   std::vector<std::string> test_name_pattern;
   std::vector<std::string> test_reporter;

test/fixtures/test-runner/bail/bail.js

@@ -0,0 +1,16 @@
+const test = require('node:test');
+
+test('nested', (t) => {
+  t.test('ok', () => {});
+  t.test('failing', () => {
+    throw new Error('first');
+  });
+});
+
+test('top level', (t) => {
+  t.test('ok', () => {});
+  t.test('failing', () => {
+    throw new Error('second');
+  });
+});
+

test/fixtures/test-runner/bail/multiple.js

@@ -0,0 +1,8 @@
+const test = require('node:test');
+
+test('multiple', (t) => {
+  t.test('ok', () => {});
+  t.test('failing', () => {
+    throw new Error('first');
+  });
+});

test/parallel/test-runner-bail.js

@@ -0,0 +1,37 @@
+'use strict';
+
+require('../common');
+const fixtures = require('../common/fixtures');
+const tmpdir = require('../common/tmpdir');
+const { describe, it } = require('node:test');
+const { spawnSync } = require('node:child_process');
+const assert = require('node:assert');
+
+const testFile = fixtures.path('test-runner/bail/bail.js');
+tmpdir.refresh();
+
+describe('node:test bail', () => {
+  it('should exit at first failure', async () => {
+    const child = spawnSync(process.execPath, ['--test', '--test-bail=1', testFile]);
+    console.log(child.stdout.toString());
+    assert.strictEqual(child.stderr.toString(), '');
+    assert.match(child.stdout.toString(), /TAP version 13/);
+    assert.match(child.stdout.toString(), /ok 1 - ok/);
+    assert.match(child.stdout.toString(), /not ok 2 - failing/);
+    assert.match(child.stdout.toString(), /not ok 1 - nested/);
+    assert.doesNotMatch(child.stdout.toString(), /not ok 2 - top level/);
+    assert.doesNotMatch(child.stdout.toString(), /Subtest: top level/);
+  });
+
+  it('should exit at second failure', async () => {
+    const child = spawnSync(process.execPath, ['--test', '--test-bail=2', testFile]);
+    console.log(child.stdout.toString());
+    assert.strictEqual(child.stderr.toString(), '');
+    assert.match(child.stdout.toString(), /TAP version 13/);
+    assert.match(child.stdout.toString(), /ok 1 - ok/);
+    assert.match(child.stdout.toString(), /not ok 2 - failing/);
+    assert.match(child.stdout.toString(), /not ok 1 - nested/);
+    assert.match(child.stdout.toString(), /not ok 2 - failing/);
+    assert.match(child.stdout.toString(), /Subtest: top level/);
+  });
+});