[WIP] Dry-run mode (#1825)

* dry-run basic implementation

* added tests and docs for dry-run

* added bootstrap option

* added --bootstrap to dry-run docs

Author: DavertMik

bin/codecept.js

@@ -143,6 +143,22 @@ program.command('run-multiple [suites...]')
 
   .action(require('../lib/command/run-multiple'));
 
+program.command('dry-run [test]')
+  .description('Executes tests multiple')
+  .option('-p, --plugins <k=v,k2=v2,...>', 'enable plugins, comma-separated')
+  .option('--bootstrap', 'enable bootstrap script for dry-run')
+  .option('-c, --config [file]', 'configuration file to be used')
+  .option('--all', 'run all suites')
+  .option('--features', 'run only *.feature files and skip tests')
+  .option('--tests', 'run only JS test files and skip features')
+  .option('-g, --grep <pattern>', 'only run tests matching <pattern>')
+  .option('-f, --fgrep <string>', 'only run tests containing <string>')
+  .option('-i, --invert', 'inverts --grep and --fgrep matches')
+  .option('--steps', 'show step-by-step execution')
+  .option('--verbose', 'output internal logging information')
+  .option('--debug', 'output additional information')
+  .action(require('../lib/command/dryRun'));
+
 program.on('command:*', (cmd) => {
   console.log(`\nUnknown command ${cmd}\n`);
   program.outputHelp();

docs/commands.md

@@ -13,152 +13,178 @@ Executes tests. Requires `codecept.conf.js` config to be present in provided pat
 Run all tests from current dir
 
 ```sh
-codeceptjs run
+npx codeceptjs run
 ```
 
 Load config and run tests from `test` dir
 
 ```sh
-codeceptjs run -c test
+npx codeceptjs run -c test
 ```
 
 Run only tests with "signin" word in name
 
 ```sh
-codeceptjs run --grep "signin"
+npx codeceptjs run --grep "signin"
 ```
 
 Run all tests without "@IEOnly" word in name
 
 ```sh
-codeceptjs run --grep "@IEOnly" --invert
+npx codeceptjs run --grep "@IEOnly" --invert
 ```
 
 Run single test [path to codecept.js] [test filename]
 
 ```sh
-codeceptjs run github_test.js
+npx codeceptjs run github_test.js
 ```
 
 Run single test with steps printed
 
 ```sh
-codeceptjs run github_test.js --steps
+npx codeceptjs run github_test.js --steps
 ```
 
 Run single test in debug mode
 
 ```sh
-codeceptjs run github_test.js --debug
+npx codeceptjs run github_test.js --debug
 ```
 
 Run test with internal logs printed (global promises, and events).
 
 ```sh
-codeceptjs run github_test.js --verbose
+npx codeceptjs run github_test.js --verbose
 ```
 
 Select config file manually (`-c` or `--config` option)
 
 ```sh
-codeceptjs run -c my.codecept.conf.js
-codeceptjs run --config path/to/codecept.json
+npx codeceptjs run -c my.codecept.conf.js
+npx codeceptjs run --config path/to/codecept.json
 ```
 
 Override config on the fly. Provide valid JSON which will be merged into current config:
 
 ```sh
-codeceptjs run --override '{ "helpers": {"WebDriver": {"browser": "chrome"}}}'
+npx codeceptjs run --override '{ "helpers": {"WebDriver": {"browser": "chrome"}}}'
 ```
 
 Run tests and produce xunit report:
 
 ```sh
-codeceptjs run --reporter xunit
+npx codeceptjs run --reporter xunit
 ```
 
 Use any of [Mocha reporters](https://github.com/mochajs/mocha/tree/master/lib/reporters) used.
 
+## Dry Run
+
+Prints test scenarios without executing them
+
+```
+npx codeceptjs dry-run
+```
+
+When passed `--steps` or `--debug` option runs tests, disabling all plugins and helpers, so you can get step-by-step report with no tests actually executed.
+
+```
+npx codeceptjs dry-run --steps
+```
+
+If a plugin needs to be enabled in `dry-run` mode, pass its name in `-p` option:
+
+```
+npx codeceptjs dry-run --steps -p allure
+```
+
+To enable bootstrap script in dry-run mode, pass in `--bootstrap` option when running with `--steps` or `--debug`
+
+```
+nox codeceptjs dry-run --steps --bootstrap
+```
+
 ## Run multiple
 
 Run multiple suites.
 
 ```sh
-codeceptjs run-multiple smoke:chrome regression:firefox
+npx codeceptjs run-multiple smoke:chrome regression:firefox
 ```
 
 ## Init
 
 Creates `codecept.conf.js` file in current directory:
 
 ```sh
-codeceptjs init
+npx codeceptjs init
 ```
 
 Or in provided path
 
 ```sh
-codecept init test
+npx codecept init test
 ```
 
 ## Migrate
 
 Migrate your current `codecept.json` to `codecept.conf.js`
 
 ```sh
-codeceptjs migrate
+npx codeceptjs migrate
 ```
 
 ## Shell
 
 Interactive shell. Allows to try `I.` commands in runtime
 
 ```sh
-codeceptjs shell
+npx codeceptjs shell
 ```
 
 ## Generators
 
 Create new test
 
 ```sh
-codeceptjs generate:test
+npx codeceptjs generate:test
 ```
 
 Create new pageobject
 
 ```sh
-codeceptjs generate:pageobject
+npx codeceptjs generate:pageobject
 ```
 
 Create new helper
 
 ```sh
-codeceptjs generate:helper
+npx codeceptjs generate:helper
 ```
 
 ## TypeScript Definitions
 
 TypeScript Definitions allows IDEs to provide autocompletion when writing tests.
 
 ```sh
-codeceptjs def
-codeceptjs def --config path/to/codecept.json
+npx codeceptjs def
+npx codeceptjs def --config path/to/codecept.json
 ```
 
 After doing that IDE should provide autocompletion for `I` object inside `Scenario` and `within` blocks.
 
 Add optional parameter `output` (or shortcat `-o`), if you want to place your definition file in specific folder:
 
 ```sh
-codeceptjs def --output ./tests/typings
-codeceptjs def -o ./tests/typings
+npx codeceptjs def --output ./tests/typings
+npx codeceptjs def -o ./tests/typings
 ```
 
 ## List Commands
 
 Prints all available methods of `I` to console
 
 ```sh
-codeceptjs list
+npx codeceptjs list
 ```

docs/reports.md

@@ -5,8 +5,6 @@ title: Reporters
 
 ## Cli
 
-(default)
-
 By default CodeceptJS provides cli reporter with console output.
 Test names and failures will be printed to screen.
 
@@ -31,9 +29,15 @@ GitHub --
 
 
   Run with --verbose flag to see NodeJS stacktrace
+
+```
+npx codeceptjs run --stepsutput add `--steps` option to `run` command:
+```
+```
+npx codeceptjs run --steps
 ```
 
-For dynamic step-by-step output add `--steps` option to `run` command:
+Output:
 
 ```sh
 GitHub --
@@ -65,10 +69,22 @@ GitHub --
  ✖ FAILED in 1260ms
 ```
 
-To get additional information about test execution use `--debug` option. This will show execution steps
+To get additional information about test execution use `--debug` option.
+
+
+```
+npx codeceptjs run --debug
+```
+
+
+This will show execution steps
 as well as notices from test runner. To get even more information with more technical details like error stack traces,
 and global promises, or events use `--verbose` mode.
 
+```
+npx codeceptjs run --verbose
+```
+
 ```sh
 GitHub --
  register
@@ -86,19 +102,54 @@ GitHub --
 
 Please use verbose output when reporting issues to GitHub.
 
+### Dry Run
 
-## Allure
+There is a way to list all tests and their steps without actually executing them. Execute tests in `dry-run` mode to see all available tests:
 
-(recommended)
+```
+npx codeceptjs dry-run
+```
 
+Output:
+
+```
+Tests from /home/davert/projects/codeceptjs/examples:
+
+Business rules --
+  ☐ do something
+Google --
+  ☐ test @123
+GitHub -- /home/davert/projects/codeceptjs/examples/github_test.js
+  ☐ Visit Home Page @retry
+  ☐ search @grop
+  ☐ signin @normal @important @slow
+  ☐ signin2
+  ☐ register
+
+  Total: 3 suites | 7 tests
+
+--- DRY MODE: No tests were executed ---
+```
+
+Pass `--steps` or `--debug` option as in `run` command to also get steps and substeps to be printed. In this mode **tests will be executed** but all helpers and plugins disabled, so no real actions will be performed.
+
+```
+npx codecepjs dry-run --debug
+```
+
+> ℹ If you use custom JavaScript code inside tests, or rely on values from `grab*` commands, dry-run may produce error output.
+
+## Allure
+
+> ℹ  We recommend using Allure reports on CI. Allure is one of the best open-source reporters designed to collect and show test reports in nicest way.
 
 [Allure reporter](http://allure.qatools.ru/#) is a tool to store and display test reports.
 It provides nice web UI which contains all important information on test execution.
 CodeceptJS has built-in support for Allure reports. Inside reports you will have all steps, substeps and screenshots.
 
 ![](https://user-images.githubusercontent.com/220264/45676511-8e052800-bb3a-11e8-8cbb-db5f73de2add.png)
 
-*Disclaimer: Allure is a standalone tool. Please refer to [Allure documentation](https://docs.qameta.io/allure/) to learn more about using Allure reports.*
+> ▶ Allure is a standalone tool. Please refer to [Allure documentation](https://docs.qameta.io/allure/) to learn more about using Allure reports.
 
 Allure requires **Java 8** to work. Then Allure can be installed via NPM:
 
@@ -109,16 +160,16 @@ npm install -g allure-commandline --save-dev
 Add [Allure plugin](https://codecept.io/plugins/#allure) in config under `plugins` section.
 
 ```js
-"plugins": {
-    "allure": {
-    }
+plugins: {
+    allure: {
+  }
 }
 ```
 
 Run tests with allure plugin enabled:
 
 ```
-codeceptjs run --plugins allure
+npx codeceptjs run --plugins allure
 ```
 
 (optionally) To enable allure plugin permanently include `"enabled": true` into plugin config:
@@ -140,6 +191,11 @@ allure serve output
 
 Allure reporter aggregates data from other plugins like [*stepByStepReport*](https://codecept.io/plugins/#stepByStepReport) and [*screenshotOnFail*](https://codecept.io/plugins/#screenshotOnFail)
 
+Allure reports can also be generated for `dry-run` command. So you can get the first report with no tests actually being executed. Enable allure plugin in dry-run options, and pass `--debug` option to print all tests on screen.
+
+```
+npx codeceptjs dry-run --debug -p allure
+```
 
 ## XML
 
@@ -301,7 +357,7 @@ Configure mocha-multi with reports that you want:
 Execute CodeceptJS with mocha-multi reporter:
 
 ```sh
-codeceptjs run --reporter mocha-multi
+npx codeceptjs run --reporter mocha-multi
 ```
 
 This will give you cli with steps in console and HTML report in `output` directory.