Merge pull request #2964 from semantic-release/test-structure

test(structure): consolidated all verification under the `test` script while also enabling isolated execution

.github/workflows/test.yml

@@ -20,6 +20,7 @@ env:
   NPM_CONFIG_COLOR: always
 
 jobs:
+  # verify against ranges defined as supported in engines.node
   test_matrix:
     strategy:
       matrix:
@@ -29,7 +30,7 @@ jobs:
           - 20
 
     runs-on: ubuntu-latest
-    timeout-minutes: 10
+    timeout-minutes: 5
 
     steps:
       - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4
@@ -41,25 +42,36 @@ jobs:
           node-version: ${{ matrix.node-version }}
           cache: npm
       - run: npm clean-install
-      - name: Ensure dependencies are compatible with the version of node
+      - run: npm audit signatures
-        run: npx ls-engines
+      - run: npm test
-      - run: npm run test:ci
+
+  # verify against the node version defined for development in the .nvmrc
+  test_dev:
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+
+    steps:
+      - uses: actions/checkout@3df4ab11eba7bda6032a0b82a6bb43b11571feac # v4
+      - run: git config --global user.name github-actions
+      - run: git config --global user.email github-actions@github.com
+      - name: Use Node.js from .nvmrc
+        uses: actions/setup-node@5e21ff4d9bc1a8cf6de233a3057d20ec6b3fb69d # v3
+        with:
+          node-version-file: .nvmrc
+          cache: npm
+      - run: npm clean-install
+      - run: npm audit signatures
+      - run: npm test
 
   # separate job to set as required in branch protection,
   # as the build names above change each time Node versions change
   test:
     runs-on: ubuntu-latest
-    needs: test_matrix
+    needs:
+      - test_dev
+      - test_matrix
     if: always()
     steps:
-      - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4
-      - uses: actions/setup-node@5e21ff4d9bc1a8cf6de233a3057d20ec6b3fb69d # v3
-        with:
-          node-version: lts/*
-          cache: npm
-      - run: npm clean-install
-      - run: npm audit signatures
-      - run: npm run lint
       - name: All matrix versions passed
         if: ${{ !(contains(needs.*.result, 'failure')) }}
         run: exit 0

CONTRIBUTING.md

@@ -146,7 +146,7 @@ If possible, make [atomic commits](https://en.wikipedia.org/wiki/Atomic_commit),
 
 - a commit should contain exactly one self-contained functional change
 - a functional change should be contained in exactly one commit
-- a commit should not create an inconsistent state (such as test errors, linting errors, partial fix, feature with documentation etc...)
+- a commit should not create an inconsistent state (such as test errors, linting errors, partial fix, feature without documentation, etc...)
 
 A complex feature can be broken down into multiple commits as long as each one maintains a consistent state and consists of a self-contained change.
 
@@ -241,42 +241,82 @@ $ git clone https://github.com/semantic-release/<repo-name>
 $ cd <repo-name>
 # Assign the original repo to a remote called "upstream"
 $ git remote add upstream https://github.com/semantic-release/<repo-name>
+# Switch your node version to the version defined by the project as the development version
+# This step assumes you have already installed and configured https://github.com/nvm-sh/nvm
+# You may need to run `nvm install` if you have not already installed the development node version
+$ nvm use
 # Install the dependencies
 $ npm install
 ```
 
-### Lint
+### Verification
 
-All the [semantic-release](https://github.com/semantic-release) repositories use [XO](https://github.com/sindresorhus/xo) for linting and [Prettier](https://prettier.io) for formatting.
+The `test` script is structured to execute as much of the verification for the project as possible.
-Prettier formatting will be automatically verified and fixed by XO.
+Ensuring that the `test` script fully passes in the node version defined as the development version in the `.nvmrc`
+minimizes the chances of the test workflow failing after pushing your changes.
 
-Before pushing your code changes make sure there are no linting errors with `npm run lint`.
+> [!IMPORTANT]
+> Before pushing your code changes, be sure to run the verification for the project with `npm test`.
 
-**Tips**:
+[npm-run-all2](https://www.npmjs.com/package/npm-run-all2) is used to enable running multiple independent lint and test
+scripts together from the `test` script.
+This enables the test script to not only run all scripts, but also parallelize some of the scripts to optimize the overall
+time required for verification.
 
-- Most linting errors can be automatically fixed with `npm run lint -- --fix`.
+When a failure occurs with the `test`, the output can be a bit confusing because there may be output from multiple parallel
-- Install the [XO plugin](https://github.com/sindresorhus/xo#editor-plugins) for your editor to see linting errors directly in your editor and automatically fix them on save.
+scripts mixed together.
+To investigate the failure with cleaner output, re-run the problematic script directly using the script name from the label
+included on the left side of the output
 
-### Tests
+```shell
+$ npm run <script-name>
+```
 
-Running the integration test requires you to install [Docker](https://docs.docker.com/engine/installation) on your machine.
+#### Lint
+
+##### Prettier
+
+All the [semantic-release](https://github.com/semantic-release) repositories use [Prettier](https://prettier.io) for formatting.
+Prettier formatting will be automatically verified by the `lint:prettier` script, included in the `test` script.
+
+> [!NOTE]
+> Most linting errors can be automatically fixed with `npm run lint:prettier:fix`.
+
+##### Other Lint Tools
+
+Other tools are used for specific compatibility concerns, but are less likely to result in failures in common contributions.
+Please follow the guidance of these tools if failures are encountered.
+
+#### Tests
+
+> [!NOTE]
+> Before pushing your code changes make sure all **tests pass** and the unit test **coverage is 100%**:
 
 All the [semantic-release](https://github.com/semantic-release) repositories use [AVA](https://github.com/avajs/ava) for writing and running tests.
 
-Before pushing your code changes make sure all **tests pass** and the **coverage is 100%**:
+During development, you can:
-
-```bash
-$ npm run test
-```
-
-**Tips:** During development you can:
 
 - run only a subset of test files with `ava <glob>`, for example `ava test/mytestfile.test.js`
 - run in watch mode with `ava -w` to automatically run a test file when you modify it
 - run only the test you are working on by adding [`.only` to the test definition](https://github.com/avajs/ava#running-specific-tests)
 
+##### Unit Tests
+
+```bash
+$ npm run test:unit
+```
+
+##### Integration Tests
+
+> [!IMPORTANT]
+> Running the integration test requires you to install [Docker](https://docs.docker.com/engine/installation) on your machine.
+
+```bash
+$ npm run test:integration
+```
+
 ### Commits
 
 All the [semantic-release](https://github.com/semantic-release) repositories use [Commitizen](https://github.com/commitizen/cz-cli) to help you create [valid commit messages](#commit-message-guidelines).
 
-After staging your changes with `git add`, run `npm run cm` to start the interactive commit message CLI.
+Assuming you have [installed Commitizen](https://github.com/commitizen/cz-cli#installing-the-command-line-tool), run `git cz` to start the interactive commit message CLI rather than `git commit` when committing.

package.json

@@ -6,7 +6,8 @@
   "author": "Stephan Bönnemann <stephan@boennemann.me> (http://boennemann.me)",
   "ava": {
     "files": [
-      "test/**/*.test.js"
+      "test/**/*.test.js",
+      "!test/integration.test.js"
     ],
     "failFast": true,
     "nodeArguments": [
@@ -61,15 +62,20 @@
     "c8": "8.0.1",
     "clear-module": "4.1.2",
     "codecov": "3.8.3",
+    "cz-conventional-changelog": "3.3.0",
     "dockerode": "3.3.5",
     "file-url": "4.0.0",
     "fs-extra": "11.1.1",
     "got": "13.0.0",
     "js-yaml": "4.1.0",
+    "lockfile-lint": "4.12.1",
+    "ls-engines": "0.9.0",
     "mockserver-client": "5.15.0",
     "nock": "13.3.3",
+    "npm-run-all2": "6.0.6",
     "p-retry": "6.1.0",
     "prettier": "3.0.3",
+    "publint": "0.2.2",
     "sinon": "16.1.0",
     "stream-buffers": "3.0.2",
     "tempy": "3.1.0",
@@ -114,6 +120,14 @@
     ],
     "all": true
   },
+  "lockfile-lint": {
+    "path": "package-lock.json",
+    "type": "npm",
+    "validate-https": true,
+    "allowed-hosts": [
+      "npm"
+    ]
+  },
   "prettier": {
     "printWidth": 120,
     "trailingComma": "es5"
@@ -126,14 +140,22 @@
     "type": "git",
     "url": "git+https://github.com/semantic-release/semantic-release.git"
   },
+  "config": {
+    "commitizen": {
+      "path": "./node_modules/cz-conventional-changelog"
+    }
+  },
   "scripts": {
     "codecov": "codecov -f coverage/coverage-final.json",
-    "lint": "prettier --check \"*.{js,json,md}\" \".github/**/*.{md,yml}\" \"docs/**/*.md\" \"{bin,lib,test}/**/*.js\"",
+    "lint:prettier": "prettier --check \"*.{js,json,md}\" \".github/**/*.{md,yml}\" \"docs/**/*.md\" \"{bin,lib,test}/**/*.js\"",
-    "lint:fix": "prettier --write \"*.{js,json,md}\" \".github/**/*.{md,yml}\" \"docs/**/*.md\" \"{bin,lib,test}/**/*.js\"",
+    "lint:prettier:fix": "prettier --write \"*.{js,json,md}\" \".github/**/*.{md,yml}\" \"docs/**/*.md\" \"{bin,lib,test}/**/*.js\"",
-    "pretest": "npm run lint",
+    "lint:lockfile": "lockfile-lint",
+    "lint:engines": "ls-engines",
+    "lint:publish": "publint --strict",
     "semantic-release": "./bin/semantic-release.js",
-    "test": "c8 ava --verbose",
+    "test": "npm-run-all --print-label --parallel lint:* --parallel test:*",
-    "test:ci": "c8 ava --verbose"
+    "test:unit": "c8 ava --verbose",
+    "test:integration": "ava --verbose test/integration.test.js"
   },
   "renovate": {
     "extends": [