fix: normalize ci: false into noCi: true after configs get merged (#1732) thanks @dominykas

This makes sure that options.ci is respected even when set inside a shareable config

CONTRIBUTING.md

@@ -229,7 +229,7 @@ Before pushing your code changes make sure there are no linting errors with `npm
 
 ### Tests
 
-Running the integration test requires you to install [Docker](https://docs.docker.com/engine/installation) on your machine. Note: the tests assume that running `git init` will create a `master` branch by default. If your local `git` is configured differently (see [`init.defaultBranch`](https://github.blog/2020-07-27-highlights-from-git-2-28/#introducing-init-defaultbranch)), change it temporarily when running the tests.
+Running the integration test requires you to install [Docker](https://docs.docker.com/engine/installation) on your machine.
 
 All the [semantic-release](https://github.com/semantic-release) repositories use [AVA](https://github.com/avajs/ava) for writing and running tests.
 

docs/extending/plugins-list.md

@@ -105,3 +105,22 @@
   - `verifyConditions`: Locate and validate a `.gemspec` file, locate and validate a `lib/**/version.rb` file, verify the presence of the `GEM_HOST_API_KEY` environment variable, and create a credentials file with the API key.
   - `prepare`: Update the version in the `lib/**/version.rb` version file and [build](https://guides.rubygems.org/command-reference/#gem-build) the gem.
   - `publish`: [Push the Ruby gem](https://guides.rubygems.org/command-reference/#gem-push) to the gem server. 
+- [semantic-release-npm-deprecate-old-versions](https://github.com/ghusse/semantic-release-npm-deprecate-old-versions)
+  - `verifyConditions`: Validates configuration.
+  - `publish`: Deprecates old versions, based on the declaration of supported versions in the config. 
+- [amanda-mitchell/semantic-release-npm-multiple](https://github.com/amanda-mitchell/semantic-release-npm-multiple)
+  - **Note**: this is a thin wrapper around the built-in npm plugin that can target multiple registries
+  - `verifyConditions`: Verify the presence and the validity of the npm authentication and release configuration for multiple registries
+  - `prepare`: Update the package.json version and create the npm package tarball
+  - `publish`: Publish the package on the npm registry for multiple registries
+- [semantic-release-license](https://github.com/cbhq/semantic-release-license) Automatically update dates and more in your license file for new releases.
+  - `verifyConditions`: Verify the presence of a license file
+  - `prepare`: Update the license file based on its type
+- [semantic-release-pypi](https://github.com/abichinger/semantic-release-pypi)
+  - `verifyConditions`: Verify the environment variable ```PYPI_TOKEN``` and installation of build tools
+  - `prepare`: Update the version in ```setup.cfg``` and create the distribution packages
+  - `publish`: Publish the python package to a repository (default: pypi)
+- [semantic-release-helm](https://github.com/m1pl/semantic-release-helm)
+  - `verifyConditions`: Validate configuration and (if present) credentials
+  - `prepare`: Update version and appVersion in ```Chart.yaml```
+  - `publish`: Publish the chart to a registry (if configured)

docs/recipes/github-actions.md

@@ -4,7 +4,7 @@
 
 The [Authentication](../usage/ci-configuration.md#authentication) environment variables can be configured with [Secret Variables](https://help.github.com/en/articles/virtual-environments-for-github-actions#creating-and-using-secrets-encrypted-variables).
 
-In this example an [`NPM_TOKEN`](https://docs.npmjs.com/creating-and-viewing-authentication-tokens) is required to publish a package to the npm registry. GitHub Actions [automatically populate](https://help.github.com/en/articles/virtual-environments-for-github-actions#github_token-secret) a [`GITHUB_TOKEN`](https://help.github.com/en/articles/creating-a-personal-access-token-for-the-command-line) environment variable which can be used in Workflows.
+In this example a publish type [`NPM_TOKEN`](https://docs.npmjs.com/creating-and-viewing-authentication-tokens) is required to publish a package to the npm registry. GitHub Actions [automatically populate](https://help.github.com/en/articles/virtual-environments-for-github-actions#github_token-secret) a [`GITHUB_TOKEN`](https://help.github.com/en/articles/creating-a-personal-access-token-for-the-command-line) environment variable which can be used in Workflows.
 
 ## Node project configuration
 

docs/usage/plugins.md

@@ -12,6 +12,7 @@ A plugin is a npm module that can implement one or more of the following steps:
 | `generateNotes`    | No       | Responsible for generating the content of the release note. If multiple plugins with a `generateNotes` step are defined, the release notes will be the result of the concatenation of each plugin output.            |
 | `prepare`          | No       | Responsible for preparing the release, for example creating or updating files such as `package.json`, `CHANGELOG.md`, documentation or compiled assets and pushing a commit.                                         |
 | `publish`          | No       | Responsible for publishing the release.                                                                                                                                                                              |
+| `addChannel`       | No       | Responsible for adding a release channel (e.g. adding an npm dist-tag to a release).                                                                                                                                 |
 | `success`          | No       | Responsible for notifying of a new release.                                                                                                                                                                          |
 | `fail`             | No       | Responsible for notifying of a failed release.                                                                                                                                                                       |
 

lib/definitions/errors.js

@@ -29,10 +29,6 @@ Please make sure to add the \`repositoryUrl\` to the [semantic-release configura
       'docs/usage/configuration.md'
     )}).`,
   }),
-  EDUPLICATEREPOSITORYKEY: ({packageJsonPath}) => ({
-    message: 'Duplicate `"repository"` key in package.json.',
-    details: `Your package.json file at ${packageJsonPath} has more than one "repository" keys.`,
-  }),
   EGITNOPERMISSION: ({options: {repositoryUrl}, branch: {name}}) => ({
     message: 'Cannot push to the Git repository.',
     details: `**semantic-release** cannot push the version tag to the branch \`${name}\` on the remote Git repository with URL \`${repositoryUrl}\`.

lib/get-config.js

@@ -1,15 +1,12 @@
-const {readFile} = require('fs').promises;
 const {castArray, pickBy, isNil, isString, isPlainObject} = require('lodash');
-const findPkgUp = require('pkg-up');
+const readPkgUp = require('read-pkg-up');
 const {cosmiconfig} = require('cosmiconfig');
 const resolveFrom = require('resolve-from');
-const findDuplicatedPropertyKeys = require('find-duplicated-property-keys');
 const debug = require('debug')('semantic-release:config');
 const {repoUrl} = require('./git');
 const PLUGINS_DEFINITIONS = require('./definitions/plugins');
 const plugins = require('./plugins');
 const {validatePlugin, parseConfig} = require('./plugins/utils');
-const getError = require('./get-error');
 
 const CONFIG_NAME = 'release';
 const CONFIG_FILES = [
@@ -30,9 +27,6 @@ module.exports = async (context, cliOptions) => {
 
   // Merge config file options and CLI/API options
   let options = {...config, ...cliOptions};
-  if (options.ci === false) {
-    options.noCi = true;
-  }
 
   const pluginsPath = {};
   let extendPaths;
@@ -77,7 +71,7 @@ module.exports = async (context, cliOptions) => {
       {name: 'beta', prerelease: true},
       {name: 'alpha', prerelease: true},
     ],
-    repositoryUrl: (await pkgRepoUrl({cwd})) || (await repoUrl({cwd, env})),
+    repositoryUrl: (await pkgRepoUrl({normalize: false, cwd})) || (await repoUrl({cwd, env})),
     tagFormat: `v\${version}`,
     plugins: [
       '@semantic-release/commit-analyzer',
@@ -90,24 +84,16 @@ module.exports = async (context, cliOptions) => {
     ...(options.branches ? {branches: castArray(options.branches)} : {}),
   };
 
+  if (options.ci === false) {
+    options.noCi = true;
+  }
+
   debug('options values: %O', options);
 
   return {options, plugins: await plugins({...context, options}, pluginsPath)};
 };
 
 async function pkgRepoUrl(options) {
-  const packageJsonPath = await findPkgUp(options);
-  if (!packageJsonPath) return;
-
-  const packageJsonString = await readFile(packageJsonPath, 'utf-8');
-  const result = findDuplicatedPropertyKeys(packageJsonString);
-
-  if (result.length > 0) {
-    throw getError('EDUPLICATEREPOSITORYKEY', {packageJsonPath});
-  }
-
-  const {repository} = require(packageJsonPath);
-  if (!repository) return;
-
-  return isPlainObject(repository) ? repository.url : repository;
+  const {packageJson} = (await readPkgUp(options)) || {};
+  return packageJson && (isPlainObject(packageJson.repository) ? packageJson.repository.url : packageJson.repository);
 }

lib/get-git-auth-url.js

@@ -4,6 +4,48 @@ const hostedGitInfo = require('hosted-git-info');
 const {verifyAuth} = require('./git');
 const debug = require('debug')('semantic-release:get-git-auth-url');
 
+/**
+ * Machinery to format a repository URL with the given credentials
+ *
+ * @param {String} protocol URL protocol (which should not be present in repositoryUrl)
+ * @param {String} repositoryUrl User-given repository URL
+ * @param {String} gitCredentials The basic auth part of the URL
+ *
+ * @return {String} The formatted Git repository URL.
+ */
+function formatAuthUrl(protocol, repositoryUrl, gitCredentials) {
+  const [match, auth, host, basePort, path] =
+    /^(?!.+:\/\/)(?:(?<auth>.*)@)?(?<host>.*?):(?<port>\d+)?:?\/?(?<path>.*)$/.exec(repositoryUrl) || [];
+  const {port, hostname, ...parsed} = parse(
+    match ? `ssh://${auth ? `${auth}@` : ''}${host}${basePort ? `:${basePort}` : ''}/${path}` : repositoryUrl
+  );
+
+  return format({
+    ...parsed,
+    auth: gitCredentials,
+    host: `${hostname}${protocol === 'ssh:' ? '' : port ? `:${port}` : ''}`,
+    protocol: protocol && /http[^s]/.test(protocol) ? 'http' : 'https',
+  });
+}
+
+/**
+ * Verify authUrl by calling git.verifyAuth, but don't throw on failure
+ *
+ * @param {Object} context semantic-release context.
+ * @param {String} authUrl Repository URL to verify
+ *
+ * @return {String} The authUrl as is if the connection was successfull, null otherwise
+ */
+async function ensureValidAuthUrl({cwd, env, branch}, authUrl) {
+  try {
+    await verifyAuth(authUrl, branch.name, {cwd, env});
+    return authUrl;
+  } catch (error) {
+    debug(error);
+    return null;
+  }
+}
+
 /**
  * Determine the the git repository URL to use to push, either:
  * - The `repositoryUrl` as is if allowed to push
@@ -15,7 +57,8 @@ const debug = require('debug')('semantic-release:get-git-auth-url');
  *
  * @return {String} The formatted Git repository URL.
  */
-module.exports = async ({cwd, env, branch, options: {repositoryUrl}}) => {
+module.exports = async (context) => {
+  const {cwd, env, branch} = context;
   const GIT_TOKENS = {
     GIT_CREDENTIALS: undefined,
     GH_TOKEN: undefined,
@@ -30,6 +73,7 @@ module.exports = async ({cwd, env, branch, options: {repositoryUrl}}) => {
     BITBUCKET_TOKEN_BASIC_AUTH: '',
   };
 
+  let {repositoryUrl} = context.options;
   const info = hostedGitInfo.fromUrl(repositoryUrl, {noGitPlus: true});
   const {protocol, ...parsed} = parse(repositoryUrl);
 
@@ -47,24 +91,30 @@ module.exports = async ({cwd, env, branch, options: {repositoryUrl}}) => {
     await verifyAuth(repositoryUrl, branch.name, {cwd, env});
   } catch (_) {
     debug('SSH key auth failed, falling back to https.');
+    const envVars = Object.keys(GIT_TOKENS).filter((envVar) => !isNil(env[envVar]));
 
-    const envVar = Object.keys(GIT_TOKENS).find((envVar) => !isNil(env[envVar]));
-    const gitCredentials = `${GIT_TOKENS[envVar] || ''}${env[envVar] || ''}`;
+    // Skip verification if there is no ambiguity on which env var to use for authentication
+    if (envVars.length === 1) {
+      const gitCredentials = `${GIT_TOKENS[envVars[0]] || ''}${env[envVars[0]]}`;
+      return formatAuthUrl(protocol, repositoryUrl, gitCredentials);
+    }
 
-    if (gitCredentials) {
-      // If credentials are set via environment variables, convert the URL to http/https and add basic auth, otherwise return `repositoryUrl` as is
-      const [match, auth, host, path] =
-        /^(?!.+:\/\/)(?:(?<auth>.*)@)?(?<host>.*?):(?<path>.*)$/.exec(repositoryUrl) || [];
-      const {port, hostname, ...parsed} = parse(
-        match ? `ssh://${auth ? `${auth}@` : ''}${host}/${path}` : repositoryUrl
-      );
+    if (envVars.length > 1) {
+      debug(`Found ${envVars.length} credentials in environment, trying all of them`);
 
-      return format({
-        ...parsed,
-        auth: gitCredentials,
-        host: `${hostname}${protocol === 'ssh:' ? '' : port ? `:${port}` : ''}`,
-        protocol: protocol && /http[^s]/.test(protocol) ? 'http' : 'https',
-      });
+      const candidateRepositoryUrls = [];
+      for (const envVar of envVars) {
+        const gitCredentials = `${GIT_TOKENS[envVar] || ''}${env[envVar]}`;
+        const authUrl = formatAuthUrl(protocol, repositoryUrl, gitCredentials);
+        candidateRepositoryUrls.push(ensureValidAuthUrl(context, authUrl));
+      }
+
+      const validRepositoryUrls = await Promise.all(candidateRepositoryUrls);
+      const chosenAuthUrlIndex = validRepositoryUrls.findIndex((url) => url !== null);
+      if (chosenAuthUrlIndex > -1) {
+        debug(`Using "${envVars[chosenAuthUrlIndex]}" to authenticate`);
+        return validRepositoryUrls[chosenAuthUrlIndex];
+      }
     }
   }
 

lib/hide-sensitive.js

@@ -11,7 +11,10 @@ module.exports = (env) => {
     return /token|password|credential|secret|private/i.test(envVar) && size(env[envVar].trim()) >= SECRET_MIN_SIZE;
   });
 
-  const regexp = new RegExp(toReplace.map((envVar) => escapeRegExp(env[envVar])).join('|'), 'g');
+  const regexp = new RegExp(
+    toReplace.map((envVar) => `${escapeRegExp(env[envVar])}|${escapeRegExp(encodeURI(env[envVar]))}`).join('|'),
+    'g'
+  );
   return (output) =>
     output && isString(output) && toReplace.length > 0 ? output.toString().replace(regexp, SECRET_REPLACEMENT) : output;
 };

media/semantic-release-logo.svg

@@ -0,0 +1,3 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="15 0 500 500" width="500px" height="500px">
+    <path fill="#494949" fill-rule="nonzero" d="M265 300a50 50 0 110-100 50 50 0 010 100zm0-15a35 35 0 100-70 35 35 0 000 70zM189 65v36c2 11 21 55 35 81-18-13-58-43-69-58a92 92 0 01-17-29l-71 40v86c9 5 23 15 31 18 11 4 59 9 88 10-21 9-66 29-85 31-16 3-27 1-34 0v85l69 39c9-4 24-12 31-17 9-7 37-46 52-71-2 22-8 71-15 89-6 15-13 23-17 29l79 45 73-42c1-9 2-28 0-38-1-11-21-55-34-81 18 14 57 44 69 59 11 13 15 24 17 30l74-42v-84c-8-5-24-15-34-19s-58-9-87-10c21-10 66-29 85-31 18-3 29-1 36 1v-87l-70-40-31 19c-9 7-37 46-53 70 3-22 9-71 16-89 6-15 13-24 18-29l-79-45-77 44zm77-65l217 125v250L266 500 49 375V125L266 0z"/>
+</svg>

package.json

@@ -31,8 +31,7 @@
     "env-ci": "^5.0.0",
     "execa": "^4.0.0",
     "figures": "^3.0.0",
-    "find-duplicated-property-keys": "^1.2.2",
-    "find-versions": "^3.0.0",
+    "find-versions": "^4.0.0",
     "get-stream": "^5.0.0",
     "git-log-parser": "^1.2.0",
     "hook-std": "^2.0.0",
@@ -43,7 +42,7 @@
     "micromatch": "^4.0.2",
     "p-each-series": "^2.1.0",
     "p-reduce": "^2.0.0",
-    "pkg-up": "^3.1.0",
+    "read-pkg-up": "^7.0.0",
     "resolve-from": "^5.0.0",
     "semver": "^7.3.2",
     "semver-diff": "^3.1.1",
@@ -129,8 +128,7 @@
     "prettier": true,
     "space": true,
     "rules": {
-      "unicorn/string-content": "off",
-      "node/no-unsupported-features/node-builtins": "off"
+      "unicorn/string-content": "off"
     }
   }
 }

test/get-config.test.js

@@ -505,10 +505,10 @@ test('Allow to unset properties defined in shareable config with "undefined"', a
 test('Throw an Error if one of the shareable config cannot be found', async (t) => {
   // Create a git repository, set the current working directory at the root of the repo
   const {cwd} = await gitRepo();
-  const pkhOptions = {extends: ['./shareable1.json', 'non-existing-path']};
+  const pkgOptions = {extends: ['./shareable1.json', 'non-existing-path']};
   const options1 = {analyzeCommits: 'analyzeCommits'};
   // Create package.json and shareable.json in repository root
-  await outputJson(path.resolve(cwd, 'package.json'), {release: pkhOptions});
+  await outputJson(path.resolve(cwd, 'package.json'), {release: pkgOptions});
   await outputJson(path.resolve(cwd, 'shareable1.json'), options1);
 
   await t.throwsAsync(t.context.getConfig({cwd}), {
@@ -517,20 +517,19 @@ test('Throw an Error if one of the shareable config cannot be found', async (t)
   });
 });
 
-test('Throw an Error if package.json has duplicate "repository" key', async (t) => {
+test('Convert "ci" option to "noCi" when set from extended config', async (t) => {
   // Create a git repository, set the current working directory at the root of the repo
   const {cwd} = await gitRepo();
+  const pkgOptions = {extends: './no-ci.json'};
+  const options = {
+    ci: false,
+  };
+  // Create package.json and shareable.json in repository root
+  await outputJson(path.resolve(cwd, 'package.json'), {release: pkgOptions});
+  await outputJson(path.resolve(cwd, 'no-ci.json'), options);
 
-  // Create package.json with duplicate "repository" key
-  await writeFile(
-    path.resolve(cwd, 'package.json'),
-    `{
-    "repository": "https://github.com/octocat/repository",
-    "repository": "https://github.com/octocat/repository"
-  }`
-  );
+  const {options: result} = await t.context.getConfig({cwd});
 
-  const error = await t.throwsAsync(t.context.getConfig({cwd}));
-  t.is(error.code, 'EDUPLICATEREPOSITORYKEY');
-  t.is(error.message, 'Duplicate `"repository"` key in package.json.');
+  t.is(result.ci, false);
+  t.is(result.noCi, true);
 });

test/get-git-auth-url.test.js

@@ -133,6 +133,32 @@ test('Return the "https" formatted URL if "gitCredentials" is defined and reposi
   );
 });
 
+test('Return the "https" formatted URL if "gitCredentials" is defined and repositoryUrl is a "git" URL without user and with a custom port', async (t) => {
+  const {cwd} = await gitRepo();
+
+  t.is(
+    await getAuthUrl({
+      cwd,
+      env: {...env, GIT_CREDENTIALS: 'user:pass'},
+      options: {branch: 'master', repositoryUrl: 'host.null:6666:owner/repo.git'},
+    }),
+    'https://user:pass@host.null:6666/owner/repo.git'
+  );
+});
+
+test('Return the "https" formatted URL if "gitCredentials" is defined and repositoryUrl is a "git" URL without user and with a custom port followed by a slash', async (t) => {
+  const {cwd} = await gitRepo();
+
+  t.is(
+    await getAuthUrl({
+      cwd,
+      env: {...env, GIT_CREDENTIALS: 'user:pass'},
+      options: {branch: 'master', repositoryUrl: 'host.null:6666:/owner/repo.git'},
+    }),
+    'https://user:pass@host.null:6666/owner/repo.git'
+  );
+});
+
 test('Return the "https" formatted URL if "gitCredentials" is defined and repositoryUrl is a "https" URL', async (t) => {
   const {cwd} = await gitRepo();
 

test/hide-sensitive.test.js

@@ -24,6 +24,14 @@ test('Replace sensitive environment variable matching specific regex for "privat
   t.is(hideSensitive(env)(`https://host.com?token=${env.privateKey}`), `https://host.com?token=${SECRET_REPLACEMENT}`);
 });
 
+test('Replace url-encoded environment variable', (t) => {
+  const env = {privateKey: 'secret '};
+  t.is(
+    hideSensitive(env)(`https://host.com?token=${encodeURI(env.privateKey)}`),
+    `https://host.com?token=${SECRET_REPLACEMENT}`
+  );
+});
+
 test('Escape regexp special characters', (t) => {
   const env = {SOME_CREDENTIALS: 'p$^{.+}\\w[a-z]o.*rd'};
   t.is(
@@ -32,6 +40,14 @@ test('Escape regexp special characters', (t) => {
   );
 });
 
+test('Escape regexp special characters in url-encoded environment variable', (t) => {
+  const env = {SOME_PASSWORD: 'secret password p$^{.+}\\w[a-z]o.*rd)('};
+  t.is(
+    hideSensitive(env)(`https://user:${encodeURI(env.SOME_PASSWORD)}@host.com`),
+    `https://user:${SECRET_REPLACEMENT}@host.com`
+  );
+});
+
 test('Accept "undefined" input', (t) => {
   t.is(hideSensitive({})(), undefined);
 });

test/integration.test.js

@@ -6,6 +6,7 @@ const {writeJson, readJson} = require('fs-extra');
 const execa = require('execa');
 const {WritableStreamBuffer} = require('stream-buffers');
 const delay = require('delay');
+const getAuthUrl = require('../lib/get-git-auth-url');
 const {SECRET_REPLACEMENT} = require('../lib/definitions/constants');
 const {
   gitHead,
@@ -656,3 +657,43 @@ test('Hide sensitive environment variable values from the logs', async (t) => {
   t.regex(stderr, new RegExp(`Error: Console token ${escapeRegExp(SECRET_REPLACEMENT)}`));
   t.regex(stderr, new RegExp(`Throw error: Exposing ${escapeRegExp(SECRET_REPLACEMENT)}`));
 });
+
+test('Use the valid git credentials when multiple are provided', async (t) => {
+  const {cwd, authUrl} = await gitbox.createRepo('test-auth');
+
+  t.is(
+    await getAuthUrl({
+      cwd,
+      env: {
+        GITHUB_TOKEN: 'dummy',
+        GITLAB_TOKEN: 'trash',
+        BB_TOKEN_BASIC_AUTH: gitbox.gitCredential,
+        GIT_ASKPASS: 'echo',
+        GIT_TERMINAL_PROMPT: 0,
+      },
+      branch: {name: 'master'},
+      options: {repositoryUrl: 'http://toto@localhost:2080/git/test-auth.git'},
+    }),
+    authUrl
+  );
+});
+
+test('Use the repository URL as is if none of the given git credentials are valid', async (t) => {
+  const {cwd} = await gitbox.createRepo('test-invalid-auth');
+  const dummyUrl = 'http://toto@localhost:2080/git/test-auth.git';
+
+  t.is(
+    await getAuthUrl({
+      cwd,
+      env: {
+        GITHUB_TOKEN: 'dummy',
+        GITLAB_TOKEN: 'trash',
+        GIT_ASKPASS: 'echo',
+        GIT_TERMINAL_PROMPT: 0,
+      },
+      branch: {name: 'master'},
+      options: {repositoryUrl: dummyUrl},
+    }),
+    dummyUrl
+  );
+});