feat: Retrieve version gitHead from git tags and unshallow the repo if necessary

Add several fixes and improvements in the identification of the last release gitHead:
- If there is no last release, unshallow the repo in order to retrieve all existing commits
- If git head is not present in last release, try to retrieve it from git tag with format ‘v\<version\>’ or ‘\<version\>’
- If the last release git head cannot be determined and  found in commit history, unshallow the repo and try again
- Throw a ENOGITHEAD error if the gitHead for the last release cannot be found in the npm metadata nor in the git tags, preventing to make release based on the all the commits in the repo as before
- Add integration test for the scenario with a packed repo from which `npm republish` fails to read the git head

Fix #447, Fix #393, Fix #280, Fix #276
This commit is contained in:
Pierre-Denis Vanduynslager 2017-09-30 00:35:47 -04:00 committed by Pierre Vanduynslager
parent cbb51a495b
commit 85dd69b3a2
6 changed files with 390 additions and 66 deletions

View File

@ -13,10 +13,6 @@ branches:
- caribou
- /^greenkeeper.*$/
# Retrieve 999 commits (default is 50) so semantic-release can analyze all commits when there is more than 50 on a PR
git:
depth: 999
# Retry install on fail to avoid failing a build on network/disk/external errors
install:
- travis_retry npm install

View File

@ -1,56 +1,92 @@
const execa = require('execa');
const log = require('npmlog');
const SemanticReleaseError = require('@semantic-release/error');
const getVersionHead = require('./get-version-head');
module.exports = async ({lastRelease, options}) => {
let stdout;
if (lastRelease.gitHead) {
/**
* Commit message.
*
* @typedef {Object} Commit
* @property {string} hash The commit hash.
* @property {string} message The commit message.
*/
/**
* Retrieve the list of commits on the current branch since the last released version, or all the commits of the current branch if there is no last released version.
*
* The commit correspoding to the last released version is determined as follow:
* - Use `lastRelease.gitHead` is defined and present in `config.options.branch` history.
* - Search for a tag named `v<version>` or `<version>` and it's associated commit sha if present in `config.options.branch` history.
*
* If a commit corresponding to the last released is not found, unshallow the repository (as most CI create a shallow clone with limited number of commits and no tags) and try again.
*
* @param {Object} config
* @param {Object} config.lastRelease The lastRelease object obtained from the getLastRelease plugin.
* @param {string} [config.lastRelease.version] The version number of the last release.
* @param {string} [config.lastRelease.gitHead] The commit sha used to make the last release.
* @param {Object} config.options The semantic-relese options.
* @param {string} config.options.branch The branch to release from.
*
* @return {Promise<Array<Commit>>} The list of commits on the branch `config.options.branch` since the last release.
*
* @throws {SemanticReleaseError} with code `ENOTINHISTORY` if `config.lastRelease.gitHead` or the commit sha derived from `config.lastRelease.version` is not in the direct history of `config.options.branch`.
* @throws {SemanticReleaseError} with code `ENOGITHEAD` if `config.lastRelease.gitHead` is undefined and no commit sha can be found for the `config.lastRelease.version`.
*/
module.exports = async ({lastRelease: {version, gitHead}, options: {branch}}) => {
if (gitHead || version) {
try {
({stdout} = await execa('git', ['branch', '--no-color', '--contains', lastRelease.gitHead]));
gitHead = await getVersionHead(version, branch, gitHead);
} catch (err) {
throw notInHistoryError(lastRelease.gitHead, options.branch);
// Unshallow the repository if the gitHead cannot be found and the branch for the last release version
await execa('git', ['fetch', '--unshallow', '--tags'], {reject: false});
}
const branches = stdout
.split('\n')
.map(branch => branch.replace('*', '').trim())
.filter(branch => !!branch);
if (!branches.includes(options.branch)) {
throw notInHistoryError(lastRelease.gitHead, options.branch, branches);
// Try to find the gitHead on the branch again with an unshallowed repository
try {
gitHead = await getVersionHead(version, branch, gitHead);
} catch (err) {
if (err.code === 'ENOTINHISTORY') {
log.error('commits', notInHistoryMessage(gitHead, branch, version, err.branches));
} else if (err.code === 'ENOGITHEAD') {
log.error('commits', noGitHeadMessage());
}
throw err;
}
} else {
// If there is no gitHead nor a version, there is no previous release. Unshallow the repo in order to retrieve all commits
await execa('git', ['fetch', '--unshallow', '--tags'], {reject: false});
}
try {
({stdout} = await execa('git', [
return (await execa('git', [
'log',
'--format=%H==SPLIT==%B==END==',
`${lastRelease.gitHead ? lastRelease.gitHead + '..' : ''}HEAD`,
]));
} catch (err) {
return [];
}
return String(stdout)
'--format=format:%H==SPLIT==%B==END==',
`${gitHead ? gitHead + '..' : ''}HEAD`,
])).stdout
.split('==END==')
.filter(raw => !!raw.trim())
.map(raw => {
const [hash, message] = raw.trim().split('==SPLIT==');
return {hash, message};
});
} catch (err) {
return [];
}
};
function notInHistoryError(gitHead, branch, branches) {
log.error(
'commits',
`
The commit the last release of this package was derived from is not in the direct history of the "${branch}" branch.
This means semantic-release can not extract the commits between now and then.
This is usually caused by force pushing, releasing from an unrelated branch, or using an already existing package name.
You can recover from this error by publishing manually or restoring the commit "${gitHead}".
${branches && branches.length
? `\nHere is a list of branches that still contain the commit in question: \n * ${branches.join('\n * ')}`
: ''}
`
);
return new SemanticReleaseError('Commit not in history', 'ENOTINHISTORY');
function noGitHeadMessage(version) {
return `The commit the last release of this package was derived from cannot be determined from the release metadata not from the repository tags.
This means semantic-release can not extract the commits between now and then.
This is usually caused by releasing from outside the repository directory or with innaccessible git metadata.
You can recover from this error by publishing manually.`;
}
function notInHistoryMessage(gitHead, branch, version, branches) {
return `The commit the last release of this package was derived from is not in the direct history of the "${branch}" branch.
This means semantic-release can not extract the commits between now and then.
This is usually caused by force pushing, releasing from an unrelated branch, or using an already existing package name.
You can recover from this error by publishing manually or restoring the commit "${gitHead}".
${branches && branches.length
? `Here is a list of branches that still contain the commit in question: \n * ${branches.join('\n * ')}`
: ''}`;
}

View File

@ -0,0 +1,67 @@
const SemanticReleaseError = require('@semantic-release/error');
const execa = require('execa');
/**
* Get the commit sha for a given tag.
*
* @param {string} tagName Tag name for which to retrieve the commit sha.
*
* @return {string} The commit sha of the tag in parameter or `null`.
*/
async function gitTagHead(tagName) {
try {
return (await execa('git', ['rev-list', '-1', '--tags', tagName])).stdout;
} catch (err) {
return null;
}
}
/**
* Get the list of branches that contains the given commit.
*
* @param {string} sha The sha of the commit to look for.
*
* @return {Array<string>} The list of branches that contains the commit sha in parameter.
*/
async function getCommitBranches(sha) {
try {
return (await execa('git', ['branch', '--no-color', '--contains', sha])).stdout
.split('\n')
.map(branch => branch.replace('*', '').trim())
.filter(branch => !!branch);
} catch (err) {
return [];
}
}
/**
* Get the commit sha for a given version, if it is contained in the given branch.
*
* @param {string} version The version corresponding to the commit sha to look for. Used to search in git tags.
* @param {string} branch The branch that must have the commit in its direct history.
* @param {string} gitHead The commit sha to verify.
*
* @return {Promise<string>} A Promise that resolves to `gitHead` if defined and if present in branch direct history or the commit sha corresponding to `version`.
*
* @throws {SemanticReleaseError} with code `ENOTINHISTORY` if `gitHead` or the commit sha dereived from `version` is not in the direct history of `branch`. The Error will have a `branches` attributes with the list of branches containing the commit.
* @throws {SemanticReleaseError} with code `ENOGITHEAD` if `gitHead` is undefined and no commit sha can be found for the `version`.
*/
module.exports = async (version, branch, gitHead) => {
if (!gitHead && version) {
// Look for the version tag only if no gitHead exists
gitHead = (await gitTagHead(`v${version}`)) || (await gitTagHead(version));
}
if (gitHead) {
// Retrieve the branches containing the gitHead and verify one of them is the branch in param
const branches = await getCommitBranches(gitHead);
if (!branches.includes(branch)) {
const error = new SemanticReleaseError('Commit not in history', 'ENOTINHISTORY');
error.branches = branches;
throw error;
}
} else {
throw new SemanticReleaseError('There is no commit associated with last release', 'ENOGITHEAD');
}
return gitHead;
};

View File

@ -1,5 +1,5 @@
import test from 'ava';
import {gitRepo, gitCommits, gitCheckout} from './helpers/git-utils';
import {gitRepo, gitCommits, gitCheckout, gitTagVersion, gitShallowClone, gitTags, gitLog} from './helpers/git-utils';
import proxyquire from 'proxyquire';
import {stub} from 'sinon';
import SemanticReleaseError from '@semantic-release/error';
@ -30,7 +30,7 @@ test.serial('Get all commits when there is no last release', async t => {
// Retrieve the commits with the commits module
const result = await getCommits({lastRelease: {}, options: {branch: 'master'}});
// The commits created and and retrieved by the module are identical
// Verify the commits created and retrieved by the module are identical
t.is(result.length, 2);
t.is(result[0].hash.substring(0, 7), commits[0].hash);
t.is(result[0].message, commits[0].message);
@ -38,7 +38,29 @@ test.serial('Get all commits when there is no last release', async t => {
t.is(result[1].message, commits[1].message);
});
test.serial('Get all commits since lastRelease gitHead', async t => {
test.serial('Get all commits when there is no last release, including the ones not in the shallow clone', async t => {
// Create a git repository, set the current working directory at the root of the repo
const repo = await gitRepo();
// Add commits to the master branch
const commits = await gitCommits(['fix: First fix', 'feat: Second feature']);
// Create a shallow clone with only 1 commit
await gitShallowClone(repo);
// Verify the shallow clone contains only one commit
t.is((await gitLog()).length, 1);
// Retrieve the commits with the commits module
const result = await getCommits({lastRelease: {}, options: {branch: 'master'}});
// Verify the commits created and retrieved by the module are identical
t.is(result.length, 2);
t.is(result[0].hash.substring(0, 7), commits[0].hash);
t.is(result[0].message, commits[0].message);
t.is(result[1].hash.substring(0, 7), commits[1].hash);
t.is(result[1].message, commits[1].message);
});
test.serial('Get all commits since gitHead (from lastRelease)', async t => {
// Create a git repository, set the current working directory at the root of the repo
await gitRepo();
// Add commits to the master branch
@ -49,7 +71,76 @@ test.serial('Get all commits since lastRelease gitHead', async t => {
lastRelease: {gitHead: commits[commits.length - 1].hash},
options: {branch: 'master'},
});
// The commits created and retrieved by the module are identical
// Verify the commits created and retrieved by the module are identical
t.is(result.length, 2);
t.is(result[0].hash.substring(0, 7), commits[0].hash);
t.is(result[0].message, commits[0].message);
t.is(result[1].hash.substring(0, 7), commits[1].hash);
t.is(result[1].message, commits[1].message);
});
test.serial('Get all commits since gitHead (from tag) ', async t => {
// Create a git repository, set the current working directory at the root of the repo
await gitRepo();
// Add commits to the master branch
let commits = await gitCommits(['fix: First fix']);
// Create the tag corresponding to version 1.0.0
await gitTagVersion('1.0.0');
// Add new commits to the master branch
commits = (await gitCommits(['feat: Second feature', 'feat: Third feature'])).concat(commits);
// Retrieve the commits with the commits module
const result = await getCommits({lastRelease: {version: `1.0.0`}, options: {branch: 'master'}});
// Verify the commits created and retrieved by the module are identical
t.is(result.length, 2);
t.is(result[0].hash.substring(0, 7), commits[0].hash);
t.is(result[0].message, commits[0].message);
t.is(result[1].hash.substring(0, 7), commits[1].hash);
t.is(result[1].message, commits[1].message);
});
test.serial('Get all commits since gitHead (from tag formatted like v<version>) ', async t => {
// Create a git repository, set the current working directory at the root of the repo
await gitRepo();
// Add commits to the master branch
let commits = await gitCommits(['fix: First fix']);
// Create the tag corresponding to version 1.0.0
await gitTagVersion('v1.0.0');
// Add new commits to the master branch
commits = (await gitCommits(['feat: Second feature', 'feat: Third feature'])).concat(commits);
// Retrieve the commits with the commits module
const result = await getCommits({lastRelease: {version: `1.0.0`}, options: {branch: 'master'}});
// Verify the commits created and retrieved by the module are identical
t.is(result.length, 2);
t.is(result[0].hash.substring(0, 7), commits[0].hash);
t.is(result[0].message, commits[0].message);
t.is(result[1].hash.substring(0, 7), commits[1].hash);
t.is(result[1].message, commits[1].message);
});
test.serial('Get all commits since gitHead from tag, when tags are mising from the shallow clone', async t => {
// Create a git repository, set the current working directory at the root of the repo
const repo = await gitRepo();
// Add commits to the master branch
let commits = await gitCommits(['fix: First fix']);
// Create the tag corresponding to version 1.0.0
await gitTagVersion('v1.0.0');
// Add new commits to the master branch
commits = (await gitCommits(['feat: Second feature', 'feat: Third feature'])).concat(commits);
// Create a shallow clone with only 1 commit and no tags
await gitShallowClone(repo);
// Verify the shallow clone does not contains any tags
t.is((await gitTags()).length, 0);
// Retrieve the commits with the commits module
const result = await getCommits({lastRelease: {version: `1.0.0`}, options: {branch: 'master'}});
// Verify the commits created and retrieved by the module are identical
t.is(result.length, 2);
t.is(result[0].hash.substring(0, 7), commits[0].hash);
t.is(result[0].message, commits[0].message);
@ -81,6 +172,25 @@ test.serial('Return empty array if lastRelease.gitHead is the last commit', asyn
t.deepEqual(result, []);
});
test.serial('Throws ENOGITHEAD error if the gitHead of the last release cannot be found', async t => {
// Create a git repository, set the current working directory at the root of the repo
await gitRepo();
// Add commits to the master branch
await gitCommits(['fix: First fix', 'feat: Second feature']);
// Retrieve the commits with the commits module
const error = await t.throws(getCommits({lastRelease: {version: '1.0.0'}, options: {branch: 'master'}}));
// Verify error code and message
t.is(error.code, 'ENOGITHEAD');
t.true(error instanceof SemanticReleaseError);
// Verify the log function has been called with a message explaining the error
t.regex(
errorLog.firstCall.args[1],
/The commit the last release of this package was derived from cannot be determined from the release metadata not from the repository tags/
);
});
test.serial('Throws ENOTINHISTORY error if gitHead is not in history', async t => {
// Create a git repository, set the current working directory at the root of the repo
await gitRepo();
@ -93,7 +203,6 @@ test.serial('Throws ENOTINHISTORY error if gitHead is not in history', async t =
// Verify error code and message
t.is(error.code, 'ENOTINHISTORY');
t.true(error instanceof SemanticReleaseError);
// Verify the log function has been called with a message mentionning the branch
t.regex(errorLog.firstCall.args[1], /history of the "master" branch/);
// Verify the log function has been called with a message mentionning the missing gitHead
@ -106,11 +215,11 @@ test.serial('Throws ENOTINHISTORY error if gitHead is not in branch history but
// Add commits to the master branch
await gitCommits(['First', 'Second']);
// Create the new branch 'other-branch' from master
await gitCheckout('other-branch', true);
await gitCheckout('other-branch');
// Add commits to the 'other-branch' branch
const commitsBranch = await gitCommits(['Third', 'Fourth']);
// Create the new branch 'another-branch' from 'other-branch'
await gitCheckout('another-branch', true);
await gitCheckout('another-branch');
// Retrieve the commits with the commits module
const error = await t.throws(
@ -120,7 +229,6 @@ test.serial('Throws ENOTINHISTORY error if gitHead is not in branch history but
// Verify error code and message
t.is(error.code, 'ENOTINHISTORY');
t.true(error instanceof SemanticReleaseError);
// Verify the log function has been called with a message mentionning the branch
t.regex(errorLog.firstCall.args[1], /history of the "master" branch/);
// Verify the log function has been called with a message mentionning the missing gitHead

View File

@ -7,16 +7,18 @@ import pMapSeries from 'p-map-series';
* Commit message informations.
*
* @typedef {Object} Commit
* @property {string} branch The commit branch
* @property {string} hash The commit hash
* @property {string} message The commit message
* @property {string} branch The commit branch.
* @property {string} hash The commit hash.
* @property {string} message The commit message.
*/
/**
* Create a temporary git repository.
* Create a temporary git repository and change the current working directory to the repository root.
*
* @method gitCommits
* @param {Array<Commit>} commits the created commits.
*
* @return {string} The path of the repository.
*/
export async function gitRepo() {
const dir = tempy.directory();
@ -24,14 +26,16 @@ export async function gitRepo() {
process.chdir(dir);
await mkdir('git-templates');
await execa('git', ['init', '--template=./git-templates']);
await gitCheckout('master');
return dir;
}
/**
* Create commits on the current git repository.
*
* @method gitCommits
* @param {Array<String>} messages commit messages
* @returns {Array<Commit>} commits the created commits, in reverse order (to match `git log` order)
* @param {Array<string>} messages commit messages.
*
* @returns {Array<Commit>} commits the created commits, in reverse order (to match `git log` order).
*/
export async function gitCommits(messages) {
return (await pMapSeries(messages, async msg => {
@ -44,18 +48,62 @@ export async function gitCommits(messages) {
/**
* Checkout a branch on the current git repository.
*
* @param {String} branch Branch name
* @param {Boolean} create `true` to create the branche ans switch, `false` to only switch
* @param {string} branch Branch name.
* @param {boolean} create `true` to create the branche ans switch, `false` to only switch.
*/
export async function gitCheckout(branch, create) {
export async function gitCheckout(branch, create = true) {
await execa('git', ['checkout', create ? '-b' : null, branch]);
}
/**
* Get the sha of the head commit in the current git repository.
*
* @return {String} The sha of the head commit in the current git repository.
* @return {string} The sha of the head commit in the current git repository.
*/
export async function gitHead() {
return (await execa('git', ['rev-parse', 'HEAD'])).stdout;
}
/**
* Create a tag on the head commit in the current git repository.
*
* @param {string} tagName The tag name to create.
*/
export async function gitTagVersion(tagName) {
await execa('git', ['tag', tagName]);
}
/**
* @return {Array<string>} The list of tags from the current git repository.
*/
export async function gitTags() {
return (await execa('git', ['tag'])).stdout.split('\n').filter(tag => !!tag);
}
/**
* @return {Array<string>} The list of commit sha from the current git repository.
*/
export async function gitLog() {
return (await execa('git', ['log', '--format=format:%H'])).stdout.split('\n').filter(sha => !!sha);
}
/**
* Create a shallow clone of a git repository and change the current working directory to the cloned repository root.
* The shallow will contain a limited number of commit and no tags.
*
* @param {string} origin The path of the repository to clone.
* @param {number} [depth=1] The number of commit to clone.
* @return {string} The path of the cloned repository.
*/
export async function gitShallowClone(origin, depth = 1) {
const dir = tempy.directory();
process.chdir(dir);
await execa('git', ['clone', '--no-hardlinks', '--no-tags', '--depth', depth, `file://${origin}`, dir]);
return dir;
}
/**
* Pack heads and tags of the current git repository.
*/
export async function gitPackRefs() {
await execa('git', ['pack-refs', '--all']);
}

View File

@ -1,19 +1,25 @@
import test from 'ava';
import {writeJson, readJson} from 'fs-extra';
import {start, stop, uri} from './helpers/registry';
import {gitRepo, gitCommits, gitHead} from './helpers/git-utils';
import {gitRepo, gitCommits, gitHead, gitTagVersion, gitPackRefs} from './helpers/git-utils';
import execa from 'execa';
test.before(async t => {
// Start the local NPM registry
await start();
});
test.beforeEach(async t => {
// Save the current working diretory
t.context.cwd = process.cwd();
// Start the local NPM registry
await start();
});
test.afterEach.always(async t => {
// Restore the current working directory
process.chdir(t.context.cwd);
});
test.after.always(async t => {
// Stop the local NPM registry
await stop();
});
@ -138,3 +144,66 @@ test.serial('Release patch, minor and major versions', async t => {
t.is(releaseGitHead, await gitHead());
t.log(`+ released ${version} with gitHead ${releaseGitHead}`);
});
test.serial('Release versions from a packed git repository, using tags to determine last release gitHead', async t => {
// Environment variables used with cli
const env = {
CI: true,
npm_config_registry: uri,
GH_TOKEN: 'github_token',
NPM_OLD_TOKEN: 'aW50ZWdyYXRpb246c3VjaHNlY3VyZQ==',
NPM_EMAIL: 'integration@test.com',
};
// Create a git repository, set the current working directory at the root of the repo
t.log('Create git repository');
await gitRepo();
// Create package.json in repository root
await writeJson('./package.json', {
name: 'test-module-2',
version: '0.0.0-dev',
repository: {url: 'git+https://github.com/semantic-release/test-module-2'},
release: {verifyConditions: require.resolve('../src/lib/plugin-noop')},
});
/** Minor release **/
t.log('Commit a feature');
await gitCommits(['feat: Initial commit']);
t.log('$ git pack-refs --all');
await gitPackRefs();
t.log('$ semantic-release pre');
let {stdout, code} = await execa(require.resolve('../bin/semantic-release'), ['pre'], {env});
// Verify package.json has been updated
t.is((await readJson('./package.json')).version, '1.0.0');
t.log('$ npm publish');
({stdout, code} = await execa('npm', ['publish'], {env}));
// Verify output of npm publish
t.regex(stdout, /test-module-2@1.0.0/);
t.is(code, 0);
// Retrieve the published package from the registry and check version and gitHead
let version = (await execa('npm', ['show', 'test-module-2', 'version'], {env})).stdout;
t.is(version, '1.0.0');
t.log(`+ released ${version}`);
// Create a tag version so the tag can be used later to determine the commit associated with the version
await gitTagVersion('v1.0.0');
t.log('Create git tag v1.0.0');
/** Patch release **/
t.log('Commit a fix');
await gitCommits(['fix: bar']);
t.log('$ semantic-release pre');
({stdout, code} = await execa(require.resolve('../bin/semantic-release'), ['pre'], {env}));
// Verify package.json has been updated
t.is((await readJson('./package.json')).version, '1.0.1');
t.log('$ npm publish');
({stdout, code} = await execa('npm', ['publish'], {env}));
// Verify output of npm publish
t.regex(stdout, /test-module-2@1.0.1/);
t.is(code, 0);
// Retrieve the published package from the registry and check version and gitHead
version = (await execa('npm', ['show', 'test-module-2', 'version'], {env})).stdout;
t.is(version, '1.0.1');
t.log(`+ released ${version}`);
});