From 3ff9b59b82d1d038517a238c731505971bd4c688 Mon Sep 17 00:00:00 2001 From: Izak Lipnik Date: Mon, 23 Oct 2017 18:23:32 +0200 Subject: [PATCH] feat(deploy): deploy commands must execute in extra job only if other jobs succeeded (closes #230) --- src/api/config.ts | 18 ++- src/api/process-manager.ts | 49 +++++++- src/api/process.ts | 19 +-- src/api/utils.ts | 14 +++ tests/unit/070_job_processes.spec.ts | 2 +- tests/unit/080_parse_yml_config.spec.ts | 161 ++++++++++++++++++++++++ tests/unit_runner.ts | 2 +- 7 files changed, 239 insertions(+), 26 deletions(-) create mode 100644 tests/unit/080_parse_yml_config.spec.ts diff --git a/src/api/config.ts b/src/api/config.ts index 0a76a6e55..421cd5239 100644 --- a/src/api/config.ts +++ b/src/api/config.ts @@ -32,10 +32,10 @@ export enum CommandTypePriority { store_cache = 8, after_success = 9, after_failure = 10, - before_deploy = 11, - deploy = 12, - after_deploy = 13, - after_script = 14 + after_script = 11, + before_deploy = 12, + deploy = 13, + after_deploy = 14 } export enum JobStage { @@ -518,9 +518,15 @@ export function generateJobsAndEnv(repo: Repository, config: Config): JobsAndEnv }); if (deployCommands.length) { + const gitCommands = [ + { command: clone, type: CommandType.git }, + { command: fetch, type: CommandType.git }, + { command: checkout, type: CommandType.git } + ]; + data.push({ - commands: installCommands.concat(deployCommands), - env: globalEnv, + commands: gitCommands.concat(installCommands).concat(deployCommands), + env: globalEnv.concat('DEPLOY'), stage: JobStage.deploy, image: config.image }); diff --git a/src/api/process-manager.ts b/src/api/process-manager.ts index 696a80aa2..3a216ad6c 100644 --- a/src/api/process-manager.ts +++ b/src/api/process-manager.ts @@ -17,7 +17,7 @@ import { getRemoteParsedConfig, JobsAndEnv, CommandType } from './config'; import { killContainer } from './docker'; import { logger, LogMessageType } from './logger'; import { blue, yellow, green, cyan } from 'chalk'; -import { getConfig, getHttpJsonResponse, getBitBucketAccessToken } from './utils'; +import { getConfig, getHttpJsonResponse, getBitBucketAccessToken, prepareCommands } from './utils'; import { sendFailureStatus, sendPendingStatus, sendSuccessStatus } from './commit-status'; import { decrypt } from './security'; import { userId } from './socket'; @@ -38,7 +38,7 @@ export interface JobMessage { export interface JobProcess { build_id?: number; job_id?: number; - status?: 'queued' | 'running' | 'cancelled' | 'errored'; + status?: 'queued' | 'running' | 'cancelled' | 'errored' | 'success'; image_name?: string; log?: string[]; commands?: { command: string, type: CommandType }[]; @@ -63,6 +63,7 @@ const config: any = getConfig(); export let jobProcesses: Subject = new Subject(); export let jobEvents: BehaviorSubject = new BehaviorSubject({}); export let terminalEvents: Subject = new Subject(); +export let buildStatuses: Subject = new Subject(); export let buildSub: { [id: number]: Subscription } = {}; export let processes: JobProcess[] = []; @@ -93,6 +94,40 @@ function execJob(proc: JobProcess): Observable<{}> { processes.push(proc); } + const buildProcesses = processes.filter(p => p.build_id === proc.build_id); + if (proc.env.findIndex(e => e === 'DEPLOY') === -1 || buildProcesses.length === 1 + || !buildProcesses.filter(p => p.status != 'success' && p.job_id != proc.job_id).length) { + return startJobProcess(proc); + } + + return new Observable(observer => { + return buildStatuses + .filter(bs => bs.build_id === proc.build_id) + .subscribe(event => { + const testProcesses = processes.filter(p => { + return p.build_id === proc.build_id && p.job_id != proc.job_id; + }); + + if (testProcesses.length) { + if (proc.job_id != event.job_id) { + const notSucceded = testProcesses.filter(p => p.status != 'success'); + const queuedOrRunning = testProcesses.filter(p => { + return p.status === 'queued' || p.status === 'running'; + }); + if (!notSucceded.length) { + startJobProcess(proc).subscribe(() => observer.complete()); + } else if (!queuedOrRunning.length) { + stopJob(proc.job_id).then(() => observer.complete()); + } + } else { + observer.complete(); + } + } + }); + }); +} + +export function startJobProcess(proc: JobProcess): Observable<{}> { return new Observable(observer => { getRepositoryByBuildId(proc.build_id) .then(repository => { @@ -134,6 +169,7 @@ function execJob(proc: JobProcess): Observable<{}> { observer.complete(); } }, err => { + proc.status = 'errored'; const time = new Date(); const msg: LogMessageType = { message: `[error]: ${err}`, type: 'error', notify: false @@ -162,6 +198,12 @@ function execJob(proc: JobProcess): Observable<{}> { data: 'build failed', additionalData: time.getTime() }); + + buildStatuses.next({ + status: 'errored', + build_id: proc.build_id, + job_id: proc.job_id + }); }) .then(() => { jobEvents.next({ @@ -181,6 +223,7 @@ function execJob(proc: JobProcess): Observable<{}> { observer.complete(); }); }, () => { + proc.status = 'success'; const time = new Date(); dbJob.getLastRunId(proc.job_id) .then(runId => { @@ -195,6 +238,8 @@ function execJob(proc: JobProcess): Observable<{}> { }) .then(() => getBuildStatus(proc.build_id)) .then(status => { + buildStatuses.next({status: status, build_id: proc.build_id, job_id: proc.job_id}); + if (status === 'success') { return updateBuild({ id: proc.build_id, end_time: time }) .then(() => getLastRunId(proc.build_id)) diff --git a/src/api/process.ts b/src/api/process.ts index dc9cf7f99..f875b89ad 100644 --- a/src/api/process.ts +++ b/src/api/process.ts @@ -1,6 +1,6 @@ import * as docker from './docker'; import * as child_process from 'child_process'; -import { generateRandomId, getFilePath } from './utils'; +import { generateRandomId, getFilePath, prepareCommands } from './utils'; import { getRepositoryByBuildId } from './db/repository'; import { Observable } from 'rxjs'; import { green, red, bold, yellow, blue, cyan } from 'chalk'; @@ -26,18 +26,6 @@ export interface ProcessOutput { data: any; } -export function prepareCommands(proc: JobProcess, allowed: CommandType[]): any { - let commands = proc.commands.filter(command => allowed.findIndex(c => c === command.type) !== -1); - return commands.sort((a, b) => { - if (CommandTypePriority[a.type] > CommandTypePriority[b.type]) { - return 1; - } else if (CommandTypePriority[a.type] < CommandTypePriority[b.type]) { - return -1; - } - return proc.commands.indexOf(a) - proc.commands.indexOf(b); - }); -} - export function startBuildProcess( proc: JobProcess, variables: string[], @@ -58,9 +46,8 @@ export function startBuildProcess( const gitTypes = [CommandType.git]; const installTypes = [CommandType.before_install, CommandType.install]; const scriptTypes = [CommandType.before_script, CommandType.script, - CommandType.after_success, CommandType.after_failure]; - const deployTypes = [CommandType.before_deploy, CommandType.deploy, - CommandType.after_deploy, CommandType.after_script]; + CommandType.after_success, CommandType.after_failure, CommandType.after_script]; + const deployTypes = [CommandType.before_deploy, CommandType.deploy, CommandType.after_deploy]; const gitCommands = prepareCommands(proc, gitTypes); const installCommands = prepareCommands(proc, installTypes); const scriptCommands = prepareCommands(proc, scriptTypes); diff --git a/src/api/utils.ts b/src/api/utils.ts index 7e1e0f28c..9e8109d71 100644 --- a/src/api/utils.ts +++ b/src/api/utils.ts @@ -18,6 +18,8 @@ import * as temp from 'temp'; import { blue, yellow, magenta, cyan, bold, red } from 'chalk'; import * as nodeRsa from 'node-rsa'; import * as glob from 'glob'; +import { CommandType, CommandTypePriority } from './config'; +import { JobProcess } from './process-manager'; const defaultConfig = { url: null, @@ -245,3 +247,15 @@ export function getBitBucketAccessToken(clientCredentials: string): Promise }); }); } + +export function prepareCommands(proc: JobProcess, allowed: CommandType[]): any { + let commands = proc.commands.filter(command => allowed.findIndex(c => c === command.type) !== -1); + return commands.sort((a, b) => { + if (CommandTypePriority[a.type] > CommandTypePriority[b.type]) { + return 1; + } else if (CommandTypePriority[a.type] < CommandTypePriority[b.type]) { + return -1; + } + return proc.commands.indexOf(a) - proc.commands.indexOf(b); + }); +} diff --git a/tests/unit/070_job_processes.spec.ts b/tests/unit/070_job_processes.spec.ts index 86e4bc889..b1593691c 100644 --- a/tests/unit/070_job_processes.spec.ts +++ b/tests/unit/070_job_processes.spec.ts @@ -1,6 +1,6 @@ import * as chai from 'chai'; import * as chaiAsPromised from 'chai-as-promised'; -import { prepareCommands } from '../../src/api/process'; +import { prepareCommands } from '../../src/api/utils'; import { CommandType } from '../../src/api/config'; chai.use(chaiAsPromised); diff --git a/tests/unit/080_parse_yml_config.spec.ts b/tests/unit/080_parse_yml_config.spec.ts new file mode 100644 index 000000000..49a05e54b --- /dev/null +++ b/tests/unit/080_parse_yml_config.spec.ts @@ -0,0 +1,161 @@ +import * as chai from 'chai'; +import * as chaiAsPromised from 'chai-as-promised'; +import { generateJobsAndEnv, Repository, + Config, parseConfig, JobStage, CommandType } from '../../src/api/config'; + +chai.use(chaiAsPromised); +const expect = chai.expect; + +describe('Parsing YML Config', () => { + it('Should parse YML config', () => { + const yml = { + image: 'abstruse', + matrix: [ { env: 'NODE_VERSION=8' } ], + preinstall: + [ 'npm config set spin false', + 'npm config set progress false', + 'npm i' ], + before_deploy: + [ 'npm config set spin false', + 'echo before_deploy', + 'npm run-script test' ], + script: [ 'npm run-script test' ], + deploy: + [ 'npm config set spin false', + 'echo deploying', + 'npm run script-test' ], + install: [ 'nvm install $NODE_VERSION', 'npm install' ], + cache: [ 'node_modules' ] + }; + const parsed = parseConfig(yml); + + expect(parsed.image).to.equal('abstruse'); + expect(parsed.os).to.equal('linux'); + expect(parsed.stage).to.equal('test'); + expect(parsed.branches).to.equal(null); + expect(parsed.env).to.equal(null); + expect(parsed.install[0].type).to.equals('install'); + expect(parsed.install[1].command).to.equals('npm install'); + expect(parsed.deploy[0].command).to.equals('npm config set spin false'); + }); + + it(`Generate Build with midex order of deployment commands`, () => { + const repository: Repository = { + clone_url: 'https://github.com/Izak88/d3-bundle.git', + branch: 'master', + pr: null, + sha: 'be39d3cacf1f337877c1660696d21367af25983b', + access_token: null, + type: 'github', + file_tree: + [ '.abstruse.yml', + '.git', + '.gitignore', + '.npmignore', + 'API.md', + 'CHANGES.md', + 'ISSUE_TEMPLATE.md', + 'LICENSE', + 'README.md', + 'd3.sublime-project', + 'img', + 'index.js', + 'jenkinsfile', + 'package.json', + 'rollup.config.js', + 'rollup.node.js', + 'test' ] + }; + const config: Config = { + image: 'abstruse', + os: 'linux', + stage: JobStage.test, + cache: [ 'node_modules' ], + branches: null, + env: null, + before_install: [], + install: + [ { command: 'nvm install $NODE_VERSION', type: CommandType.install }, + { command: 'npm install', type: CommandType.install } ], + before_script: [], + script: [ { command: 'npm run-script test', type: CommandType.script } ], + before_cache: [], + after_success: [], + after_failure: [], + before_deploy: + [ { command: 'npm config set spin false', type: CommandType.before_deploy }, + { command: 'echo before_deploy', type: CommandType.before_deploy }, + { command: 'npm run-script test', type: CommandType.before_deploy } ], + deploy: + [ { command: 'npm config set spin false', type: CommandType.deploy }, + { command: 'echo deploying', type: CommandType.deploy }, + { command: 'npm run script-test', type: CommandType.deploy } ], + after_deploy: [], + after_script: [], + jobs: { include: [], exclude: [] }, + matrix: { include: [ { env: 'NODE_VERSION=8' } ], exclude: [], allow_failures: [] } + }; + + const commands = generateJobsAndEnv(repository, config); + + expect(commands.length).to.equal(2); + expect(commands[0].stage).to.equals('test'); + expect(commands[1].stage).to.equals('deploy'); + }); + + it(`Generate Build with no deployment commands`, () => { + const repository: Repository = { + clone_url: 'https://github.com/Izak88/d3-bundle.git', + branch: 'master', + pr: null, + sha: 'be39d3cacf1f337877c1660696d21367af25983b', + access_token: null, + type: 'github', + file_tree: + [ '.abstruse.yml', + '.git', + '.gitignore', + '.npmignore', + 'API.md', + 'CHANGES.md', + 'ISSUE_TEMPLATE.md', + 'LICENSE', + 'README.md', + 'd3.sublime-project', + 'img', + 'index.js', + 'jenkinsfile', + 'package.json', + 'rollup.config.js', + 'rollup.node.js', + 'test' ] + }; + const config: Config = { + image: 'abstruse', + os: 'linux', + stage: JobStage.test, + cache: [ 'node_modules' ], + branches: null, + env: null, + before_install: [], + install: + [ { command: 'nvm install $NODE_VERSION', type: CommandType.install }, + { command: 'npm install', type: CommandType.install } ], + before_script: [], + script: [ { command: 'npm run-script test', type: CommandType.script } ], + before_cache: [], + after_success: [], + after_failure: [], + before_deploy: [], + deploy: [], + after_deploy: [], + after_script: [], + jobs: { include: [], exclude: [] }, + matrix: { include: [ { env: 'NODE_VERSION=8' } ], exclude: [], allow_failures: [] } + }; + + const commands = generateJobsAndEnv(repository, config); + expect(commands.length).to.equal(1); + expect(commands[0].stage).to.equals('test'); + }); +}); diff --git a/tests/unit_runner.ts b/tests/unit_runner.ts index 89a2b8fe3..559f15715 100644 --- a/tests/unit_runner.ts +++ b/tests/unit_runner.ts @@ -10,7 +10,7 @@ const argv = minimist(process.argv.slice(2), { }); const specFiles = glob.sync(path.resolve(__dirname, './unit/**/*.spec.*')); -const mo = new Mocha({ timeout: 60000, reporter: 'spec' }); +const mo = new Mocha({ timeout: 180000, reporter: 'spec' }); Promise.resolve() .then((): any => {