diff --git a/local/configs/jenkins.yaml b/local/configs/jenkins.yaml index 31a0a1b92..57fc065ea 100644 --- a/local/configs/jenkins.yaml +++ b/local/configs/jenkins.yaml @@ -131,6 +131,7 @@ jobs: - file: "/var/pipeline-library/src/test/resources/jobs/pipelineManager.dsl" - file: "/var/pipeline-library/src/test/resources/jobs/rebuildPipeline.dsl" - file: "/var/pipeline-library/src/test/resources/jobs/stashV2.dsl" + - file: "/var/pipeline-library/src/test/resources/jobs/tar.dsl" - file: "/var/pipeline-library/src/test/resources/jobs/timeout/downstream.dsl" - file: "/var/pipeline-library/src/test/resources/jobs/timeout/parentstream.dsl" - file: "/var/pipeline-library/src/test/resources/jobs/withGitRelease.dsl" diff --git a/src/test/groovy/ApmBasePipelineTest.groovy b/src/test/groovy/ApmBasePipelineTest.groovy index db1fa201a..8756bf715 100644 --- a/src/test/groovy/ApmBasePipelineTest.groovy +++ b/src/test/groovy/ApmBasePipelineTest.groovy @@ -415,6 +415,10 @@ class ApmBasePipelineTest extends DeclarativePipelineTest { return script.call(m, c) }) helper.registerAllowedMethod('sendDataToElasticsearch', [Map.class], { "OK" }) + helper.registerAllowedMethod('tar', [Map.class], { m -> + def script = loadScript('vars/tar.groovy') + return script.call(m) + }) helper.registerAllowedMethod('toJSON', [Map.class], { m -> def script = loadScript('vars/toJSON.groovy') return script.call(m) diff --git a/src/test/groovy/StashV2StepTests.groovy b/src/test/groovy/StashV2StepTests.groovy index 38e7d9483..9a0e88ad6 100644 --- a/src/test/groovy/StashV2StepTests.groovy +++ b/src/test/groovy/StashV2StepTests.groovy @@ -76,7 +76,6 @@ class StashV2StepTests extends ApmBasePipelineTest { script.call(name: 'source', bucket: 'foo', credentialsId: 'secret') printCallStack() assertTrue(assertMethodCallContainsPattern('log', 'stashV2: JOB_GCS_BUCKET is set. bucket param got precedency instead.')) - assertTrue(assertMethodCallContainsPattern('sh', 'tar --exclude=source.tgz -czf source.tgz .')) assertTrue(assertMethodCallContainsPattern('googleStorageUpload', 'bucket=gs://foo')) assertTrue(assertMethodCallContainsPattern('googleStorageUpload', 'credentialsId=secret')) assertTrue(assertMethodCallContainsPattern('googleStorageUpload', 'pattern=source.tgz')) @@ -91,7 +90,6 @@ class StashV2StepTests extends ApmBasePipelineTest { script.call(name: 'source', bucket: 'foo', credentialsId: 'my-super-credentials') printCallStack() assertTrue(assertMethodCallContainsPattern('log', 'stashV2: JOB_GCS_CREDENTIALS is set. credentialsId param got precedency instead.')) - assertTrue(assertMethodCallContainsPattern('sh', 'tar --exclude=source.tgz -czf source.tgz .')) assertTrue(assertMethodCallContainsPattern('googleStorageUpload', 'bucket=gs://foo')) assertTrue(assertMethodCallContainsPattern('googleStorageUpload', 'credentialsId=my-super-credentials')) assertTrue(assertMethodCallContainsPattern('googleStorageUpload', 'pattern=source.tgz')) @@ -106,8 +104,6 @@ class StashV2StepTests extends ApmBasePipelineTest { printCallStack() assertFalse(assertMethodCallContainsPattern('log', 'got precedency instead.')) assertNull(assertMethodCall('bat')) - assertTrue(assertMethodCallContainsPattern('writeFile', 'file=source.tgz')) - assertTrue(assertMethodCallContainsPattern('sh', 'tar --exclude=source.tgz -czf source.tgz .')) assertTrue(assertMethodCallContainsPattern('sh', 'rm source.tgz')) assertTrue(bucketUri.contains('source/source.tgz')) assertJobStatusSuccess() @@ -121,9 +117,6 @@ class StashV2StepTests extends ApmBasePipelineTest { printCallStack() assertFalse(assertMethodCallContainsPattern('log', 'got precedency instead.')) assertNull(assertMethodCall('sh')) - assertTrue(assertMethodCallContainsPattern('writeFile', 'file=source.tgz')) - assertTrue(assertMethodCallContainsPattern('withEnv', 'C:\\Windows\\System32')) - assertTrue(assertMethodCallContainsPattern('bat', 'tar --exclude=source.tgz -czf source.tgz .')) assertTrue(assertMethodCallContainsPattern('bat', 'del source.tgz')) assertJobStatusSuccess() } diff --git a/src/test/groovy/TarStepTests.groovy b/src/test/groovy/TarStepTests.groovy index fe84ad4ce..480753df8 100644 --- a/src/test/groovy/TarStepTests.groovy +++ b/src/test/groovy/TarStepTests.groovy @@ -30,38 +30,59 @@ class TarStepTests extends ApmBasePipelineTest { } @Test - void test() throws Exception { + void test_with_all_the_parameters() throws Exception { def script = loadScript(scriptName) - script.call(file:'archive.tgz', dir: 'folder', pathPrefix: 'folder', allowMissing: false, archive: true) + script.call(file:'archive.tgz', dir: 'folder', allowMissing: false, archive: true) printCallStack() + assertTrue(assertMethodCallContainsPattern('writeFile', 'file=archive.tgz')) + assertTrue(assertMethodCallContainsPattern('sh', 'tar --exclude=archive.tgz -czf archive.tgz folder')) + assertTrue(assertMethodCallOccurrences('bat', 0)) assertJobStatusSuccess() } @Test - void testError() throws Exception { + void test_with_an_error_and_without_allowMissing() throws Exception { def script = loadScript(scriptName) helper.registerAllowedMethod('sh', [Map.class], { throw new Exception("Error") }) - script.call(file:'archive.tgz', dir: 'folder', pathPrefix: 'folder', allowMissing: false, archive: true) + script.call(file:'archive.tgz', dir: 'folder', allowMissing: false, archive: true) printCallStack() assertJobStatusUnstable() } @Test - void testAllowMissing() throws Exception { + void test_with_allowMissing() throws Exception { def script = loadScript(scriptName) helper.registerAllowedMethod('sh', [String.class], { throw new Exception("Error") }) - script.call(file:'archive.tgz', dir: 'folder', pathPrefix: 'folder', allowMissing: true, archive: false) + script.call(file:'archive.tgz', dir: 'folder', allowMissing: true, archive: false) printCallStack() assertJobStatusSuccess() } @Test - void testIsNotUnix() throws Exception { + void test_with_an_error_without_failNever() throws Exception { + def script = loadScript(scriptName) + helper.registerAllowedMethod('sh', [Map.class], { throw new Exception('Error') }) + try { + script.call(file:'archive.tgz', dir: 'folder', failNever: false, archive: true) + } catch(err) { + //NOOP + println err + } + printCallStack() + assertTrue(assertMethodCallOccurrences('error', 1)) + assertJobStatusFailure() + } + + @Test + void test_windows() throws Exception { def script = loadScript(scriptName) helper.registerAllowedMethod("isUnix", [], {false}) - script.call(file:'archive.tgz', dir: 'folder', pathPrefix: 'folder', allowMissing: true) + script.call(file:'archive.tgz', dir: 'folder', allowMissing: true) printCallStack() - assertTrue(assertMethodCallContainsPattern('log', 'tar step is compatible only with unix systems')) + assertTrue(assertMethodCallContainsPattern('withEnv', 'C:\\Windows\\System32')) + assertTrue(assertMethodCallContainsPattern('writeFile', 'file=archive.tgz')) + assertTrue(assertMethodCallContainsPattern('bat', 'tar --exclude=archive.tgz -czf archive.tgz folder')) + assertTrue(assertMethodCallOccurrences('sh', 0)) assertJobStatusSuccess() } } diff --git a/src/test/resources/jobs/tar.dsl b/src/test/resources/jobs/tar.dsl new file mode 100644 index 000000000..29ac956e6 --- /dev/null +++ b/src/test/resources/jobs/tar.dsl @@ -0,0 +1,60 @@ +NAME = 'it/tar' +DSL = '''pipeline { + agent none + stages { + stage('linux') { + agent { label 'linux && immutable' } + steps { + dir('linux') { + writeFile(file: 'bar.txt', text: 'bar') + } + tar(file: 'linux.tgz', archive: true, dir: 'linux', allowMissing: true) + } + } + stage('linux_without_allowMissing') { + agent { label 'linux && immutable' } + steps { + script { + tar(file: 'linux.tgz', archive: true, dir: 'force_failure', allowMissing: false) + if (currentBuild.result.equals('UNSTABLE')) { + echo 'Assertion passed' + } else { + echo 'Expected to fail the tar step since force_failure folder does not exist' + error('Assertion failed') + } + } + } + } + stage('linux_without_failNever') { + agent { label 'linux && immutable' } + steps { + script { + try { + tar(file: 'linux.tgz', archive: true, dir: 'force_failure', failNever: false) + echo 'Expected to fail the tar step since force_failure folder does not exist' + error('Assertion failed') + } catch(e) { + echo 'Assertion passed' + } + } + } + } + stage('windows') { + agent { label 'windows-immutable' } + steps { + dir('windows') { + writeFile(file: 'foo.txt', text: 'foo') + } + tar(file: 'windows.tgz', archive: true, dir: 'windows', allowMissing: true) + } + } + } +}''' + +pipelineJob(NAME) { + definition { + cps { + script(DSL.stripIndent()) + } + } +} diff --git a/vars/README.md b/vars/README.md index 8a1ad827a..a12080434 100644 --- a/vars/README.md +++ b/vars/README.md @@ -1113,7 +1113,7 @@ nexusUploadStagingArtifact( * file_path: The location on local disk where the artifact to be uploaded can be found. ## nodeOS - Return the name of the Operating system based on the labels of the Node. + Return the name of the Operating system based on the labels of the Node [linux, windows, darwin]. ``` def os = nodeOS() @@ -1139,6 +1139,7 @@ notifyBuildResult(es: 'http://elastisearch.example.com:9200', secret: 'secret/te * shouldNotify: boolean value to decide to send or not the email notifications, by default it send emails on Failed builds that are not pull request. * prComment: Whether to add a comment in the PR with the build summary as a comment. Default: `true`. +* analyzeFlakey: Whether or not to add a comment in the PR with tests which have been detected as flakey. Default: `false`. * rebuild: Whether to rebuild the pipeline in case of any environmental issues. Default true * downstreamJobs: The map of downstream jobs that were launched within the upstream pipeline. Default empty. @@ -1252,7 +1253,7 @@ retryWithSleep(retries: 3, seconds: 5, backoff: true) { * retries: the number of retries. Mandatory * seconds: the seconds to wait for. Optional. Default 10. -* backoff: whether the wait period backs off backoffly after each retry. Optional. Default false +* backoff: whether the wait period backs off after each retry. Optional. Default false * sleepFirst: whether to sleep before running the command. Optional. Default false ## rubygemsLogin @@ -1428,16 +1429,14 @@ It requires [Google Cloud Storage plugin](https://plugins.jenkins.io/google-stor Compress a folder into a tar file. ``` -tar(file: 'archive.tgz', -archive: true, -dir: '.' -pathPrefix: '') +tar(file: 'archive.tgz', archive: true, dir: '.') ``` * *file*: Name of the tar file to create. * *archive*: If true the file will be archive in Jenkins (default true). * *dir*: The folder to compress (default .), it should not contain the compress file. -* *pathPrefix*: Path that contains the folder to compress, the step will make a "cd pathPrefix" before to compress the folder. +* *allowMissing*: whether to report UNSTABLE if tar command failed. Optional. Default 'true' +* *failNever*: Never fail the build, regardless of the step result. Optional. Default 'true' ## toJSON This step converts a JSON string to net.sf.json.JSON or and POJO to net.sf.json.JSON. @@ -1702,7 +1701,7 @@ withGithubNotify(context: 'Release', tab: 'artifacts') { } ``` -* version: Go version to install, if it is not set, it'll use GO_VERSION env var or '1.14.2' +* version: Go version to install, if it is not set, it'll use GO_VERSION env var or the default one set in the withGoEnv step * pkgs: Go packages to install with Go get before to execute any command. ## withNpmrc @@ -1793,3 +1792,4 @@ writeVaultSecret(secret: 'secret/apm-team/ci/temp/github-comment', data: ['secre * secret: Name of the secret on the the vault root path. Mandatory * data: What's the data to be written. Mandatory + diff --git a/vars/stashV2.groovy b/vars/stashV2.groovy index 01066c093..45cacdd40 100644 --- a/vars/stashV2.groovy +++ b/vars/stashV2.groovy @@ -76,15 +76,7 @@ def call(Map params = [:]) { } def compress(String filename) { - writeFile(file: "${filename}", text: '') - def command = "tar --exclude=${filename} -czf ${filename} ." - if(isUnix()) { - sh(label: 'Compress', script: command) - } else { - withEnv(["PATH+SYSTEM=C:\\Windows\\System32"]) { - bat(label: 'Compress', script: command) - } - } + tar(file: filename, dir: '.', archive: false, failNever: false) } def cleanup(String filename) { diff --git a/vars/tar.groovy b/vars/tar.groovy index 8de96d0c2..ec46dfa14 100644 --- a/vars/tar.groovy +++ b/vars/tar.groovy @@ -18,36 +18,45 @@ /** Compress a folder into a tar file. - tar(file: 'archive.tgz', - archive: true, - dir: '.' - pathPrefix: '', - allowMissing: true) + tar(file: 'archive.tgz', dir: '.', archive: true, allowMissing: true) */ -def call(Map params = [:]) { - def file = params.containsKey('file') ? params.file : 'archive.tgz' - def archive = params.containsKey('archive') ? params.archive : true - def dir = params.containsKey('dir') ? params.dir : "." - def pathPrefix = params.containsKey('pathPrefix') ? "cd '" + params.pathPrefix + "' && " : "" - def allowMissing = params.containsKey('allowMissing') ? params.allowMissing : true +def call(Map args = [:]) { + def file = args.get('file', 'archive.tgz') + def archive = args.get('archive', true) + def dir = args.get('dir', '.') + def allowMissing = args.get('allowMissing', true) + def failNever = args.get('failNever', true) - if(!isUnix()){ - log(level: 'INFO', text: "tar step is compatible only with unix systems") - return + // NOTE: pathPrefix is not required anymore since tar --exclude has been enabled + if (args.pathPrefix?.trim()) { + log(level: 'WARN', text: 'tar: pathPrefix parameter is deprecated.') } + try { - sh label: 'Generating tar file', script: "${pathPrefix} tar -czf '${WORKSPACE}/${file}' '${dir}'" + compress(file: file, dir: dir) if(archive){ - archiveArtifacts(allowEmptyArchive: true, - artifacts: file, - onlyIfSuccessful: false) + archiveArtifacts(allowEmptyArchive: true, artifacts: file, onlyIfSuccessful: false) } } catch (e){ - log(level: 'INFO', text: "${file} was not compresesd or archived : ${e?.message}") - if(!allowMissing){ - currentBuild.result = "UNSTABLE" + log(level: 'INFO', text: "${file} was not compressed or archived : ${e?.message}") + if (failNever) { + currentBuild.result = allowMissing ? 'SUCCESS' : 'UNSTABLE' } else { - currentBuild.result = "SUCCESS" + error("tar: step failled with error ${e?.message}") + } + } +} + +def compress(Map args = [:]) { + writeFile(file: "${args.file}", text: '') + def command = "tar --exclude=${args.file} -czf ${args.file} ${args.dir}" + if(isUnix()) { + sh(label: 'Compress', script: command) + } else { + // Some CI Windows workers got the tar binary in the system32 + // As long as those are not defined in the PATH let's use this hack + withEnv(["PATH+SYSTEM=C:\\Windows\\System32"]) { + bat(label: 'Compress', script: command) } } } diff --git a/vars/tar.txt b/vars/tar.txt index 300d1bb9e..b658cb835 100644 --- a/vars/tar.txt +++ b/vars/tar.txt @@ -1,13 +1,11 @@ Compress a folder into a tar file. ``` -tar(file: 'archive.tgz', -archive: true, -dir: '.' -pathPrefix: '') +tar(file: 'archive.tgz', archive: true, dir: '.') ``` * *file*: Name of the tar file to create. * *archive*: If true the file will be archive in Jenkins (default true). * *dir*: The folder to compress (default .), it should not contain the compress file. -* *pathPrefix*: Path that contains the folder to compress, the step will make a "cd pathPrefix" before to compress the folder. +* *allowMissing*: whether to report UNSTABLE if tar command failed. Optional. Default 'true' +* *failNever*: Never fail the build, regardless of the step result. Optional. Default 'true'