tar step supports windows (#612)

local/configs/jenkins.yaml

@@ -131,6 +131,7 @@ jobs:
   - file: "/var/pipeline-library/src/test/resources/jobs/pipelineManager.dsl"
   - file: "/var/pipeline-library/src/test/resources/jobs/rebuildPipeline.dsl"
   - file: "/var/pipeline-library/src/test/resources/jobs/stashV2.dsl"
+  - file: "/var/pipeline-library/src/test/resources/jobs/tar.dsl"
   - file: "/var/pipeline-library/src/test/resources/jobs/timeout/downstream.dsl"
   - file: "/var/pipeline-library/src/test/resources/jobs/timeout/parentstream.dsl"
   - file: "/var/pipeline-library/src/test/resources/jobs/withGitRelease.dsl"

src/test/groovy/ApmBasePipelineTest.groovy

@@ -415,6 +415,10 @@ class ApmBasePipelineTest extends DeclarativePipelineTest {
       return script.call(m, c)
     })
     helper.registerAllowedMethod('sendDataToElasticsearch', [Map.class], { "OK" })
+    helper.registerAllowedMethod('tar', [Map.class], { m ->
+      def script = loadScript('vars/tar.groovy')
+      return script.call(m)
+    })
     helper.registerAllowedMethod('toJSON', [Map.class], { m ->
       def script = loadScript('vars/toJSON.groovy')
       return script.call(m)

src/test/groovy/StashV2StepTests.groovy

@@ -76,7 +76,6 @@ class StashV2StepTests extends ApmBasePipelineTest {
     script.call(name: 'source', bucket: 'foo', credentialsId: 'secret')
     printCallStack()
     assertTrue(assertMethodCallContainsPattern('log', 'stashV2: JOB_GCS_BUCKET is set. bucket param got precedency instead.'))
-    assertTrue(assertMethodCallContainsPattern('sh', 'tar --exclude=source.tgz -czf source.tgz .'))
     assertTrue(assertMethodCallContainsPattern('googleStorageUpload', 'bucket=gs://foo'))
     assertTrue(assertMethodCallContainsPattern('googleStorageUpload', 'credentialsId=secret'))
     assertTrue(assertMethodCallContainsPattern('googleStorageUpload', 'pattern=source.tgz'))
@@ -91,7 +90,6 @@ class StashV2StepTests extends ApmBasePipelineTest {
     script.call(name: 'source', bucket: 'foo', credentialsId: 'my-super-credentials')
     printCallStack()
     assertTrue(assertMethodCallContainsPattern('log', 'stashV2: JOB_GCS_CREDENTIALS is set. credentialsId param got precedency instead.'))
-    assertTrue(assertMethodCallContainsPattern('sh', 'tar --exclude=source.tgz -czf source.tgz .'))
     assertTrue(assertMethodCallContainsPattern('googleStorageUpload', 'bucket=gs://foo'))
     assertTrue(assertMethodCallContainsPattern('googleStorageUpload', 'credentialsId=my-super-credentials'))
     assertTrue(assertMethodCallContainsPattern('googleStorageUpload', 'pattern=source.tgz'))
@@ -106,8 +104,6 @@ class StashV2StepTests extends ApmBasePipelineTest {
     printCallStack()
     assertFalse(assertMethodCallContainsPattern('log', 'got precedency instead.'))
     assertNull(assertMethodCall('bat'))
-    assertTrue(assertMethodCallContainsPattern('writeFile', 'file=source.tgz'))
-    assertTrue(assertMethodCallContainsPattern('sh', 'tar --exclude=source.tgz -czf source.tgz .'))
     assertTrue(assertMethodCallContainsPattern('sh', 'rm source.tgz'))
     assertTrue(bucketUri.contains('source/source.tgz'))
     assertJobStatusSuccess()
@@ -121,9 +117,6 @@ class StashV2StepTests extends ApmBasePipelineTest {
     printCallStack()
     assertFalse(assertMethodCallContainsPattern('log', 'got precedency instead.'))
     assertNull(assertMethodCall('sh'))
-    assertTrue(assertMethodCallContainsPattern('writeFile', 'file=source.tgz'))
-    assertTrue(assertMethodCallContainsPattern('withEnv', 'C:\\Windows\\System32'))
-    assertTrue(assertMethodCallContainsPattern('bat', 'tar --exclude=source.tgz -czf source.tgz .'))
     assertTrue(assertMethodCallContainsPattern('bat', 'del source.tgz'))
     assertJobStatusSuccess()
   }

src/test/groovy/TarStepTests.groovy

@@ -30,38 +30,59 @@ class TarStepTests extends ApmBasePipelineTest {
   }
 
   @Test
-  void test() throws Exception {
+  void test_with_all_the_parameters() throws Exception {
     def script = loadScript(scriptName)
-    script.call(file:'archive.tgz', dir: 'folder', pathPrefix: 'folder', allowMissing: false, archive: true)
+    script.call(file:'archive.tgz', dir: 'folder', allowMissing: false, archive: true)
     printCallStack()
+    assertTrue(assertMethodCallContainsPattern('writeFile', 'file=archive.tgz'))
+    assertTrue(assertMethodCallContainsPattern('sh', 'tar --exclude=archive.tgz -czf archive.tgz folder'))
+    assertTrue(assertMethodCallOccurrences('bat', 0))
     assertJobStatusSuccess()
   }
 
   @Test
-  void testError() throws Exception {
+  void test_with_an_error_and_without_allowMissing() throws Exception {
     def script = loadScript(scriptName)
     helper.registerAllowedMethod('sh', [Map.class], { throw new Exception("Error") })
-    script.call(file:'archive.tgz', dir: 'folder', pathPrefix: 'folder', allowMissing: false, archive: true)
+    script.call(file:'archive.tgz', dir: 'folder', allowMissing: false, archive: true)
     printCallStack()
     assertJobStatusUnstable()
   }
 
   @Test
-  void testAllowMissing() throws Exception {
+  void test_with_allowMissing() throws Exception {
     def script = loadScript(scriptName)
     helper.registerAllowedMethod('sh', [String.class], { throw new Exception("Error") })
-    script.call(file:'archive.tgz', dir: 'folder', pathPrefix: 'folder', allowMissing: true, archive: false)
+    script.call(file:'archive.tgz', dir: 'folder', allowMissing: true, archive: false)
     printCallStack()
     assertJobStatusSuccess()
   }
 
   @Test
-  void testIsNotUnix() throws Exception {
+  void test_with_an_error_without_failNever() throws Exception {
+    def script = loadScript(scriptName)
+    helper.registerAllowedMethod('sh', [Map.class], { throw new Exception('Error') })
+    try {
+      script.call(file:'archive.tgz', dir: 'folder', failNever: false, archive: true)
+    } catch(err) {
+      //NOOP
+      println err
+    }
+    printCallStack()
+    assertTrue(assertMethodCallOccurrences('error', 1))
+    assertJobStatusFailure()
+  }
+
+  @Test
+  void test_windows() throws Exception {
     def script = loadScript(scriptName)
     helper.registerAllowedMethod("isUnix", [], {false})
-    script.call(file:'archive.tgz', dir: 'folder', pathPrefix: 'folder', allowMissing: true)
+    script.call(file:'archive.tgz', dir: 'folder', allowMissing: true)
     printCallStack()
-    assertTrue(assertMethodCallContainsPattern('log', 'tar step is compatible only with unix systems'))
+    assertTrue(assertMethodCallContainsPattern('withEnv', 'C:\\Windows\\System32'))
+    assertTrue(assertMethodCallContainsPattern('writeFile', 'file=archive.tgz'))
+    assertTrue(assertMethodCallContainsPattern('bat', 'tar --exclude=archive.tgz -czf archive.tgz folder'))
+    assertTrue(assertMethodCallOccurrences('sh', 0))
     assertJobStatusSuccess()
   }
 }

src/test/resources/jobs/tar.dsl

@@ -0,0 +1,60 @@
+NAME = 'it/tar'
+DSL = '''pipeline {
+  agent none
+  stages {
+    stage('linux') {
+      agent { label 'linux && immutable' }
+      steps {
+        dir('linux') {
+          writeFile(file: 'bar.txt', text: 'bar')
+        }
+        tar(file: 'linux.tgz', archive: true, dir: 'linux', allowMissing: true)
+      }
+    }
+    stage('linux_without_allowMissing') {
+      agent { label 'linux && immutable' }
+      steps {
+        script {
+          tar(file: 'linux.tgz', archive: true, dir: 'force_failure', allowMissing: false)
+          if (currentBuild.result.equals('UNSTABLE')) {
+            echo 'Assertion passed'
+          } else {
+            echo 'Expected to fail the tar step since force_failure folder does not exist'
+            error('Assertion failed')
+          }
+        }
+      }
+    }
+    stage('linux_without_failNever') {
+      agent { label 'linux && immutable' }
+      steps {
+        script {
+          try {
+            tar(file: 'linux.tgz', archive: true, dir: 'force_failure', failNever: false)
+            echo 'Expected to fail the tar step since force_failure folder does not exist'
+            error('Assertion failed')
+          } catch(e) {
+            echo 'Assertion passed'
+          }
+        }
+      }
+    }
+    stage('windows') {
+      agent { label 'windows-immutable' }
+      steps {
+        dir('windows') {
+          writeFile(file: 'foo.txt', text: 'foo')
+        }
+        tar(file: 'windows.tgz', archive: true, dir: 'windows', allowMissing: true)
+      }
+    }
+  }
+}'''
+
+pipelineJob(NAME) {
+  definition {
+    cps {
+      script(DSL.stripIndent())
+    }
+  }
+}

vars/README.md

@@ -1113,7 +1113,7 @@ nexusUploadStagingArtifact(
   * file_path: The location on local disk where the artifact to be uploaded can be found.
 
 ## nodeOS
- Return the name of the Operating system based on the labels of the Node.
+ Return the name of the Operating system based on the labels of the Node [linux, windows, darwin].
 
 ```
  def os = nodeOS()
@@ -1139,6 +1139,7 @@ notifyBuildResult(es: 'http://elastisearch.example.com:9200', secret: 'secret/te
 * shouldNotify: boolean value to decide to send or not the email notifications, by default it send
 emails on Failed builds that are not pull request.
 * prComment: Whether to add a comment in the PR with the build summary as a comment. Default: `true`.
+* analyzeFlakey: Whether or not to add a comment in the PR with tests which have been detected as flakey. Default: `false`.
 * rebuild: Whether to rebuild the pipeline in case of any environmental issues. Default true
 * downstreamJobs: The map of downstream jobs that were launched within the upstream pipeline. Default empty.
 
@@ -1252,7 +1253,7 @@ retryWithSleep(retries: 3, seconds: 5, backoff: true) {
 
 * retries: the number of retries. Mandatory
 * seconds: the seconds to wait for. Optional. Default 10.
-* backoff: whether the wait period backs off backoffly after each retry. Optional. Default false
+* backoff: whether the wait period backs off after each retry. Optional. Default false
 * sleepFirst: whether to sleep before running the command. Optional. Default false
 
 ## rubygemsLogin
@@ -1428,16 +1429,14 @@ It requires [Google Cloud Storage plugin](https://plugins.jenkins.io/google-stor
 Compress a folder into a tar file.
 
 ```
-tar(file: 'archive.tgz',
-archive: true,
-dir: '.'
-pathPrefix: '')
+tar(file: 'archive.tgz', archive: true, dir: '.')
 ```
 
 * *file*: Name of the tar file to create.
 * *archive*: If true the file will be archive in Jenkins (default true).
 * *dir*: The folder to compress (default .), it should not contain the compress file.
-* *pathPrefix*: Path that contains the folder to compress, the step will make a "cd pathPrefix" before to compress the folder.
+* *allowMissing*: whether to report UNSTABLE if tar command failed. Optional. Default 'true'
+* *failNever*: Never fail the build, regardless of the step result. Optional. Default 'true'
 
 ## toJSON
 This step converts a JSON string to net.sf.json.JSON or and POJO to net.sf.json.JSON.
@@ -1702,7 +1701,7 @@ withGithubNotify(context: 'Release', tab: 'artifacts') {
   }
 ```
 
-* version: Go version to install, if it is not set, it'll use GO_VERSION env var or '1.14.2'
+* version: Go version to install, if it is not set, it'll use GO_VERSION env var or the default one set in the withGoEnv step
 * pkgs: Go packages to install with Go get before to execute any command.
 
 ## withNpmrc
@@ -1793,3 +1792,4 @@ writeVaultSecret(secret: 'secret/apm-team/ci/temp/github-comment', data: ['secre
 
 * secret: Name of the secret on the the vault root path. Mandatory
 * data: What's the data to be written. Mandatory
+

vars/stashV2.groovy

@@ -76,15 +76,7 @@ def call(Map params = [:]) {
 }
 
 def compress(String filename) {
-  writeFile(file: "${filename}", text: '')
-  def command = "tar --exclude=${filename} -czf ${filename} ."
-  if(isUnix()) {
-    sh(label: 'Compress', script: command)
-  } else {
-    withEnv(["PATH+SYSTEM=C:\\Windows\\System32"]) {
-      bat(label: 'Compress', script: command)
-    }
-  }
+  tar(file: filename, dir: '.', archive: false, failNever: false)
 }
 
 def cleanup(String filename) {

vars/tar.groovy

@@ -18,36 +18,45 @@
 /**
  Compress a folder into a tar file.
 
- tar(file: 'archive.tgz',
-  archive: true,
-  dir: '.'
-  pathPrefix: '',
-  allowMissing: true)
+ tar(file: 'archive.tgz', dir: '.', archive: true, allowMissing: true)
 */
-def call(Map params = [:]) {
-  def file = params.containsKey('file') ? params.file : 'archive.tgz'
-  def archive = params.containsKey('archive') ? params.archive : true
-  def dir = params.containsKey('dir') ? params.dir : "."
-  def pathPrefix = params.containsKey('pathPrefix') ? "cd '" + params.pathPrefix + "' && " : ""
-  def allowMissing = params.containsKey('allowMissing') ? params.allowMissing : true
+def call(Map args = [:]) {
+  def file = args.get('file', 'archive.tgz')
+  def archive = args.get('archive', true)
+  def dir = args.get('dir', '.')
+  def allowMissing = args.get('allowMissing', true)
+  def failNever = args.get('failNever', true)
 
-  if(!isUnix()){
-    log(level: 'INFO', text: "tar step is compatible only with unix systems")
-    return
+  // NOTE: pathPrefix is not required anymore since tar --exclude has been enabled 
+  if (args.pathPrefix?.trim()) {
+    log(level: 'WARN', text: 'tar: pathPrefix parameter is deprecated.')
   }
+
   try {
-    sh label: 'Generating tar file', script: "${pathPrefix} tar -czf '${WORKSPACE}/${file}' '${dir}'"
+    compress(file: file, dir: dir)
     if(archive){
-      archiveArtifacts(allowEmptyArchive: true,
-                      artifacts: file,
-                      onlyIfSuccessful: false)
+      archiveArtifacts(allowEmptyArchive: true, artifacts: file, onlyIfSuccessful: false)
     }
   } catch (e){
-    log(level: 'INFO', text: "${file} was not compresesd or archived : ${e?.message}")
-    if(!allowMissing){
-      currentBuild.result = "UNSTABLE"
+    log(level: 'INFO', text: "${file} was not compressed or archived : ${e?.message}")
+    if (failNever) {
+      currentBuild.result = allowMissing ? 'SUCCESS' : 'UNSTABLE'
     } else {
-      currentBuild.result = "SUCCESS"
+      error("tar: step failled with error ${e?.message}")
+    }
+  }
+}
+
+def compress(Map args = [:]) {
+  writeFile(file: "${args.file}", text: '')
+  def command = "tar --exclude=${args.file} -czf ${args.file} ${args.dir}"
+  if(isUnix()) {
+    sh(label: 'Compress', script: command)
+  } else {
+    // Some CI Windows workers got the tar binary in the system32
+    // As long as those are not defined in the PATH let's use this hack
+    withEnv(["PATH+SYSTEM=C:\\Windows\\System32"]) {
+      bat(label: 'Compress', script: command)
     }
   }
 }

vars/tar.txt

@@ -1,13 +1,11 @@
 Compress a folder into a tar file.
 
 ```
-tar(file: 'archive.tgz',
-archive: true,
-dir: '.'
-pathPrefix: '')
+tar(file: 'archive.tgz', archive: true, dir: '.')
 ```
 
 * *file*: Name of the tar file to create.
 * *archive*: If true the file will be archive in Jenkins (default true).
 * *dir*: The folder to compress (default .), it should not contain the compress file.
-* *pathPrefix*: Path that contains the folder to compress, the step will make a "cd pathPrefix" before to compress the folder.
+* *allowMissing*: whether to report UNSTABLE if tar command failed. Optional. Default 'true'
+* *failNever*: Never fail the build, regardless of the step result. Optional. Default 'true'