Archive artifacts

Jenkinsfile

@@ -608,6 +608,7 @@ def targetWithoutNode(Map args = [:]) {
         }
       }
       withTools(k8s: installK8s, gcp: withGCP, nodejs: withNodejs) {
+        try{
         // make commands use -C <folder> while mage commands require the dir(folder)
         // let's support this scenario with the location variable.
         dir(isMage ? directory : '') {
@@ -621,6 +622,13 @@ def targetWithoutNode(Map args = [:]) {
               cmd(label: "${args.id?.trim() ? args.id : env.STAGE_NAME} - ${command}", script: "${command}")
           }
         }
+        } catch(err) {
+          // Upload the generated files ONLY if the step failed. This will avoid any overhead with Google Storage
+          error("Error '${err.toString()}'")
+        } finally {
+            archiveArtifacts(allowEmptyArchive: true, artifacts: "**/filebeat/build/*")
+        }
+      }
       }
       // Publish packages should happen always to easily consume those artifacts if the
       // e2e were triggered and failed.
@@ -633,7 +641,7 @@ def targetWithoutNode(Map args = [:]) {
       }
     }
   }
-}
+
 
 /**
 * This method wraps all the environment setup and pre-requirements to run any commands.

x-pack/filebeat/input/awss3/input_integration_test.go

@@ -143,6 +143,25 @@ visibility_timeout: 30s
 region: us-east-1
 endpoint: http://localhost:4566
 path_style: true
+file_selectors:
+-
+  regex: 'events-array.json$'
+  expand_event_list_from_field: Events
+  include_s3_metadata:
+    - last-modified
+    - x-amz-version-id
+    - x-amz-storage-class
+    - Content-Length
+    - Content-Type
+-
+  regex: '\.(?:nd)?json(\.gz)?$'
+-
+  regex: 'multiline.txt$'
+  parsers:
+    - multiline:
+        pattern: "^<Event"
+        negate:  true
+        match:   after
 `, queueURL))
 }
 
@@ -226,14 +245,8 @@ func TestInputRunSQSOnLocalstack(t *testing.T) {
 		cancel()
 	})
 
-	t.Logf("Terraform BucketName %s", tfConfig.BucketName)
-	t.Logf("Terraform QueueURL %s", tfConfig.QueueURL)
-
 	s3Input := createInput(t, makeTestConfigSQSForLocalstack(tfConfig.QueueURL))
 
-	t.Logf("s3Input name %s", s3Input.Name())
-	t.Logf("s3Input queue_url %s", s3Input.config.QueueURL)
-
 	// Run S3 Input with desired context
 	var errGroup errgroup.Group
 	errGroup.Go(func() error {
@@ -250,8 +263,8 @@ func TestInputRunSQSOnLocalstack(t *testing.T) {
 	assert.EqualValues(t, s3Input.metrics.sqsMessagesReturnedTotal.Get(), 1) // Invalid JSON is returned so that it can eventually be DLQed.
 	assert.EqualValues(t, s3Input.metrics.sqsVisibilityTimeoutExtensionsTotal.Get(), 0)
 	assert.EqualValues(t, s3Input.metrics.s3ObjectsInflight.Get(), 0)
-	assert.EqualValues(t, s3Input.metrics.s3ObjectsRequestedTotal.Get(), 8)
-	assert.EqualValues(t, s3Input.metrics.s3EventsCreatedTotal.Get(), uint64(0x13))
+	assert.EqualValues(t, s3Input.metrics.s3ObjectsRequestedTotal.Get(), 7)
+	assert.EqualValues(t, s3Input.metrics.s3EventsCreatedTotal.Get(), 12)
 	assert.Greater(t, s3Input.metrics.sqsLagTime.Mean(), 0.0)
 	assert.EqualValues(t, s3Input.metrics.sqsWorkerUtilization.Get(), 0.0) // Workers are reset after processing and hence utilization should be 0 at the end
 }