Issue#896 Workflow steps with non-existant output artifact path will …

…succeed (#1277)

* Issue#896 Workflow steps with non-existant output artifact path will succeed

Issue: #897
Solution: Added new element "optional" in Artifact. The default is false.  This flag will make artifact as optional and existence check will be ignored if input/output artifact has optional=true.

Output Artifact ( optional=true ):
Artifact existence check will be ignored during the save artifact in destination and continued workflow

Input Artifact ( optional=true ):
Artifact exist check will be ignored during load artifact from source and continued workflow

* added end of line

* removed unwanted whitespace

* Deleted test code

* go formatted

* added formatting directives

* updated Codegen

* Fixed format on merge conflict

* format fix

* updated comments

* improved error case

api/openapi-spec/swagger.json

@@ -83,6 +83,10 @@
           "description": "name of the artifact. must be unique within a template's inputs/outputs.",
           "type": "string"
         },
+        "optional": {
+          "description": "Make Artifacts optional, if Artifacts doesn't generate or exist",
+          "type": "boolean"
+        },
         "path": {
           "description": "Path is the container path to the artifact",
           "type": "string"

errors/errors.go

@@ -14,7 +14,7 @@ const (
 	CodeBadRequest     = "ERR_BAD_REQUEST"
 	CodeForbidden      = "ERR_FORBIDDEN"
 	CodeNotFound       = "ERR_NOT_FOUND"
-	CodeNotImplemented = "ERR_NOT_INPLEMENTED"
+	CodeNotImplemented = "ERR_NOT_IMPLEMENTED"
 	CodeTimeout        = "ERR_TIMEOUT"
 	CodeInternal       = "ERR_INTERNAL"
 )

pkg/apis/workflow/v1alpha1/types.go

@@ -319,6 +319,9 @@ type Artifact struct {
 
 	// Archive controls how the artifact will be saved to the artifact repository.
 	Archive *ArchiveStrategy `json:"archive,omitempty"`
+
+	// Make Artifacts optional, if Artifacts doesn't generate or exist
+	Optional bool `json:"optional,omitempty"`
 }
 
 // ArchiveStrategy describes how to archive files/directory when saving artifacts

test/e2e/expectedfailures/input-artifact-not-optional.yaml

@@ -0,0 +1,22 @@
+# This example demonstrates the input artifacts not optionals
+# from one step to the next.
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+  generateName: input-artifact-not-optional-
+spec:
+  entrypoint: http-artifact-example
+  templates:
+    - name: http-artifact-example
+      inputs:
+        artifacts:
+          - name: kubectl
+            path: /bin/kubectl
+            mode: 0755
+            optional: false
+            http:
+              url: ""
+      container:
+        image: debian:9.4
+        command: [sh, -c]
+        args: ["echo NoKubectl"]

test/e2e/expectedfailures/output-artifact-not-optional.yaml

@@ -0,0 +1,24 @@
+# This example demonstrates the output artifacts not optionals
+# from one step to the next.
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+  generateName: output-artifact-not-optional-
+spec:
+  entrypoint: artifact-example
+  templates:
+    - name: artifact-example
+      steps:
+        - - name: generate-artifact
+            template: whalesay
+
+    - name: whalesay
+      container:
+        image: docker/whalesay:latest
+        command: [sh, -c]
+        args: ["cowsay hello world | tee /tmp/hello_world12.txt"]
+      outputs:
+        artifacts:
+          - name: hello-art
+            optional: false
+            path: /tmp/hello_world.txt

test/e2e/functional/input-artifact-optional.yaml

@@ -0,0 +1,22 @@
+# This example demonstrates the input artifacts optionals
+# from one step to the next.
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+  generateName: input-artifact-optional-
+spec:
+  entrypoint: http-artifact-example
+  templates:
+    - name: http-artifact-example
+      inputs:
+        artifacts:
+          - name: kubectl
+            path: /bin/kubectl
+            mode: 0755
+            optional: true
+            http:
+              url: ""
+      container:
+        image: debian:9.4
+        command: [sh, -c]
+        args: ["echo NoKubectl"]

test/e2e/functional/output-artifact-optional.yaml

@@ -0,0 +1,24 @@
+# This example demonstrates the output artifacts optionals
+# from one step to the next.
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+  generateName: output-artifact-optional-
+spec:
+  entrypoint: artifact-example
+  templates:
+    - name: artifact-example
+      steps:
+        - - name: generate-artifact
+            template: whalesay
+
+    - name: whalesay
+      container:
+        image: docker/whalesay:latest
+        command: [sh, -c]
+        args: ["cowsay hello world | tee /tmp/hello_world12.txt"]
+      outputs:
+        artifacts:
+          - name: hello-art
+            optional: true
+            path: /tmp/hello_world.txt

test/e2e/functional/output-input-artifact-optional.yaml

@@ -0,0 +1,40 @@
+# This example demonstrates the output and input artifacts are optionals
+# from one step to the next.
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+  generateName: output-input-artifact-optional-
+spec:
+  entrypoint: artifact-example
+  templates:
+    - name: artifact-example
+      steps:
+        - - name: generate-artifact
+            template: whalesay
+        - - name: consume-artifact
+            template: print-message
+            arguments:
+              artifacts:
+                - name: message
+                  from: "{{steps.generate-artifact.outputs.artifacts.hello-art}}"
+    - name: whalesay
+      container:
+        image: docker/whalesay:latest
+        command: [sh, -c]
+        args: ["cowsay hello world | tee /tmp/hello_world123.txt"]
+      outputs:
+        artifacts:
+          - name: hello-art
+            optional: true
+            path: /tmp/hello_world.txt
+
+    - name: print-message
+      inputs:
+        artifacts:
+          - name: message
+            path: /tmp/message
+            optional: true
+      container:
+        image: alpine:latest
+        command: [sh, -c]
+        args: ["echo /tmp/message"]

workflow/common/util.go

@@ -148,10 +148,10 @@ func ProcessArgs(tmpl *wfv1.Template, args wfv1.Arguments, globalParams, localPa
 		}
 		// artifact must be supplied
 		argArt := args.GetArtifactByName(inArt.Name)
-		if argArt == nil {
+		if !inArt.Optional && argArt == nil {
 			return nil, errors.Errorf(errors.CodeBadRequest, "inputs.artifacts.%s was not supplied", inArt.Name)
 		}
-		if !argArt.HasLocation() && !validateOnly {
+		if !inArt.Optional && !argArt.HasLocation() && !validateOnly {
 			return nil, errors.Errorf(errors.CodeBadRequest, "inputs.artifacts.%s missing location information", inArt.Name)
 		}
 		argArt.Path = inArt.Path

workflow/executor/docker/docker.go

@@ -65,7 +65,9 @@ func (d *DockerExecutor) CopyFile(containerID string, sourcePath string, destPat
 		return err
 	}
 	if !file.ExistsInTar(sourcePath, tar.NewReader(gzipReader)) {
-		return errors.InternalErrorf("path %s does not exist (or %s is empty) in archive %s", sourcePath, sourcePath, destPath)
+		errMsg := fmt.Sprintf("path %s does not exist (or %s is empty) in archive %s", sourcePath, sourcePath, destPath)
+		log.Warn(errMsg)
+		return errors.Errorf(errors.CodeNotFound, errMsg)
 	}
 	log.Infof("Archiving completed")
 	return nil

workflow/executor/executor.go

@@ -114,7 +114,17 @@ func (we *WorkflowExecutor) LoadArtifacts() error {
 	log.Infof("Start loading input artifacts...")
 
 	for _, art := range we.Template.Inputs.Artifacts {
+
 		log.Infof("Downloading artifact: %s", art.Name)
+
+		if !art.HasLocation() {
+			if art.Optional {
+				log.Warnf("Artifact %s is not supplied. Artifact configured as an optional so, Artifact will be  ignored", art.Name)
+				continue
+			} else {
+				return errors.New("required artifact %s not supplied", art.Name)
+			}
+		}
 		artDriver, err := we.InitDriver(art)
 		if err != nil {
 			return err
@@ -232,6 +242,10 @@ func (we *WorkflowExecutor) saveArtifact(tempOutArtDir string, mainCtrID string,
 	localArtPath := path.Join(tempOutArtDir, fileName)
 	err := we.RuntimeExecutor.CopyFile(mainCtrID, art.Path, localArtPath)
 	if err != nil {
+		if art.Optional && errors.IsCode(errors.CodeNotFound, err) {
+			log.Warnf("Error in saving Artifact. Artifact configured as an optional so, Error will be  ignored. Error= %v", err)
+			return nil
+		}
 		return err
 	}
 	fileName, localArtPath, err = stageArchiveFile(fileName, localArtPath, art)
@@ -502,6 +516,7 @@ func (we *WorkflowExecutor) InitDriver(art wfv1.Artifact) (artifact.ArtifactDriv
 	if art.Raw != nil {
 		return &raw.RawArtifactDriver{}, nil
 	}
+
 	return nil, errors.Errorf(errors.CodeBadRequest, "Unsupported artifact driver for %s", art.Name)
 }