Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support HDFS Artifact #1159

Merged
merged 9 commits into from
Jan 12, 2019
Merged

Support HDFS Artifact #1159

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
d9570d2
Return bytes of Secret instead of string
dtaniwaki Dec 27, 2018
febd4e0
Define getConfigMaps
dtaniwaki Dec 29, 2018
4283b03
Define resource interface
dtaniwaki Jan 4, 2019
92473da
Implement HDFS artifact
dtaniwaki Jan 5, 2019
a49423e
Fix typo
dtaniwaki Jan 6, 2019
5962f33
Rename GetConfigMaps to GetConfigMapKey
dtaniwaki Jan 11, 2019
89a06b8
Use util.Close()
dtaniwaki Jan 11, 2019
504c418
Remove unnecessary log
dtaniwaki Jan 11, 2019
34ed28d
Support default HDFS configuration
dtaniwaki Jan 11, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
102 changes: 102 additions & 0 deletions Gopkg.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 7 additions & 0 deletions Gopkg.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,3 +55,10 @@ required = [
name = "github.com/Azure/go-autorest"
revision = "1ff28809256a84bb6966640ff3d0371af82ccba4"

[[constraint]]
name = "github.com/colinmarc/hdfs"
revision = "48eb8d6c34a97ffc73b406356f0f2e1c569b42a5"

[[constraint]]
name = "gopkg.in/jcmturner/gokrb5.v5"
version = "5.3.0"
132 changes: 132 additions & 0 deletions api/openapi-spec/swagger.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,10 @@
"description": "GlobalName exports an output artifact to the global scope, making it available as '{{workflow.outputs.artifacts.XXXX}} and in workflow.status.outputs.artifacts",
"type": "string"
},
"hdfs": {
"description": "HDFS contains HDFS artifact location details",
"$ref": "#/definitions/io.argoproj.workflow.v1alpha1.HDFSArtifact"
},
"http": {
"description": "HTTP contains HTTP artifact location details",
"$ref": "#/definitions/io.argoproj.workflow.v1alpha1.HTTPArtifact"
Expand Down Expand Up @@ -108,6 +112,10 @@
"description": "Git contains git artifact location details",
"$ref": "#/definitions/io.argoproj.workflow.v1alpha1.GitArtifact"
},
"hdfs": {
"description": "HDFS contains HDFS artifact location details",
"$ref": "#/definitions/io.argoproj.workflow.v1alpha1.HDFSArtifact"
},
"http": {
"description": "HTTP contains HTTP artifact location details",
"$ref": "#/definitions/io.argoproj.workflow.v1alpha1.HTTPArtifact"
Expand Down Expand Up @@ -249,6 +257,130 @@
}
}
},
"io.argoproj.workflow.v1alpha1.HDFSArtifact": {
"description": "HDFSArtifact is the location of an HDFS artifact",
"required": [
"addresses",
"path"
],
"properties": {
"addresses": {
"description": "Addresses is accessible addresses of HDFS name nodes",
"type": "array",
"items": {
"type": "string"
}
},
"force": {
"description": "Force copies a file forcibly even if it exists (default: false)",
"type": "boolean"
},
"hdfsUser": {
"description": "HDFSUser is the user to access HDFS file system. It is ignored if either ccache or keytab is used.",
"type": "string"
},
"krbCCacheSecret": {
"description": "KrbCCacheSecret is the secret selector for Kerberos ccache Either ccache or keytab can be set to use Kerberos.",
"$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector"
},
"krbConfigConfigMap": {
"description": "KrbConfig is the configmap selector for Kerberos config as string It must be set if either ccache or keytab is used.",
"$ref": "#/definitions/io.k8s.api.core.v1.ConfigMapKeySelector"
},
"krbKeytabSecret": {
"description": "KrbKeytabSecret is the secret selector for Kerberos keytab Either ccache or keytab can be set to use Kerberos.",
"$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector"
},
"krbRealm": {
"description": "KrbRealm is the Kerberos realm used with Kerberos keytab It must be set if keytab is used.",
"type": "string"
},
"krbServicePrincipalName": {
"description": "KrbServicePrincipalName is the principal name of Kerberos service It must be set if either ccache or keytab is used.",
"type": "string"
},
"krbUsername": {
"description": "KrbUsername is the Kerberos username used with Kerberos keytab It must be set if keytab is used.",
"type": "string"
},
"path": {
"description": "Path is a file path in HDFS",
"type": "string"
}
}
},
"io.argoproj.workflow.v1alpha1.HDFSConfig": {
"description": "HDFSConfig is configurations for HDFS",
"required": [
"addresses"
],
"properties": {
"addresses": {
"description": "Addresses is accessible addresses of HDFS name nodes",
"type": "array",
"items": {
"type": "string"
}
},
"hdfsUser": {
"description": "HDFSUser is the user to access HDFS file system. It is ignored if either ccache or keytab is used.",
"type": "string"
},
"krbCCacheSecret": {
"description": "KrbCCacheSecret is the secret selector for Kerberos ccache Either ccache or keytab can be set to use Kerberos.",
"$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector"
},
"krbConfigConfigMap": {
"description": "KrbConfig is the configmap selector for Kerberos config as string It must be set if either ccache or keytab is used.",
"$ref": "#/definitions/io.k8s.api.core.v1.ConfigMapKeySelector"
},
"krbKeytabSecret": {
"description": "KrbKeytabSecret is the secret selector for Kerberos keytab Either ccache or keytab can be set to use Kerberos.",
"$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector"
},
"krbRealm": {
"description": "KrbRealm is the Kerberos realm used with Kerberos keytab It must be set if keytab is used.",
"type": "string"
},
"krbServicePrincipalName": {
"description": "KrbServicePrincipalName is the principal name of Kerberos service It must be set if either ccache or keytab is used.",
"type": "string"
},
"krbUsername": {
"description": "KrbUsername is the Kerberos username used with Kerberos keytab It must be set if keytab is used.",
"type": "string"
}
}
},
"io.argoproj.workflow.v1alpha1.HDFSKrbConfig": {
"description": "HDFSKrbConfig is auth configurations for Kerberos",
"properties": {
"krbCCacheSecret": {
"description": "KrbCCacheSecret is the secret selector for Kerberos ccache Either ccache or keytab can be set to use Kerberos.",
"$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector"
},
"krbConfigConfigMap": {
"description": "KrbConfig is the configmap selector for Kerberos config as string It must be set if either ccache or keytab is used.",
"$ref": "#/definitions/io.k8s.api.core.v1.ConfigMapKeySelector"
},
"krbKeytabSecret": {
"description": "KrbKeytabSecret is the secret selector for Kerberos keytab Either ccache or keytab can be set to use Kerberos.",
"$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector"
},
"krbRealm": {
"description": "KrbRealm is the Kerberos realm used with Kerberos keytab It must be set if keytab is used.",
"type": "string"
},
"krbServicePrincipalName": {
"description": "KrbServicePrincipalName is the principal name of Kerberos service It must be set if either ccache or keytab is used.",
"type": "string"
},
"krbUsername": {
"description": "KrbUsername is the Kerberos username used with Kerberos keytab It must be set if keytab is used.",
"type": "string"
}
}
},
"io.argoproj.workflow.v1alpha1.HTTPArtifact": {
"description": "HTTPArtifact allows an file served on HTTP to be placed as an input artifact in a container",
"required": [
Expand Down
Loading