From e49474309ab504ba014e73ddb6ab3db17313a4ef Mon Sep 17 00:00:00 2001 From: Markus Maga Date: Fri, 8 Nov 2019 16:10:13 +0100 Subject: [PATCH 1/4] feat: add validation to CRD --- README.md | 6 +- config/index.js | 8 ++- crd.yaml | 78 ++++++++++++++++++++++ custom-resource-manifest.json | 40 ----------- examples/data-from-example.yml | 8 +++ examples/dockerconfig-example.yml | 5 +- examples/hello-service-external-secret.yml | 8 ++- examples/secretsmanager-example.yaml | 4 +- examples/ssm-example.yaml | 4 +- examples/tls-example.yml | 5 +- lib/poller.js | 17 ++--- lib/poller.test.js | 32 ++++++++- package-lock.json | 2 +- package.json | 1 + 14 files changed, 153 insertions(+), 65 deletions(-) create mode 100644 crd.yaml delete mode 100644 custom-resource-manifest.json create mode 100644 examples/data-from-example.yml diff --git a/README.md b/README.md index 34c2b297..b1b7fa92 100644 --- a/README.md +++ b/README.md @@ -238,7 +238,7 @@ aws secretsmanager create-secret --region us-west-2 --name hello-service/credent We can declare which properties we want from hello-service/credentials: ```yml -apiVersion: 'kubernetes-client.io/v1' +apiVersion: kubernetes-client.io/v1 kind: ExternalSecret metadata: name: hello-service @@ -258,7 +258,7 @@ spec: alternatively you can use `dataFrom` and get all the values from hello-service/credentials: ```yml -apiVersion: 'kubernetes-client.io/v1' +apiVersion: kubernetes-client.io/v1 kind: ExternalSecret metadata: name: hello-service @@ -273,7 +273,7 @@ spec: `data` and `dataFrom` can of course be combined, any naming conflicts will use the last defined, with `data` overriding `dataFrom` ```yml -apiVersion: 'kubernetes-client.io/v1' +apiVersion: kubernetes-client.io/v1 kind: ExternalSecret metadata: name: hello-service diff --git a/config/index.js b/config/index.js index 04bfab25..31c1dbd4 100644 --- a/config/index.js +++ b/config/index.js @@ -3,14 +3,20 @@ const kube = require('kubernetes-client') const KubeRequest = require('kubernetes-client/backends/request') const pino = require('pino') +const yaml = require('js-yaml') +const fs = require('fs') +const path = require('path') const awsConfig = require('./aws-config') const envConfig = require('./environment') const CustomResourceManager = require('../lib/custom-resource-manager') -const customResourceManifest = require('../custom-resource-manifest.json') const SecretsManagerBackend = require('../lib/backends/secrets-manager-backend') const SystemManagerBackend = require('../lib/backends/system-manager-backend') +// Get document, or throw exception on error +// eslint-disable-next-line security/detect-non-literal-fs-filename +const customResourceManifest = yaml.safeLoad(fs.readFileSync(path.resolve(__dirname, '../crd.yaml'), 'utf8')) + const kubeconfig = new kube.KubeConfig() kubeconfig.loadFromDefault() const kubeBackend = new KubeRequest({ kubeconfig }) diff --git a/crd.yaml b/crd.yaml new file mode 100644 index 00000000..1b89f960 --- /dev/null +++ b/crd.yaml @@ -0,0 +1,78 @@ +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: externalsecrets.kubernetes-client.io +spec: + group: kubernetes-client.io + version: v1 + scope: Namespaced + + names: + shortNames: + - es + kind: ExternalSecret + plural: externalsecrets + singular: externalsecret + + additionalPrinterColumns: + - JSONPath: .status.lastSync + name: Last Sync + type: date + - JSONPath: .status.status + name: status + type: string + - JSONPath: .metadata.creationTimestamp + name: Age + type: date + + validation: + openAPIV3Schema: + properties: + spec: + type: object + properties: + type: + description: Deprecated! Use template.type instead + type: string + template: + description: Template which will be deep merged without mutating + any existing fields. into generated secret, can be used to + set for example annotations or type on the generated secret + type: object + backendType: + type: string + dataFrom: + type: array + items: + type: string + data: + type: array + items: + type: object + properties: + key: + description: Secret key in backend + type: string + name: + description: Name set for this key in the generated secret + type: string + property: + description: Property to extract if secret in backend is a JSON object + required: + - name + - key + roleArn: + type: string + required: + - backendType + anyOf: + - required: + - data + - required: + - dataFrom + + secretDescriptor: + description: Deprecated! Use spec instead + + subresources: + status: {} diff --git a/custom-resource-manifest.json b/custom-resource-manifest.json deleted file mode 100644 index af5326ff..00000000 --- a/custom-resource-manifest.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "kind": "CustomResourceDefinition", - "spec": { - "scope": "Namespaced", - "additionalPrinterColumns": [ - { - "JSONPath": ".status.lastSync", - "name": "Last Sync", - "type": "date" - }, - { - "JSONPath": ".status.status", - "name": "status", - "type": "string" - }, - { - "JSONPath": ".metadata.creationTimestamp", - "name": "Age", - "type": "date" - } - ], - "version": "v1", - "group": "kubernetes-client.io", - "names": { - "shortNames": [ - "es" - ], - "kind": "ExternalSecret", - "plural": "externalsecrets", - "singular": "externalsecret" - }, - "subresources": { - "status": {} - } - }, - "apiVersion": "apiextensions.k8s.io/v1beta1", - "metadata": { - "name": "externalsecrets.kubernetes-client.io" - } -} diff --git a/examples/data-from-example.yml b/examples/data-from-example.yml new file mode 100644 index 00000000..7d7fcfc3 --- /dev/null +++ b/examples/data-from-example.yml @@ -0,0 +1,8 @@ +apiVersion: kubernetes-client.io/v1 +kind: ExternalSecret +metadata: + name: data-from-example +spec: + backendType: systemManager + dataFrom: + - /foo/name1 diff --git a/examples/dockerconfig-example.yml b/examples/dockerconfig-example.yml index d9f93bd8..ccab067f 100644 --- a/examples/dockerconfig-example.yml +++ b/examples/dockerconfig-example.yml @@ -1,10 +1,11 @@ apiVersion: kubernetes-client.io/v1 kind: ExternalSecret metadata: - name: dockerhub-secret + name: dockerconfig-example spec: backendType: secretsManager - type: kubernetes.io/dockerconfigjson + template: + type: kubernetes.io/dockerconfigjson data: - key: /development/dockerhub name: .dockerconfigjson diff --git a/examples/hello-service-external-secret.yml b/examples/hello-service-external-secret.yml index d6f4ff68..2c10dba7 100644 --- a/examples/hello-service-external-secret.yml +++ b/examples/hello-service-external-secret.yml @@ -1,9 +1,15 @@ -apiVersion: 'kubernetes-client.io/v1' +apiVersion: kubernetes-client.io/v1 kind: ExternalSecret metadata: name: hello-service spec: + template: + metadata: + annotations: + external-secret: 'Yes please!' backendType: secretsManager data: - key: hello-service/password name: password + dataFrom: + - hello-service/secret-envs diff --git a/examples/secretsmanager-example.yaml b/examples/secretsmanager-example.yaml index bb3f91f8..46f1b055 100644 --- a/examples/secretsmanager-example.yaml +++ b/examples/secretsmanager-example.yaml @@ -1,7 +1,7 @@ -apiVersion: 'kubernetes-client.io/v1' +apiVersion: kubernetes-client.io/v1 kind: ExternalSecret metadata: - name: demo-service + name: secretsmanager-example spec: backendType: secretsManager # optional: specify role to assume when retrieving the data diff --git a/examples/ssm-example.yaml b/examples/ssm-example.yaml index 6db65679..be62b905 100644 --- a/examples/ssm-example.yaml +++ b/examples/ssm-example.yaml @@ -1,7 +1,7 @@ -apiVersion: 'kubernetes-client.io/v1' +apiVersion: kubernetes-client.io/v1 kind: ExternalSecret metadata: - name: ssm-secret-key + name: ssm-example spec: backendType: systemManager # optional: specify role to assume when retrieving the data diff --git a/examples/tls-example.yml b/examples/tls-example.yml index c15279f5..cd3ee51f 100644 --- a/examples/tls-example.yml +++ b/examples/tls-example.yml @@ -1,10 +1,11 @@ apiVersion: kubernetes-client.io/v1 kind: ExternalSecret metadata: - name: dockerhub-secret + name: tls-secret spec: backendType: secretsManager - type: kubernetes.io/tls + template: + type: kubernetes.io/tls data: - key: /development/certificate property: crt diff --git a/lib/poller.js b/lib/poller.js index 069aeff7..123f723c 100644 --- a/lib/poller.js +++ b/lib/poller.js @@ -75,10 +75,15 @@ class Poller { */ async _createSecretManifest () { const spec = this._spec - const template = spec.template + const template = spec.template || {} + + // spec.type for backwards compat + const type = template.type || spec.type || 'Opaque' + const data = await this._backends[spec.backendType] .getSecretManifestData({ spec }) - let secretManifest = { + + const secretManifest = { apiVersion: 'v1', kind: 'Secret', metadata: { @@ -87,15 +92,11 @@ class Poller { this._ownerReference ] }, - type: spec.type || 'Opaque', + type, data } - if (template) { - secretManifest = merge(clonedeep(template), secretManifest) - } - - return secretManifest + return merge(clonedeep(template), secretManifest) } /** diff --git a/lib/poller.test.js b/lib/poller.test.js index afc6d2e5..6e6db67e 100644 --- a/lib/poller.test.js +++ b/lib/poller.test.js @@ -183,12 +183,12 @@ describe('Poller', () => { }) }) - it('creates secret manifest - with type', async () => { + it('creates secret manifest - with type (backwards compat)', async () => { const poller = pollerFactory({ type: 'dummy-test-type', backendType: 'fakeBackendType', name: 'fakeSecretName', - properties: [ + data: [ 'fakePropertyName1', 'fakePropertyName2' ] @@ -206,7 +206,7 @@ describe('Poller', () => { type: 'dummy-test-type', backendType: 'fakeBackendType', name: 'fakeSecretName', - properties: [ + data: [ 'fakePropertyName1', 'fakePropertyName2' ] @@ -228,6 +228,32 @@ describe('Poller', () => { }) }) + it('creates secret manifest - with template type (should work with backwards compat type)', async () => { + const poller = pollerFactory({ + template: { + type: 'dummy-test-type' + }, + backendType: 'fakeBackendType', + name: 'fakeSecretName', + data: [] + }) + + backendMock.getSecretManifestData.resolves({}) + + const secretManifest = await poller._createSecretManifest() + + expect(secretManifest).deep.equals({ + apiVersion: 'v1', + kind: 'Secret', + metadata: { + name: 'fakeSecretName', + ownerReferences: [getOwnerReference()] + }, + type: 'dummy-test-type', + data: {} + }) + }) + it('creates secret manifest - with template', async () => { const poller = pollerFactory({ type: 'dummy-test-type', diff --git a/package-lock.json b/package-lock.json index e60c0606..98481ee6 100644 --- a/package-lock.json +++ b/package-lock.json @@ -6755,7 +6755,7 @@ }, "sprintf-js": { "version": "1.0.3", - "resolved": "http://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=" }, "sshpk": { diff --git a/package.json b/package.json index e48621dc..45fff705 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,7 @@ "dependencies": { "aws-sdk": "^2.566.0", "express": "^4.17.1", + "js-yaml": "^3.13.1", "json-stream": "^1.0.0", "kubernetes-client": "^8.3.0", "lodash.clonedeep": "^4.5.0", From 30e7b16dbe0df4bce64ec6c36d67b92785efbeea Mon Sep 17 00:00:00 2001 From: Markus Maga Date: Fri, 8 Nov 2019 18:11:56 +0100 Subject: [PATCH 2/4] fix: backend type validation as enum --- crd.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crd.yaml b/crd.yaml index 1b89f960..c9acd93a 100644 --- a/crd.yaml +++ b/crd.yaml @@ -41,6 +41,10 @@ spec: type: object backendType: type: string + enum: + - secretsManager + - systemManager + - vault dataFrom: type: array items: From de89037e303548f499973bdad1b82b4bd2ac1c76 Mon Sep 17 00:00:00 2001 From: Markus Maga Date: Fri, 8 Nov 2019 18:12:11 +0100 Subject: [PATCH 3/4] docs: add deprecation notice --- README.md | 11 +++++++++++ crd.yaml | 6 ------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index b1b7fa92..6a7634d3 100644 --- a/README.md +++ b/README.md @@ -215,6 +215,17 @@ data: password: MTIzNA== ``` +## Deprecations + +A few properties has changed name overtime, we still maintain backwards compatbility with these but they will eventually be removed, and they are not validated using the CRD validation. + +| Old | New | +| ----------------------------- | ------------------------------ | +| `secretDescriptor` | `spec` | +| `spec.type` | `spec.template.type` | +| `spec.properties` | `spec.data` | +| `backendType: secretManager` | `backendType: secretsManager` | + ## Backends kubernetes-external-secrets supports both AWS Secrets Manager and AWS System Manager. diff --git a/crd.yaml b/crd.yaml index c9acd93a..86311e19 100644 --- a/crd.yaml +++ b/crd.yaml @@ -31,9 +31,6 @@ spec: spec: type: object properties: - type: - description: Deprecated! Use template.type instead - type: string template: description: Template which will be deep merged without mutating any existing fields. into generated secret, can be used to @@ -75,8 +72,5 @@ spec: - required: - dataFrom - secretDescriptor: - description: Deprecated! Use spec instead - subresources: status: {} From 11eab14e33691231dece0bbe42a5b468f59c41d8 Mon Sep 17 00:00:00 2001 From: Markus Maga Date: Fri, 8 Nov 2019 21:15:22 +0100 Subject: [PATCH 4/4] docs: bump prereq kubernetes version --- charts/kubernetes-external-secrets/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/charts/kubernetes-external-secrets/README.md b/charts/kubernetes-external-secrets/README.md index 6ccf7b78..09c1ec5e 100644 --- a/charts/kubernetes-external-secrets/README.md +++ b/charts/kubernetes-external-secrets/README.md @@ -10,7 +10,7 @@ $ helm install external-secrets/kubernetes-external-secrets ## Prerequisites -* Kubernetes 1.7+ +* Kubernetes 1.12+ ## Installing the Chart