diff --git a/README.md b/README.md index 34dad1d1..52141b38 100644 --- a/README.md +++ b/README.md @@ -385,6 +385,33 @@ spec: property: password ``` +# AWS SSM Parameter Store + +You can scrape values from SSM Parameter Store individually or by providing a path to fetch all keys inside. + +Additionally you can also scrape all sub paths (child paths) if you need to. The default is not to scrape child paths + +```yml +apiVersion: kubernetes-client.io/v1 +kind: ExternalSecret +metadata: + name: hello-service +spec: + backendType: secretsManager + # optional: specify role to assume when retrieving the data + roleArn: arn:aws:iam::123456789012:role/test-role + # optional: specify region + region: us-east-1 + data: + - key: /foo/name + name: fooName + - path: /extra-people/ + recursive: false +``` + + + + ### Hashicorp Vault kubernetes-external-secrets supports fetching secrets from [Hashicorp Vault](https://www.vaultproject.io/), using the [Kubernetes authentication method](https://www.vaultproject.io/docs/auth/kubernetes). diff --git a/charts/kubernetes-external-secrets/crds/kubernetes-client.io_externalsecrets_crd.yaml b/charts/kubernetes-external-secrets/crds/kubernetes-client.io_externalsecrets_crd.yaml index b6b93f39..69f45aa2 100644 --- a/charts/kubernetes-external-secrets/crds/kubernetes-client.io_externalsecrets_crd.yaml +++ b/charts/kubernetes-external-secrets/crds/kubernetes-client.io_externalsecrets_crd.yaml @@ -72,30 +72,35 @@ spec: type: array items: type: object - properties: - key: - description: Secret key in backend - type: string - name: - description: Name set for this key in the generated secret - type: string - property: - description: Property to extract if secret in backend is a JSON object - isBinary: - description: >- - Whether the backend secret shall be treated as binary data - represented by a base64-encoded string. You must set this to true - for any base64-encoded binary data in the backend - to ensure it - is not encoded in base64 again. Default is false. - type: boolean - recursive: - description: Allow to recurse thru all child keys on a given path - type: boolean - oneOf: - - required: + anyOf: + - properties: + key: + description: Secret key in backend + type: string + name: + description: Name set for this key in the generated secret + type: string + property: + description: Property to extract if secret in backend is a JSON object + isBinary: + description: >- + Whether the backend secret shall be treated as binary data + represented by a base64-encoded string. You must set this to true + for any base64-encoded binary data in the backend - to ensure it + is not encoded in base64 again. Default is false. + type: boolean + required: - key - name - - required: + - properties: + path: + description: >- + Path from SSM to scrape secrets + This will fetch all secrets and use the key from the secret as variable name + recursive: + description: Allow to recurse thru all child keys on a given path + type: boolean + required: - path roleArn: type: string diff --git a/e2e/tests/crd.test.js b/e2e/tests/crd.test.js index ed6fe6dd..99f0ca7f 100644 --- a/e2e/tests/crd.test.js +++ b/e2e/tests/crd.test.js @@ -56,3 +56,4 @@ describe('CRD', () => { .catch(err => expect(err).to.be.an('error')) }) }) + diff --git a/e2e/tests/ssm.test.js b/e2e/tests/ssm.test.js index 23b76f70..3e279a87 100644 --- a/e2e/tests/ssm.test.js +++ b/e2e/tests/ssm.test.js @@ -53,6 +53,54 @@ describe('ssm', async () => { expect(secret.body.data.name).to.equal('Zm9v') }) + it('should pull existing secrets from ssm path and create a secret from it', async () => { + let name1 = await putParameter({ + Name: `/e2e/${uuid}-names/name1`, + Type: 'String', + Value: 'foo' + }).catch(err => { + expect(err).to.equal(null) + }) + + let name2 = await putParameter({ + Name: `/e2e/${uuid}-names/name2`, + Type: 'String', + Value: 'bar' + }).catch(err => { + expect(err).to.equal(null) + }) + + let result = await kubeClient + .apis[customResourceManifest.spec.group] + .v1.namespaces('default')[customResourceManifest.spec.names.plural] + .post({ + body: { + apiVersion: 'kubernetes-client.io/v1', + kind: 'ExternalSecret', + metadata: { + name: `e2e-ssm-${uuid}-names` + }, + spec: { + backendType: 'systemManager', + data: [ + { + path: `/e2e/${uuid}-names`, + } + ] + } + } + }) + + expect(name1).to.not.equal(undefined) + expect(name2).to.not.equal(undefined) + expect(result).to.not.equal(undefined) + expect(result.statusCode).to.equal(201) + + const secret = await waitForSecret('default', `e2e-ssm-${uuid}-names`) + expect(secret.body.data.name1).to.equal('Zm9v') // Expect base64 foo + expect(secret.body.data.name2).to.equal('YmFy') // Expect base64 bar + }) + it('should pull existing secret from ssm in a different region', async () => { const ssmEU = awsConfig.systemManagerFactory({ region: 'eu-west-1' diff --git a/examples/ssm-example.yaml b/examples/ssm-example.yaml index 99b018e8..8cf9f685 100644 --- a/examples/ssm-example.yaml +++ b/examples/ssm-example.yaml @@ -1,7 +1,6 @@ apiVersion: kubernetes-client.io/v1 kind: ExternalSecret metadata: - namespace: test name: ssm-example spec: backendType: systemManager @@ -10,10 +9,11 @@ spec: # optional: specify region region: us-west-2 data: - # can either be key+name or all keys from a given path or even both - # order is important if you have same key name on different paths that you want to include + # Can either be key+name or all keys from a given path or even both + # Order below is important. Values are fetched from SSM in the same order you put them here (top to bottom) + # This means that if a given key is found duplicate, the last value found has precedence - key: /foo/name name: variable-name - path: /bar/ - # optional: choose wether to scrape all child paths or not. Default is false + # optional: choose whether to scrape all child paths or not. Default is false recursive: false diff --git a/lib/backends/kv-backend.js b/lib/backends/kv-backend.js index 0c87b777..fd42aece 100644 --- a/lib/backends/kv-backend.js +++ b/lib/backends/kv-backend.js @@ -27,25 +27,22 @@ class KVBackend extends AbstractBackend { * @returns {Promise} Promise object representing secret property values. */ _fetchDataValues ({ data, specOptions }) { - return Promise.all(data.map(async dataItem => { const { name, property = null, key, path, ...keyOptions } = dataItem let response = {} - let plainOrObjValue; + let plainOrObjValue // Supporting fetching by key or by path - // If path is not defined, than key will be for sure because of CRD validation + // If 'path' is not defined, we can assume 'key' will exist due to CRD validation let singleParameterKey = true - if (path) { - singleParameterKey = false - } + if (path) { singleParameterKey = false } if (singleParameterKey) { // Single secret plainOrObjValue = await this._get({ key, keyOptions, specOptions }) } else { - // Whole path + // All secrets inside the specified path plainOrObjValue = await this._getByPath({ path, keyOptions, specOptions }) } @@ -59,8 +56,8 @@ class KVBackend extends AbstractBackend { parsedValue = JSON.parse(value) } catch (err) { this._logger.warn(`Failed to JSON.parse value for '${key}',` + - ' please verify that your secret value is correctly formatted as JSON.' + - ` To use plain text secret remove the 'property: ${property}'`) + ' please verify that your secret value is correctly formatted as JSON.' + + ` To use plain text secret remove the 'property: ${property}'`) return } @@ -86,13 +83,12 @@ class KVBackend extends AbstractBackend { response = { [name]: value } } else { // Returning dict with path keys and values - for (var records in value) { - //console.log(value[records]) + for (const records in value) { response[records] = value[records] } } - return response + return response })) } @@ -110,7 +106,7 @@ class KVBackend extends AbstractBackend { return JSON.parse(value) } catch (err) { this._logger.warn(`Failed to JSON.parse value for '${key}',` + - ' please verify that your secret value is correctly formatted as JSON.') + ' please verify that your secret value is correctly formatted as JSON.') } })) } @@ -126,6 +122,17 @@ class KVBackend extends AbstractBackend { throw new Error('_get not implemented') } + /** + * Get a secret property value from Key Value backend. + * @param {string} path - Path from where to fetch secrets on the backend. + * @param {string} keyOptions - Options for this specific key, eg version etc. + * @param {string} specOptions - Options for this external secret, eg role + * @returns {Promise} Promise object representing secret property values. + */ + _getByPath ({ path, keyOptions, specOptions }) { + throw new Error('_getByPath not implemented') + } + /** * Convert secret value to buffer * @param {(string|Buffer|object)} plainValue - plain secret value diff --git a/lib/backends/system-manager-backend.js b/lib/backends/system-manager-backend.js index 596b8361..aa840d97 100644 --- a/lib/backends/system-manager-backend.js +++ b/lib/backends/system-manager-backend.js @@ -73,18 +73,13 @@ class SystemManagerBackend extends KVBackend { * @returns {Promise} Promise object representing secret property value. */ async _getByPath ({ path, keyOptions, specOptions: { roleArn, region } }) { - let client = this._client let factoryArgs = null - let recursive = false - if (keyOptions['recursive']) { - // Recursing thru all childrens from this key - recursive = true - } + const recursive = keyOptions.recursive || false - this._logger.info(`fetching all secrets ${recursive ? "(recursively)" : ""}inside path ${path} with role ${roleArn ? roleArn : " from pod"} in region ${region}`) + this._logger.info(`fetching all secrets ${recursive ? '(recursively)' : ''} inside path ${path} with role ${roleArn !== ' from pod'} in region ${region}`) - if (roleArn) { + if (roleArn) { const credentials = this._assumeRole({ RoleArn: roleArn, RoleSessionName: 'k8s-external-secrets' @@ -105,42 +100,36 @@ class SystemManagerBackend extends KVBackend { } try { const getAllParameters = async () => { - const EMPTY = Symbol("empty"); + const EMPTY = Symbol('empty') this._logger.info(`fetching parameters for path ${path}`) - const res = []; - for await (const lf of (async function* () { - let NextToken = EMPTY; + const res = [] + for await (const lf of (async function * () { + let NextToken = EMPTY while (NextToken || NextToken === EMPTY) { const parameters = await client.getParametersByPath({ Path: path, WithDecryption: true, Recursive: recursive, NextToken: NextToken !== EMPTY ? NextToken : undefined - }).promise(); - yield* parameters.Parameters; - NextToken = parameters.NextToken; + }).promise() + yield * parameters.Parameters + NextToken = parameters.NextToken } })()) { - res.push(lf); + res.push(lf) } - return res; + return res } - var parameters = {} - - const ssmData = await getAllParameters(); - - for (var ssmRecord in ssmData) { - - var paramName = require('path').basename(ssmData[ssmRecord].Name) - var paramValue = ssmData[ssmRecord].Value - + const parameters = {} + const ssmData = await getAllParameters() + for (const ssmRecord in ssmData) { + const paramName = require('path').basename(ssmData[String(ssmRecord)].Name) + const paramValue = ssmData[ssmRecord].Value parameters[paramName] = paramValue - } return parameters - } catch (err) { if (err.code === 'ParameterNotFound' && (!err.message || err.message === 'null')) { err.message = `ParameterNotFound: ${path} could not be found.` diff --git a/package.json b/package.json index d9119df6..5a0061a2 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "coverage": "nyc ./node_modules/mocha/bin/_mocha --recursive lib", "lint": "eslint --fix --ignore-pattern /coverage/ ./", "local": "LOCALSTACK=1 AWS_ACCESS_KEY_ID=foobar AWS_SECRET_ACCESS_KEY=foobar nodemon", - "localstack": "docker run -it -p 4566:4566 -p 4583:4583 -p 4584:4584 -p 4592:4592 -p 9999:8080 -e SERVICES=ssm,secretsmanager,sts -e DEBUG=1 --rm localstack/localstack:latest", + "localstack": "docker run -it -p 4566:4566 -p 9999:8080 -e SERVICES=ssm,secretsmanager,sts -e DEBUG=1 --rm localstack/localstack:latest", "release": "standard-version --tag-prefix='' && ./release.sh", "start": "./bin/daemon.js", "nodemon": "nodemon ./bin/daemon.js",