Multiple changes related with improvements

Updating Readme
Improving code with users suggestions
Lint fixes
Adding tests for ssm path feature

README.md

@@ -385,6 +385,33 @@ spec:
       property: password
 ```
 
+# AWS SSM Parameter Store
+
+You can scrape values from SSM Parameter Store individually or by providing a path to fetch all keys inside.
+
+Additionally you can also scrape all sub paths (child paths) if you need to. The default is not to scrape child paths
+
+```yml
+apiVersion: kubernetes-client.io/v1
+kind: ExternalSecret
+metadata:
+  name: hello-service
+spec:
+  backendType: secretsManager
+  # optional: specify role to assume when retrieving the data
+  roleArn: arn:aws:iam::123456789012:role/test-role
+  # optional: specify region
+  region: us-east-1
+  data:
+    - key: /foo/name
+      name: fooName
+    - path: /extra-people/
+      recursive: false
+```
+
+
+
+
 ### Hashicorp Vault
 
 kubernetes-external-secrets supports fetching secrets from [Hashicorp Vault](https://www.vaultproject.io/), using the [Kubernetes authentication method](https://www.vaultproject.io/docs/auth/kubernetes).

charts/kubernetes-external-secrets/crds/kubernetes-client.io_externalsecrets_crd.yaml

@@ -72,30 +72,35 @@ spec:
               type: array
               items:
                 type: object
-                properties:
-                  key:
-                    description: Secret key in backend
-                    type: string
-                  name:
-                    description: Name set for this key in the generated secret
-                    type: string
-                  property:
-                    description: Property to extract if secret in backend is a JSON object
-                  isBinary:
-                    description: >-
-                      Whether the backend secret shall be treated as binary data
-                      represented by a base64-encoded string. You must set this to true
-                      for any base64-encoded binary data in the backend - to ensure it
-                      is not encoded in base64 again. Default is false.
-                    type: boolean
-                  recursive:
-                    description: Allow to recurse thru all child keys on a given path
-                    type: boolean
-                oneOf:
-                  - required:
+                anyOf:
+                  - properties:
+                      key:
+                        description: Secret key in backend
+                        type: string
+                      name:
+                        description: Name set for this key in the generated secret
+                        type: string
+                      property:
+                        description: Property to extract if secret in backend is a JSON object
+                      isBinary:
+                        description: >-
+                          Whether the backend secret shall be treated as binary data
+                          represented by a base64-encoded string. You must set this to true
+                          for any base64-encoded binary data in the backend - to ensure it
+                          is not encoded in base64 again. Default is false.
+                        type: boolean
+                    required:
                       - key
                       - name
-                  - required:
+                  - properties:
+                      path:
+                        description: >-
+                          Path from SSM to scrape secrets
+                          This will fetch all secrets and use the key from the secret as variable name
+                      recursive:
+                        description: Allow to recurse thru all child keys on a given path
+                        type: boolean
+                    required:
                       - path
             roleArn:
               type: string

e2e/tests/crd.test.js

@@ -56,3 +56,4 @@ describe('CRD', () => {
       .catch(err => expect(err).to.be.an('error'))
   })
 })
+

e2e/tests/ssm.test.js

@@ -53,6 +53,54 @@ describe('ssm', async () => {
     expect(secret.body.data.name).to.equal('Zm9v')
   })
 
+  it('should pull existing secrets from ssm path and create a secret from it', async () => {
+    let name1 = await putParameter({
+      Name: `/e2e/${uuid}-names/name1`,
+      Type: 'String',
+      Value: 'foo'
+    }).catch(err => {
+      expect(err).to.equal(null)
+    })
+
+    let name2 = await putParameter({
+      Name: `/e2e/${uuid}-names/name2`,
+      Type: 'String',
+      Value: 'bar'
+    }).catch(err => {
+      expect(err).to.equal(null)
+    })
+
+    let result = await kubeClient
+      .apis[customResourceManifest.spec.group]
+      .v1.namespaces('default')[customResourceManifest.spec.names.plural]
+      .post({
+        body: {
+          apiVersion: 'kubernetes-client.io/v1',
+          kind: 'ExternalSecret',
+          metadata: {
+            name: `e2e-ssm-${uuid}-names`
+          },
+          spec: {
+            backendType: 'systemManager',
+            data: [
+              {
+                path: `/e2e/${uuid}-names`,
+              }
+            ]
+          }
+        }
+      })
+
+    expect(name1).to.not.equal(undefined)
+    expect(name2).to.not.equal(undefined)
+    expect(result).to.not.equal(undefined)
+    expect(result.statusCode).to.equal(201)
+
+    const secret = await waitForSecret('default', `e2e-ssm-${uuid}-names`)
+    expect(secret.body.data.name1).to.equal('Zm9v') // Expect base64 foo
+    expect(secret.body.data.name2).to.equal('YmFy') // Expect base64 bar
+  })
+
   it('should pull existing secret from ssm in a different region', async () => {
     const ssmEU = awsConfig.systemManagerFactory({
       region: 'eu-west-1'

examples/ssm-example.yaml

@@ -1,7 +1,6 @@
 apiVersion: kubernetes-client.io/v1
 kind: ExternalSecret
 metadata:
-  namespace: test
   name: ssm-example
 spec:
   backendType: systemManager
@@ -10,10 +9,11 @@ spec:
   # optional: specify region
   region: us-west-2
   data:
-    # can either be key+name or all keys from a given path or even both
-    # order is important if you have same key name on different paths that you want to include
+    # Can either be key+name or all keys from a given path or even both
+    # Order below is important. Values are fetched from SSM in the same order you put them here (top to bottom)
+    # This means that if a given key is found duplicate, the last value found has precedence
     - key: /foo/name
       name: variable-name
     - path: /bar/
-      # optional: choose wether to scrape all child paths or not. Default is false
+      # optional: choose whether to scrape all child paths or not. Default is false
       recursive: false

lib/backends/kv-backend.js

@@ -27,25 +27,22 @@ class KVBackend extends AbstractBackend {
    * @returns {Promise} Promise object representing secret property values.
    */
   _fetchDataValues ({ data, specOptions }) {
-
     return Promise.all(data.map(async dataItem => {
       const { name, property = null, key, path, ...keyOptions } = dataItem
 
       let response = {}
-      let plainOrObjValue;
+      let plainOrObjValue
 
       // Supporting fetching by key or by path
-      // If path is not defined, than key will be for sure because of CRD validation
+      // If 'path' is not defined, we can assume 'key' will exist due to CRD validation
       let singleParameterKey = true
-      if (path) {
-        singleParameterKey = false
-      }
+      if (path) { singleParameterKey = false }
 
       if (singleParameterKey) {
         // Single secret
         plainOrObjValue = await this._get({ key, keyOptions, specOptions })
       } else {
-        // Whole path
+        // All secrets inside the specified path
         plainOrObjValue = await this._getByPath({ path, keyOptions, specOptions })
       }
 
@@ -59,8 +56,8 @@ class KVBackend extends AbstractBackend {
           parsedValue = JSON.parse(value)
         } catch (err) {
           this._logger.warn(`Failed to JSON.parse value for '${key}',` +
-           ' please verify that your secret value is correctly formatted as JSON.' +
-           ` To use plain text secret remove the 'property: ${property}'`)
+            ' please verify that your secret value is correctly formatted as JSON.' +
+            ` To use plain text secret remove the 'property: ${property}'`)
           return
         }
 
@@ -86,13 +83,12 @@ class KVBackend extends AbstractBackend {
         response = { [name]: value }
       } else {
         // Returning dict with path keys and values
-        for (var records in value) {
-          //console.log(value[records])
+        for (const records in value) {
           response[records] = value[records]
         }
       }
 
-      return response      
+      return response
     }))
   }
 
@@ -110,7 +106,7 @@ class KVBackend extends AbstractBackend {
         return JSON.parse(value)
       } catch (err) {
         this._logger.warn(`Failed to JSON.parse value for '${key}',` +
-           ' please verify that your secret value is correctly formatted as JSON.')
+          ' please verify that your secret value is correctly formatted as JSON.')
       }
     }))
   }
@@ -126,6 +122,17 @@ class KVBackend extends AbstractBackend {
     throw new Error('_get not implemented')
   }
 
+  /**
+   * Get a secret property value from Key Value backend.
+   * @param {string} path - Path from where to fetch secrets on the backend.
+   * @param {string} keyOptions - Options for this specific key, eg version etc.
+   * @param {string} specOptions - Options for this external secret, eg role
+   * @returns {Promise} Promise object representing secret property values.
+   */
+  _getByPath ({ path, keyOptions, specOptions }) {
+    throw new Error('_getByPath not implemented')
+  }
+
   /**
    * Convert secret value to buffer
    * @param {(string|Buffer|object)} plainValue - plain secret value

lib/backends/system-manager-backend.js

@@ -73,18 +73,13 @@ class SystemManagerBackend extends KVBackend {
    * @returns {Promise} Promise object representing secret property value.
    */
   async _getByPath ({ path, keyOptions, specOptions: { roleArn, region } }) {
-
     let client = this._client
     let factoryArgs = null
-    let recursive = false
-    if (keyOptions['recursive']) {
-      // Recursing thru all childrens from this key
-      recursive = true
-    }
+    const recursive = keyOptions.recursive || false
 
-    this._logger.info(`fetching all secrets ${recursive ? "(recursively)" : ""}inside path ${path} with role ${roleArn ? roleArn : " from pod"} in region ${region}`)
+    this._logger.info(`fetching all secrets ${recursive ? '(recursively)' : ''} inside path ${path} with role ${roleArn !== ' from pod'} in region ${region}`)
 
-    if (roleArn) {  
+    if (roleArn) {
       const credentials = this._assumeRole({
         RoleArn: roleArn,
         RoleSessionName: 'k8s-external-secrets'
@@ -105,42 +100,36 @@ class SystemManagerBackend extends KVBackend {
     }
     try {
       const getAllParameters = async () => {
-        const EMPTY = Symbol("empty");
+        const EMPTY = Symbol('empty')
         this._logger.info(`fetching parameters for path ${path}`)
-        const res = [];
-        for await (const lf of (async function* () {
-          let NextToken = EMPTY;
+        const res = []
+        for await (const lf of (async function * () {
+          let NextToken = EMPTY
           while (NextToken || NextToken === EMPTY) {
             const parameters = await client.getParametersByPath({
               Path: path,
               WithDecryption: true,
               Recursive: recursive,
               NextToken: NextToken !== EMPTY ? NextToken : undefined
-            }).promise();
-            yield* parameters.Parameters;
-            NextToken = parameters.NextToken;
+            }).promise()
+            yield * parameters.Parameters
+            NextToken = parameters.NextToken
           }
         })()) {
-          res.push(lf);
+          res.push(lf)
         }
-        return res;
+        return res
       }
 
-      var parameters = {}
-
-      const ssmData = await getAllParameters();
-
-      for (var ssmRecord in ssmData) {
-
-        var paramName = require('path').basename(ssmData[ssmRecord].Name)
-        var paramValue = ssmData[ssmRecord].Value
-
+      const parameters = {}
+      const ssmData = await getAllParameters()
+      for (const ssmRecord in ssmData) {
+        const paramName = require('path').basename(ssmData[String(ssmRecord)].Name)
+        const paramValue = ssmData[ssmRecord].Value
         parameters[paramName] = paramValue
-
       }
 
       return parameters
-
     } catch (err) {
       if (err.code === 'ParameterNotFound' && (!err.message || err.message === 'null')) {
         err.message = `ParameterNotFound: ${path} could not be found.`

package.json

@@ -7,7 +7,7 @@
     "coverage": "nyc ./node_modules/mocha/bin/_mocha --recursive lib",
     "lint": "eslint --fix --ignore-pattern /coverage/ ./",
     "local": "LOCALSTACK=1 AWS_ACCESS_KEY_ID=foobar AWS_SECRET_ACCESS_KEY=foobar nodemon",
-    "localstack": "docker run -it -p 4566:4566 -p 4583:4583 -p 4584:4584 -p 4592:4592 -p 9999:8080 -e SERVICES=ssm,secretsmanager,sts -e DEBUG=1 --rm localstack/localstack:latest",
+    "localstack": "docker run -it -p 4566:4566 -p 9999:8080 -e SERVICES=ssm,secretsmanager,sts -e DEBUG=1 --rm localstack/localstack:latest",
     "release": "standard-version --tag-prefix='' && ./release.sh",
     "start": "./bin/daemon.js",
     "nodemon": "nodemon ./bin/daemon.js",