Merge branch 'master' into add-twitter-generic-example

CHANGELOG.md

@@ -12,6 +12,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - MySQL and PostgreSQL connectors support SSL host name verification with
   `verify-full` SSL mode. Also adds optional `sslhost` configuration parameter
   that is compared to the server's certificate SAN. [#548](https://github.com/cyberark/secretless-broker/issues/548)
+- Generic HTTP connector now supports `queryParam` as a configurable section
+  in the secretless configuration file, under `config`. This allows the
+  construction of a query string which can have credentials injected
+  as needed. [#1290](https://github.com/cyberark/secretless-broker/issues/1290)
 
 ## [1.6.0] - 2020-05-04
 

README.md

@@ -5,6 +5,7 @@
 - [Quick Start](#quick-start)
 - [Additional demos](#run-more-secretless-demos)
 - [Using Secretless](#using-secretless)
+  - [Using This Project With Conjur-OSS](#using-secretless-broker-with-conjur-oss)
   - [About our releases](#about-our-releases)
 - [Community](#community)
 - [Performance](#performance)
@@ -140,6 +141,17 @@ For an even more in-depth demo, check out our [Deploying to Kubernetes](https://
 
 For complete documentation on using Secretless, please see [our documentation](https://docs.secretless.io/Latest/en/Content/Resources/_TopNav/cc_Home.htm). The documentation includes comprehensive guides for how to get up and running with Secretless.
 
+## Using secretless-broker with Conjur OSS
+
+Are you using this project with [Conjur OSS](https://github.com/cyberark/conjur)? 
+Then we **strongly** recommend choosing the version of this project to use from 
+the latest [Conjur OSS suite release](https://docs.conjur.org/Latest/en/Content/Overview/Conjur-OSS-Suite-Overview.html).
+Conjur maintainers perform additional testing on the suite release versions to ensure
+compatibility. When possible, upgrade your Conjur version to match the
+[latest suite release](https://docs.conjur.org/Latest/en/Content/ReleaseNotes/ConjurOSS-suite-RN.htm);
+when using integrations, choose the latest suite release that matches your Conjur version. 
+For any questions, please contact us on [Discourse](https://discuss.cyberarkcommons.org/c/conjur/5).
+
 ## About our releases
 
 ### Docker images

examples/generic_connector_configs/README.md

@@ -254,6 +254,7 @@ one should be used.
       Store the token from your request in your local credential manager so
       that it may be retrieved in your <code>secretless.yml</code>
     </li>
+    <li>Save the local token from Slack into the OSX keychain</li>
     <li>Run Secretless locally</li>
     <code>
     ./dist/darwin/amd64/secretless-broker \

examples/generic_connector_configs/stripe_secretless.yml

@@ -0,0 +1,90 @@
+version: 2
+services:
+  # The stripe service supports connecting to Stripe's API via a Bearer token.
+  # A Bearer token can be used
+  #
+  # More information about this service can be found here:
+  # https://stripe.com/docs/api/authentication
+  stripe:
+    connector: generic_http
+    listenOn: tcp://0.0.0.0:8071
+    credentials:
+      token:
+        from: keychain
+        get: service#stripe/token
+    config:
+      headers:
+        Authorization: Bearer {{ .token }}
+      forceSSL: true
+      authenticateURLsMatching:
+        - ^http[s]*\:\/\/api\.stripe\.com*
+  # The stripe-account service supports a Bearer token and a Stripe Account
+  # header. This service can be used if you want to securely connect a client's
+  # account to the Stripe API using Secretless.
+  #
+  # More information about this service can be found here:
+  # https://stripe.com/docs/api/connected_accounts
+  stripe-account:
+    connector: generic_http
+    listenOn: tcp://0.0.0.0:8081
+    credentials:
+      token:
+        from: keychain
+        get: service#stripe/token
+      stripe_account:
+        from: keychain
+        get: service#stripe/account-id
+    config:
+      headers:
+        Authorization: Bearer {{ .token }}
+        Stripe-Account: "{{ .stripe_account }}"
+      forceSSL: true
+      authenticateURLsMatching:
+        - ^http[s]*\:\/\/api\.stripe\.com*
+  # The stripe-idempotency service supports a Bearer token and an
+  # Indempotency-Key header. This is useful when an API call is disrupted in
+  # transit and you do not receive a response.
+  #
+  # More information about this service can be found here:
+  # https://stripe.com/docs/api/idempotent_requests
+  stripe-idempotency:
+    connector: generic_http
+    listenOn: tcp://0.0.0.0:8091
+    credentials:
+      token:
+        from: keychain
+        get: service#stripe/token
+      idempotency_key:
+        from: keychain
+        get: service#stripe/indempotency-key
+    config:
+      headers:
+        Authorization: Bearer {{ .token }}
+        Idempotency-Key: "{{ .idempotency_key }}"
+      forceSSL: true
+      authenticateURLsMatching:
+        - ^http[s]*\:\/\/api\.stripe\.com*
+  # The stripe-account-dempotency service supports a Bearer token, Stripe
+  # Account and Idempotency-Key header. This service can be used if an API call
+  # to a Stripe Account is disrupted in transit and does not receive a response.
+  stripe-account-idempotency:
+    connector: generic_http
+    listenOn: tcp://0.0.0.0:9001
+    credentials:
+      token:
+        from: keychain
+        get: service#stripe/token
+      stripe_account:
+        from: keychain
+        get: service#stripe/account-id
+      idempotency_key:
+        from: keychain
+        get: service#service/idempotency-key
+    config:
+      headers:
+        Authorization: Bearer {{ .token }}
+        Stripe-Account: "{{ .stripe_account }}"
+        Idempotency-Key: "{{ .idempotency_key }}"
+      forceSSL: true
+      authenticateURLsMatching:
+        - ^http[s]*\:\/\/api\.stripe\.com*

internal/plugin/connectors/http/generic/README.md

@@ -67,11 +67,16 @@ services:
       password:
         from: conjur
         get: somepassword
+      address:
+        from: conjur
+        get: address
     config:
       credentialValidations:
         username: '[^:]+'    # username cannot contain a colon
       headers:
         Authorization: "Basic {{ printf \"%s:%s\" .username .password | base64 }}"
+      queryParams:
+        location: "{{ .address }}"
       forceSSL: true
       authenticateURLsMatching:
         - ^http
@@ -103,6 +108,41 @@ powerful transformation features.  You can use `printf` for formatting and
 compose functions using pipes `|`.  See the text template package docs linked
 above for detailed information on these and other features.
 
+### `queryParams`
+
+Like `headers`, this is another key section. The `queryParams` section
+is used to generate a query string, which is appended to your existing URL
+without replacing any existing query parameters.
+
+The _keys_ of the queryParams are defined by the yaml keys.  In the examples
+above, the query parameter key is `location`.
+
+The query parameter _values_ are defined using a [Go text
+template](https://golang.org/pkg/text/template/), as defined in the
+`text/template` package.
+
+In the above example, let us say that your request URL looks like the following,
+
+```
+http://anything.com/foo?fruit=apple
+```
+
+After proxying through secretless, your request URL would look like the following,
+
+```
+http://anything.com/foo?location=valueofaddress&fruit=apple
+```
+
+You can refer to your credentials in this template using the credential name
+preceded by a `.` (eg, `.address` will refer to the credential
+`address`). At runtime, Secretless will replace these
+credential references with your real credentials.
+
+As you can see in the Basic auth example, the `text/template` package has
+powerful transformation features.  You can use `printf` for formatting and
+you can compose functions using pipes `|`.  See the text template package docs linked
+above for detailed information on these and other features.
+
 #### `credentialValidations`
 
 This section lets you use regular expressions to define validations for the

internal/plugin/connectors/http/generic/config.go

@@ -3,6 +3,7 @@ package generic
 import (
 	"encoding/base64"
 	"fmt"
+	"net/url"
 	"regexp"
 	"strings"
 	"text/template"
@@ -14,6 +15,7 @@ import (
 type config struct {
 	CredentialPatterns map[string]*regexp.Regexp
 	Headers            map[string]*template.Template
+	QueryParams        map[string]*template.Template
 	ForceSSL           bool
 }
 
@@ -34,34 +36,35 @@ func (c *config) validate(credsByID connector.CredentialValuesByID) error {
 	return nil
 }
 
-// renderedHeaders returns the config's header templates filled in with the
+// renderTemplates returns the config's templates filled in with the
 // given credentialValues.
-func (c *config) renderedHeaders(
+func renderTemplates(
+	template map[string]*template.Template,
 	credsByID connector.CredentialValuesByID,
 ) (map[string]string, error) {
 	errs := validation.Errors{}
-	headers := make(map[string]string)
+	args := make(map[string]string)
 
 	// Creds must be strings to work with templates
 	credStringsByID := make(map[string]string)
 	for credName, credBytes := range credsByID {
 		credStringsByID[credName] = string(credBytes)
 	}
 
-	for header, tmpl := range c.Headers {
+	for arg, tmpl := range template {
 		builder := &strings.Builder{}
 		if err := tmpl.Execute(builder, credStringsByID); err != nil {
-			errs[header] = fmt.Errorf("couldn't render template: %q", err)
+			errs[arg] = fmt.Errorf("couldn't render template: %q", err)
 			continue
 		}
-		headers[header] = builder.String()
+		args[arg] = builder.String()
 	}
 
 	if err := errs.Filter(); err != nil {
 		return nil, err
 	}
 
-	return headers, nil
+	return args, nil
 }
 
 // newConfig takes a ConfigYAML, validates it, and converts it into a
@@ -71,7 +74,6 @@ func newConfig(cfgYAML *ConfigYAML) (*config, error) {
 
 	cfg := &config{
 		CredentialPatterns: make(map[string]*regexp.Regexp),
-		Headers:            make(map[string]*template.Template),
 		ForceSSL:           cfgYAML.ForceSSL,
 	}
 
@@ -85,23 +87,45 @@ func newConfig(cfgYAML *ConfigYAML) (*config, error) {
 		cfg.CredentialPatterns[cred] = re
 	}
 
-	// Validate and save header template strings
-	for header, tmplStr := range cfgYAML.Headers {
-		tmpl := newHeaderTemplate(header)
+	cfg.Headers, errs = stringsToTemplates(cfgYAML.Headers, errs)
+	cfg.QueryParams, errs = stringsToTemplates(cfgYAML.QueryParams, errs)
+
+	if err := errs.Filter(); err != nil {
+		return nil, err
+	}
+
+	return cfg, nil
+}
+
+func stringsToTemplates(
+	templates map[string]string,
+	errs validation.Errors,
+) (map[string]*template.Template, validation.Errors) {
+	parsedTemplates := make(map[string]*template.Template)
+	// Validate and save template strings
+	for tmplName, tmplStr := range templates {
+		tmpl := newHTTPTemplate(tmplName)
 		// Ignore pointer to receiver returned by Parse(): it's just "tmpl".
 		_, err := tmpl.Parse(tmplStr)
 		if err != nil {
-			errs[header] = fmt.Errorf("invalid header template: %q", err)
+			errs[tmplName] = fmt.Errorf("invalid template: %q", err)
 			continue
 		}
-		cfg.Headers[header] = tmpl
+		parsedTemplates[tmplName] = tmpl
 	}
+	return parsedTemplates, errs
+}
 
-	if err := errs.Filter(); err != nil {
-		return nil, err
+func appendQueryParams(URL url.URL, params map[string]string) string {
+	query := url.Values{}
+	if len(URL.RawQuery) > 0 {
+		query = URL.Query()
+	}
+	for key, value := range params {
+		query.Add(key, value)
 	}
 
-	return cfg, nil
+	return query.Encode()
 }
 
 // templateFuncs is a map holding the custom functions available for use within
@@ -113,6 +137,6 @@ var templateFuncs = template.FuncMap{
 	},
 }
 
-func newHeaderTemplate(name string) *template.Template {
+func newHTTPTemplate(name string) *template.Template {
 	return template.New(name).Funcs(templateFuncs)
 }