From c9c435ddafc22158b0b6ce1cf30a83765924f2d6 Mon Sep 17 00:00:00 2001 From: Mitchell Dodell Date: Wed, 1 Jul 2020 14:48:54 -0700 Subject: [PATCH 1/5] Added a generic HTTP config for the Stripe API --- examples/generic_connector_configs/README.md | 36 ++++++++ .../stripe_secretless.yml | 90 +++++++++++++++++++ 2 files changed, 126 insertions(+) create mode 100644 examples/generic_connector_configs/stripe_secretless.yml diff --git a/examples/generic_connector_configs/README.md b/examples/generic_connector_configs/README.md index 4ae61bfe7..0a2098cd1 100644 --- a/examples/generic_connector_configs/README.md +++ b/examples/generic_connector_configs/README.md @@ -7,6 +7,7 @@ * [OAuth 2.0 API](#oauth-20-api) * [Slack Web API](#slack-web-api) * [Splunk API](#splunk-api) + * [Stripe API](#stripe-api) * [Contributing](#contributing) ## Introduction @@ -217,6 +218,41 @@ to the backend server uses SSL. ___ +### Stripe API +This example can be used to interact with [Stripe's API](https://stripe.com/docs/api). + +The configuration file for the Stripe API can be found at [stripe_secretless.yml](./stripe_secretless.yml). + +This example supports several header configurations, so it is recommended to +look at [stripe_secretless.yml](./stripe_secretless.yml) to figure out which +one should be used. + +#### How to use this connector +* Get the [Stripe API Key](https://dashboard.stripe.com/apikeys), which can be used as a Bearer token +* Get a [connected account](https://stripe.com/docs/connect/authentication) or generate an [idempotency key](https://stripe.com/docs/api/idempotent_requests) if needed +* Query the Striple API using `http_proxy=localhost:80*1 curl api.stripe.com/{route}`. + +#### Example Usage +
+ How to use this connector locally +
    +
  1. Get the Stripe test API Key
    2. +
  2. Save the local token from Slack into the OSX keychain
    3. +
  3. Run Secretless locally
    4. + + ./dist/darwin/amd64/secretless-broker \ +
    + -f examples/generic_connector_configs/stripe_secretless.yml +     +
  4. On another terminal window, make a request to Stripe using Secretless
    5. + + http_proxy=localhost:{secretless-server} curl api.stripe.com/v1/charges + +
+
+ +___ + ## Contributing Do you have an HTTP service that you use? Can you write a Secretless generic diff --git a/examples/generic_connector_configs/stripe_secretless.yml b/examples/generic_connector_configs/stripe_secretless.yml new file mode 100644 index 000000000..57fcd7d5a --- /dev/null +++ b/examples/generic_connector_configs/stripe_secretless.yml @@ -0,0 +1,90 @@ +version: 2 +services: + # The stripe service supports connecting to Stripe's API via a Bearer token. + # A Bearer token can be used + # + # More information about this service can be found here: + # https://stripe.com/docs/api/authentication + stripe: + connector: generic_http + listenOn: tcp://0.0.0.0:8071 + credentials: + token: + from: keychain + get: service#stripe/token + config: + headers: + Authorization: Bearer {{ .token }} + forceSSL: true + authenticateURLsMatching: + - ^http[s]*\:\/\/api\.stripe\.com* + # The stripe-account service supports a Bearer token and a Stripe Account + # header. This service can be used if you want to securely connect a client's + # account to the Stripe API using Secretless. + # + # More information about this service can be found here: + # https://stripe.com/docs/api/connected_accounts + stripe-account: + connector: generic_http + listenOn: tcp://0.0.0.0:8081 + credentials: + token: + from: keychain + get: service#stripe/token + stripe_account: + from: keychain + get: service#stripe/account-id + config: + headers: + Authorization: Bearer {{ .token }} + Stripe-Account: "{{ .stripe_account }}" + forceSSL: true + authenticateURLsMatching: + - ^http[s]*\:\/\/api\.stripe\.com* + # The stripe-idempotency service supports a Bearer token and an + # Indempotency-Key header. This is useful when an API call is disrupted in + # transit and you do not receive a response. + # + # More information about this service can be found here: + # https://stripe.com/docs/api/idempotent_requests + stripe-idempotency: + connector: generic_http + listenOn: tcp://0.0.0.0:8091 + credentials: + token: + from: keychain + get: service#stripe/token + idempotency_key: + from: keychain + get: service#stripe/indempotency-key + config: + headers: + Authorization: Bearer {{ .token }} + Idempotency-Key: "{{ .idempotency_key }}" + forceSSL: true + authenticateURLsMatching: + - ^http[s]*\:\/\/api\.stripe\.com* + # The stripe-account-dempotency service supports a Bearer token, Stripe + # Account and Idempotency-Key header. This service can be used if an API call + # to a Stripe Account is disrupted in transit and does not receive a response. + stripe-account-idempotency: + connector: generic_http + listenOn: tcp://0.0.0.0:9001 + credentials: + token: + from: keychain + get: service#stripe/token + stripe_account: + from: keychain + get: service#stripe/account-id + idempotency_key: + from: keychain + get: service#service/idempotency-key + config: + headers: + Authorization: Bearer {{ .token }} + Stripe-Account: "{{ .stripe_account }}" + Idempotency-Key: "{{ .idempotency_key }}" + forceSSL: true + authenticateURLsMatching: + - ^http[s]*\:\/\/api\.stripe\.com* From 26e0787ea3835173081d657a229d7b33c1bd0a1d Mon Sep 17 00:00:00 2001 From: Bradley Boutcher Date: Mon, 6 Jul 2020 15:09:07 -0400 Subject: [PATCH 2/5] Add Conjur OSS information to README --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index 0af804ffb..7895ced38 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,7 @@ - [Quick Start](#quick-start) - [Additional demos](#run-more-secretless-demos) - [Using Secretless](#using-secretless) + - [Using This Project With Conjur-OSS](#using-secretless-broker-with-conjur-oss) - [About our releases](#about-our-releases) - [Community](#community) - [Performance](#performance) @@ -140,6 +141,17 @@ For an even more in-depth demo, check out our [Deploying to Kubernetes](https:// For complete documentation on using Secretless, please see [our documentation](https://docs.secretless.io/Latest/en/Content/Resources/_TopNav/cc_Home.htm). The documentation includes comprehensive guides for how to get up and running with Secretless. +## Using secretless-broker with Conjur OSS + +Are you using this project with [Conjur OSS](https://github.com/cyberark/conjur)? +Then we **strongly** recommend choosing the version of this project to use from +the latest [Conjur OSS suite release](https://docs.conjur.org/Latest/en/Content/Overview/Conjur-OSS-Suite-Overview.html). +Conjur maintainers perform additional testing on the suite release versions to ensure +compatibility. When possible, upgrade your Conjur version to match the +[latest suite release](https://docs.conjur.org/Latest/en/Content/ReleaseNotes/ConjurOSS-suite-RN.htm); +when using integrations, choose the latest suite release that matches your Conjur version. +For any questions, please contact us on [Discourse](https://discuss.cyberarkcommons.org/c/conjur/5). + ## About our releases ### Docker images From 3d509a6fc5258d9c7c70610cd59cd1e156772f8f Mon Sep 17 00:00:00 2001 From: Bradley Boutcher Date: Fri, 26 Jun 2020 15:52:08 -0400 Subject: [PATCH 3/5] HTTP Generic Connector supports URL parameter injection - modified 'renderHeader' function to allow rendering of multiple input fields of the same type - Moved the validation and saving of template strings to its own function to reduce code reuse - Added support for passing query parameters through a secretless configuration file, and appending them to the url in the http request - Added unit tests for parsing of query params - Modified references to the URL struct in proxy_service.go that printed the host and query params, now only printing the host --- CHANGELOG.md | 4 + .../plugin/connectors/http/generic/README.md | 40 ++++++++++ .../plugin/connectors/http/generic/config.go | 58 ++++++++++---- .../connectors/http/generic/config_test.go | 80 ++++++++++++++++++- .../connectors/http/generic/connector.go | 9 ++- .../connectors/http/generic/external_api.go | 1 + .../plugin/connectors/http/proxy_service.go | 9 +-- 7 files changed, 174 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4ca6cb54e..0bd5a612f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - MySQL and PostgreSQL connectors support SSL host name verification with `verify-full` SSL mode. Also adds optional `sslhost` configuration parameter that is compared to the server's certificate SAN. [#548](https://github.com/cyberark/secretless-broker/issues/548) +- Generic HTTP connector now supports `queryParam` as a configurable section + in the secretless configuration file, under `config`. This allows the + construction of a query string which can have credentials injected + as needed. [#1290](https://github.com/cyberark/secretless-broker/issues/1290) ## [1.6.0] - 2020-05-04 diff --git a/internal/plugin/connectors/http/generic/README.md b/internal/plugin/connectors/http/generic/README.md index 70e907a1f..f4b0fd40b 100644 --- a/internal/plugin/connectors/http/generic/README.md +++ b/internal/plugin/connectors/http/generic/README.md @@ -67,11 +67,16 @@ services: password: from: conjur get: somepassword + address: + from: conjur + get: address config: credentialValidations: username: '[^:]+' # username cannot contain a colon headers: Authorization: "Basic {{ printf \"%s:%s\" .username .password | base64 }}" + queryParams: + location: "{{ .address }}" forceSSL: true authenticateURLsMatching: - ^http @@ -103,6 +108,41 @@ powerful transformation features. You can use `printf` for formatting and compose functions using pipes `|`. See the text template package docs linked above for detailed information on these and other features. +### `queryParams` + +Like `headers`, this is another key section. The `queryParams` section +is used to generate a query string, which is appended to your existing URL +without replacing any existing query parameters. + +The _keys_ of the queryParams are defined by the yaml keys. In the examples +above, these query parameter key is `location`. + +The query parameter _values_ are defined using a [Go text +template](https://golang.org/pkg/text/template/), as defined in the +`text/template` package. + +In the above example, let us say that your request URL looks like the following, + +``` +http://anything.com/foo?fruit=apple +``` + +After proxying through secretless, your request URL would look like the following, + +``` +http://anything.com/foo?location=valueofaddress&fruit=apple +``` + +You can refer to your credentials in this template using the credential name +preceded by a `.` (eg, `.address` will refer to the credential +`address`). At runtime, Secretless will replace these +credential references with your real credentials. + +As you can see in the Basic auth example, the `text/template` package has +powerful transformation features. You can use `printf` for formatting and +compose functions using pipes `|`. See the text template package docs linked +above for detailed information on these and other features. + #### `credentialValidations` This section lets you use regular expressions to define validations for the diff --git a/internal/plugin/connectors/http/generic/config.go b/internal/plugin/connectors/http/generic/config.go index 48574d040..1bbc34718 100644 --- a/internal/plugin/connectors/http/generic/config.go +++ b/internal/plugin/connectors/http/generic/config.go @@ -3,6 +3,7 @@ package generic import ( "encoding/base64" "fmt" + "net/url" "regexp" "strings" "text/template" @@ -14,6 +15,7 @@ import ( type config struct { CredentialPatterns map[string]*regexp.Regexp Headers map[string]*template.Template + QueryParams map[string]*template.Template ForceSSL bool } @@ -34,13 +36,14 @@ func (c *config) validate(credsByID connector.CredentialValuesByID) error { return nil } -// renderedHeaders returns the config's header templates filled in with the +// renderTemplates returns the config's templates filled in with the // given credentialValues. -func (c *config) renderedHeaders( +func renderTemplates( + template map[string]*template.Template, credsByID connector.CredentialValuesByID, ) (map[string]string, error) { errs := validation.Errors{} - headers := make(map[string]string) + args := make(map[string]string) // Creds must be strings to work with templates credStringsByID := make(map[string]string) @@ -48,20 +51,20 @@ func (c *config) renderedHeaders( credStringsByID[credName] = string(credBytes) } - for header, tmpl := range c.Headers { + for arg, tmpl := range template { builder := &strings.Builder{} if err := tmpl.Execute(builder, credStringsByID); err != nil { - errs[header] = fmt.Errorf("couldn't render template: %q", err) + errs[arg] = fmt.Errorf("couldn't render template: %q", err) continue } - headers[header] = builder.String() + args[arg] = builder.String() } if err := errs.Filter(); err != nil { return nil, err } - return headers, nil + return args, nil } // newConfig takes a ConfigYAML, validates it, and converts it into a @@ -71,7 +74,6 @@ func newConfig(cfgYAML *ConfigYAML) (*config, error) { cfg := &config{ CredentialPatterns: make(map[string]*regexp.Regexp), - Headers: make(map[string]*template.Template), ForceSSL: cfgYAML.ForceSSL, } @@ -85,23 +87,45 @@ func newConfig(cfgYAML *ConfigYAML) (*config, error) { cfg.CredentialPatterns[cred] = re } - // Validate and save header template strings - for header, tmplStr := range cfgYAML.Headers { - tmpl := newHeaderTemplate(header) + cfg.Headers, errs = stringsToTemplates(cfgYAML.Headers, errs) + cfg.QueryParams, errs = stringsToTemplates(cfgYAML.QueryParams, errs) + + if err := errs.Filter(); err != nil { + return nil, err + } + + return cfg, nil +} + +func stringsToTemplates( + templates map[string]string, + errs validation.Errors, +) (map[string]*template.Template, validation.Errors) { + parsedTemplates := make(map[string]*template.Template) + // Validate and save template strings + for tmplName, tmplStr := range templates { + tmpl := newHTTPTemplate(tmplName) // Ignore pointer to receiver returned by Parse(): it's just "tmpl". _, err := tmpl.Parse(tmplStr) if err != nil { - errs[header] = fmt.Errorf("invalid header template: %q", err) + errs[tmplName] = fmt.Errorf("invalid template: %q", err) continue } - cfg.Headers[header] = tmpl + parsedTemplates[tmplName] = tmpl } + return parsedTemplates, errs +} - if err := errs.Filter(); err != nil { - return nil, err +func appendQueryParams(URL url.URL, params map[string]string) string { + query := url.Values{} + if len(URL.RawQuery) > 0 { + query = URL.Query() + } + for key, value := range params { + query.Add(key, value) } - return cfg, nil + return query.Encode() } // templateFuncs is a map holding the custom functions available for use within @@ -113,6 +137,6 @@ var templateFuncs = template.FuncMap{ }, } -func newHeaderTemplate(name string) *template.Template { +func newHTTPTemplate(name string) *template.Template { return template.New(name).Funcs(templateFuncs) } diff --git a/internal/plugin/connectors/http/generic/config_test.go b/internal/plugin/connectors/http/generic/config_test.go index 1e71471a3..8ef190eba 100644 --- a/internal/plugin/connectors/http/generic/config_test.go +++ b/internal/plugin/connectors/http/generic/config_test.go @@ -1,6 +1,7 @@ package generic import ( + "net/url" "strings" "testing" @@ -16,6 +17,9 @@ func sampleConfigYAML() []byte { Authorization: 'Basic {{ printf "%s:%s" .username .password | base64 }}' Name-With-Dashes: '{{ .username }}' SimpleConcatenation: '{{ .username }} - {{ .password }}' + queryParams: + Key: '{{ .key | base64 }}' + NameWithSpaces: '{{ .name }}' forceSSL: true authenticateURLsMatching: - ^http @@ -42,10 +46,12 @@ func Test_newConfig(t *testing.T) { return } - headers, err := cfg.renderedHeaders(map[string][]byte{ - "username": []byte("Jonah"), - "password": []byte("secret"), - }) + headers, err := renderTemplates( + cfg.Headers, + map[string][]byte{ + "username": []byte("Jonah"), + "password": []byte("secret"), + }) assert.NoError(t, err) if err != nil { @@ -59,6 +65,32 @@ func Test_newConfig(t *testing.T) { // Assert against value calculated with independent base64 encoder. assert.Equal(t, "Basic Sm9uYWg6c2VjcmV0", headers["Authorization"]) }) + + t.Run("creates expected query params", func(t *testing.T) { + cfg, err := sampleConfig() + assert.NoError(t, err) + if err != nil { + return + } + + params, err := renderTemplates( + cfg.QueryParams, + map[string][]byte{ + "key": []byte("someKey"), + "name": []byte("Foo Bar"), + }) + + assert.NoError(t, err) + if err != nil { + return + } + + // A simple test case + assert.Equal(t, "Foo Bar", params["NameWithSpaces"]) + + // Assert against value calculated with independent base64 encoder. + assert.Equal(t, "c29tZUtleQ==", params["Key"]) + }) } func Test_validate(t *testing.T) { @@ -112,3 +144,43 @@ func Test_validate(t *testing.T) { }) } } + +func Test_appendQueryParams(t *testing.T) { + testCases := []struct { + description string + params map[string]string + url url.URL + expectedRawQuery string + }{ + { + description: "appends params without replacing existing params", + params: map[string]string{ + "foo": "bar", + }, + url: url.URL{ + RawQuery: "some=place", + }, + expectedRawQuery: "foo=bar&some=place", + }, + { + description: "special characters encode properly", + params: map[string]string{ + "space": "bar biz", + "key": "abc==", + "special": "@#$%^&*(){}[].,+", + }, + url: url.URL{ + RawQuery: "some=place", + }, + expectedRawQuery: "key=abc%3D%3D&some=place&space=bar+biz&special=%40%23%24" + + "%25%5E%26%2A%28%29%7B%7D%5B%5D.%2C%2B", + }, + } + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + actualRawQuery := appendQueryParams(tc.url, tc.params) + + assert.Equal(t, tc.expectedRawQuery, actualRawQuery) + }) + } +} diff --git a/internal/plugin/connectors/http/generic/connector.go b/internal/plugin/connectors/http/generic/connector.go index a25869c5e..ab4a37df1 100644 --- a/internal/plugin/connectors/http/generic/connector.go +++ b/internal/plugin/connectors/http/generic/connector.go @@ -30,7 +30,7 @@ func (c *Connector) Connect( } // Add configured headers to request - headers, err := c.config.renderedHeaders(credentialsByID) + headers, err := renderTemplates(c.config.Headers, credentialsByID) if err != nil { return fmt.Errorf("failed to render headers: %s", err) } @@ -38,5 +38,12 @@ func (c *Connector) Connect( r.Header.Set(headerName, headerVal) } + // Add configured params to request + params, err := renderTemplates(c.config.QueryParams, credentialsByID) + if err != nil { + return fmt.Errorf("failed to render query params: %s", err) + } + r.URL.RawQuery = appendQueryParams(*r.URL, params) + return nil } diff --git a/internal/plugin/connectors/http/generic/external_api.go b/internal/plugin/connectors/http/generic/external_api.go index 287fd245d..7ec04088b 100644 --- a/internal/plugin/connectors/http/generic/external_api.go +++ b/internal/plugin/connectors/http/generic/external_api.go @@ -20,6 +20,7 @@ import ( type ConfigYAML struct { CredentialValidations map[string]string `yaml:"credentialValidations"` Headers map[string]string `yaml:"headers"` + QueryParams map[string]string `yaml:"queryParams"` ForceSSL bool `yaml:"forceSSL"` } diff --git a/internal/plugin/connectors/http/proxy_service.go b/internal/plugin/connectors/http/proxy_service.go index 38ca3ad50..5176b5f1d 100644 --- a/internal/plugin/connectors/http/proxy_service.go +++ b/internal/plugin/connectors/http/proxy_service.go @@ -136,20 +136,20 @@ func (proxy *proxyService) selectSubservice(r *gohttp.Request) *Subservice { // No match: Warn! if len(matchingSubs) == 0 { msg := "No subservices matched request '%s'" - proxy.logger.Warnf(msg, r.URL.String()) + proxy.logger.Warnf(msg, r.URL.Host) return nil } // Multiple matches: Warn! if len(matchingSubs) > 1 { msg := "Multiple subservices matched request '%s': %v\n" - proxy.logger.Warnf(msg, r.URL.String(), matchingSubs) + proxy.logger.Warnf(msg, r.URL.Host, matchingSubs) } // Select first (or only) match subservice := matchingSubs[0] msg := "Using connector '%s' for request %s" - proxy.logger.Debugf(msg, subservice.ConnectorID, r.URL.String()) + proxy.logger.Debugf(msg, subservice.ConnectorID, r.URL.Host) return &subservice } @@ -158,9 +158,8 @@ func (proxy *proxyService) ServeHTTP(w gohttp.ResponseWriter, r *gohttp.Request) logger := proxy.logger // Log request - logMsg := "Got request %v %v %v %v" - logger.Debugf(logMsg, r.URL.Path, r.Host, r.Method, r.URL.String()) + logger.Debugf(logMsg, r.URL.Path, r.Host, r.Method, r.URL.Hostname()) // Validate request From 544237d245a747db0b61bcba54a5631636535aa2 Mon Sep 17 00:00:00 2001 From: Bradley Boutcher Date: Tue, 7 Jul 2020 12:33:44 -0400 Subject: [PATCH 4/5] Update internal/plugin/connectors/http/generic/README.md Co-authored-by: Geri Jennings --- internal/plugin/connectors/http/generic/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/plugin/connectors/http/generic/README.md b/internal/plugin/connectors/http/generic/README.md index f4b0fd40b..7760ba0b8 100644 --- a/internal/plugin/connectors/http/generic/README.md +++ b/internal/plugin/connectors/http/generic/README.md @@ -115,7 +115,7 @@ is used to generate a query string, which is appended to your existing URL without replacing any existing query parameters. The _keys_ of the queryParams are defined by the yaml keys. In the examples -above, these query parameter key is `location`. +above, the query parameter key is `location`. The query parameter _values_ are defined using a [Go text template](https://golang.org/pkg/text/template/), as defined in the From 79e23afda3098ed497eaacd5b02982603aa59b1e Mon Sep 17 00:00:00 2001 From: Bradley Boutcher Date: Tue, 7 Jul 2020 12:33:58 -0400 Subject: [PATCH 5/5] Update internal/plugin/connectors/http/generic/README.md Co-authored-by: Geri Jennings --- internal/plugin/connectors/http/generic/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/plugin/connectors/http/generic/README.md b/internal/plugin/connectors/http/generic/README.md index 7760ba0b8..2ffd478aa 100644 --- a/internal/plugin/connectors/http/generic/README.md +++ b/internal/plugin/connectors/http/generic/README.md @@ -140,7 +140,7 @@ credential references with your real credentials. As you can see in the Basic auth example, the `text/template` package has powerful transformation features. You can use `printf` for formatting and -compose functions using pipes `|`. See the text template package docs linked +you can compose functions using pipes `|`. See the text template package docs linked above for detailed information on these and other features. #### `credentialValidations`