From f75f48da789c1719b81e3cbebfcd7039b8b3e230 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20=C3=81lvarez?= Date: Thu, 5 Dec 2019 13:28:06 +0100 Subject: [PATCH 01/36] Add apikey subcommand --- _meta/beat.yml | 5 +- apm-server.docker.yml | 5 +- apm-server.yml | 5 +- beater/api/mux.go | 12 +- beater/authorization/allow.go | 4 +- beater/authorization/apikey.go | 89 ++--- beater/authorization/apikey_test.go | 26 +- beater/authorization/bearer.go | 4 +- beater/authorization/builder.go | 17 +- beater/authorization/builder_test.go | 15 +- beater/authorization/deny.go | 4 +- beater/authorization/privilege.go | 28 +- beater/authorization/privilege_test.go | 8 +- beater/beater.go | 1 + beater/config/api_key.go | 4 +- beater/config/config.go | 31 +- beater/config/config_test.go | 33 +- beater/middleware/authorization_middleware.go | 3 +- cmd/apikey.go | 326 ++++++++++++++++++ cmd/root.go | 224 +++++++++++- elasticsearch/client.go | 143 ++++++-- elasticsearch/security_api.go | 220 ++++++++++++ x-pack/apm-server/cmd/root.go | 2 +- 23 files changed, 1030 insertions(+), 179 deletions(-) create mode 100644 cmd/apikey.go create mode 100644 elasticsearch/security_api.go diff --git a/_meta/beat.yml b/_meta/beat.yml index d9511fb8917..bd87dea8c2e 100644 --- a/_meta/beat.yml +++ b/_meta/beat.yml @@ -153,7 +153,10 @@ apm-server: #protocol: "http" - # Optional HTTP Path. + #username: "elastic" + #password: "changeme" + + # Optional HTTP Path. #path: "" # Proxy server url. diff --git a/apm-server.docker.yml b/apm-server.docker.yml index ebca44674f4..419b940f95a 100644 --- a/apm-server.docker.yml +++ b/apm-server.docker.yml @@ -153,7 +153,10 @@ apm-server: #protocol: "http" - # Optional HTTP Path. + #username: "elastic" + #password: "changeme" + + # Optional HTTP Path. #path: "" # Proxy server url. diff --git a/apm-server.yml b/apm-server.yml index 8dff2ae9554..c367b5f940a 100644 --- a/apm-server.yml +++ b/apm-server.yml @@ -153,7 +153,10 @@ apm-server: #protocol: "http" - # Optional HTTP Path. + #username: "elastic" + #password: "changeme" + + # Optional HTTP Path. #path: "" # Proxy server url. diff --git a/beater/api/mux.go b/beater/api/mux.go index 1c42ed933e3..f52e2eb4c77 100644 --- a/beater/api/mux.go +++ b/beater/api/mux.go @@ -83,7 +83,7 @@ func NewMux(beaterConfig *config.Config, report publish.Reporter) (*http.ServeMu mux := http.NewServeMux() logger := logp.NewLogger(logs.Handler) - auth, err := authorization.NewBuilder(beaterConfig) + auth, err := authorization.NewBuilder(*beaterConfig) if err != nil { return nil, err } @@ -125,7 +125,7 @@ func NewMux(beaterConfig *config.Config, report publish.Reporter) (*http.ServeMu func profileHandler(cfg *config.Config, builder *authorization.Builder, reporter publish.Reporter) (request.Handler, error) { h := profile.Handler(systemMetadataDecoder(cfg, emptyDecoder), transform.Config{}, reporter) - authHandler := builder.ForPrivilege(authorization.PrivilegeEventWrite) + authHandler := builder.ForPrivilege(authorization.PrivilegeEventWrite.Action) return middleware.Wrap(h, backendMiddleware(cfg, authHandler, profile.MonitoringMap)...) } @@ -137,7 +137,7 @@ func backendIntakeHandler(cfg *config.Config, builder *authorization.Builder, re MaxEventSize: cfg.MaxEventSize, }, reporter) - authHandler := builder.ForPrivilege(authorization.PrivilegeEventWrite) + authHandler := builder.ForPrivilege(authorization.PrivilegeEventWrite.Action) return middleware.Wrap(h, backendMiddleware(cfg, authHandler, intake.MonitoringMap)...) } @@ -162,12 +162,12 @@ func sourcemapHandler(cfg *config.Config, builder *authorization.Builder, report return nil, err } h := sourcemap.Handler(systemMetadataDecoder(cfg, decoder.DecodeSourcemapFormData), psourcemap.Processor, *tcfg, reporter) - authHandler := builder.ForPrivilege(authorization.PrivilegeSourcemapWrite) + authHandler := builder.ForPrivilege(authorization.PrivilegeSourcemapWrite.Action) return middleware.Wrap(h, sourcemapMiddleware(cfg, authHandler)...) } func backendAgentConfigHandler(cfg *config.Config, builder *authorization.Builder, _ publish.Reporter) (request.Handler, error) { - authHandler := builder.ForPrivilege(authorization.PrivilegeAgentConfigRead) + authHandler := builder.ForPrivilege(authorization.PrivilegeAgentConfigRead.Action) return agentConfigHandler(cfg, authHandler, backendMiddleware) } @@ -193,7 +193,7 @@ func agentConfigHandler(cfg *config.Config, authHandler *authorization.Handler, func rootHandler(cfg *config.Config, builder *authorization.Builder, _ publish.Reporter) (request.Handler, error) { return middleware.Wrap(root.Handler(), - rootMiddleware(cfg, builder.ForAnyOfPrivileges(authorization.PrivilegesAll))...) + rootMiddleware(cfg, builder.ForAnyOfPrivileges(authorization.ActionAny))...) } func apmMiddleware(m map[request.ResultID]*monitoring.Int) []middleware.Middleware { diff --git a/beater/authorization/allow.go b/beater/authorization/allow.go index 1a3bba19aa0..56ea3460296 100644 --- a/beater/authorization/allow.go +++ b/beater/authorization/allow.go @@ -17,11 +17,13 @@ package authorization +import "github.com/elastic/apm-server/elasticsearch" + // AllowAuth implements the Authorization interface. It allows all authorization requests. type AllowAuth struct{} // AuthorizedFor always returns true -func (AllowAuth) AuthorizedFor(_ string) (bool, error) { +func (AllowAuth) AuthorizedFor(_ elasticsearch.Resource) (bool, error) { return true, nil } diff --git a/beater/authorization/apikey.go b/beater/authorization/apikey.go index 997f409d9f8..d05010f9865 100644 --- a/beater/authorization/apikey.go +++ b/beater/authorization/apikey.go @@ -18,43 +18,39 @@ package authorization import ( - "encoding/json" - "net/http" - "strings" + "fmt" "time" "github.com/pkg/errors" - "github.com/elastic/apm-server/beater/headers" - "github.com/elastic/apm-server/elasticsearch" + es "github.com/elastic/apm-server/elasticsearch" ) -const ( - //DefaultResource for apm backend enabled API Keys - DefaultResource = "-" +const cleanupInterval = 60 * time.Second - application = "apm" - sep = `","` - - cleanupInterval = 60 * time.Second +var ( + // Constant mapped to the "application" field for the Elasticsearch security API + // This identifies privileges and keys created for APM + Application = es.AppName("apm") + // Only valid for first authorization of a request. + // The API Key needs to grant privileges to additional resources for successful processing of requests. + ResourceInternal = es.Resource("-") + ResourceAny = es.Resource("*") ) type apikeyBuilder struct { - esClient elasticsearch.Client + esClient es.Client cache *privilegesCache - anyOfPrivileges []string + anyOfPrivileges []es.Privilege } type apikeyAuth struct { *apikeyBuilder + // key is base64(id:apiKey) key string } -type hasPrivilegesResponse struct { - Applications map[string]map[string]privileges `json:"application"` -} - -func newApikeyBuilder(client elasticsearch.Client, cache *privilegesCache, anyOfPrivileges []string) *apikeyBuilder { +func newApikeyBuilder(client es.Client, cache *privilegesCache, anyOfPrivileges []es.Privilege) *apikeyBuilder { return &apikeyBuilder{client, cache, anyOfPrivileges} } @@ -69,12 +65,8 @@ func (a *apikeyAuth) IsAuthorizationConfigured() bool { // AuthorizedFor checks if the configured api key is authorized. // An api key is considered to be authorized when the api key has the configured privileges for the requested resource. -// Privileges are fetched from Elasticsearch and then cached in a global cache. -func (a *apikeyAuth) AuthorizedFor(resource string) (bool, error) { - if resource == "" { - resource = DefaultResource - } - +// PrivilegeGroup are fetched from Elasticsearch and then cached in a global cache. +func (a *apikeyAuth) AuthorizedFor(resource es.Resource) (bool, error) { //fetch from cache if allowed, found := a.fromCache(resource); found { return allowed, nil @@ -98,7 +90,7 @@ func (a *apikeyAuth) AuthorizedFor(resource string) (bool, error) { return allowed, nil } -func (a *apikeyAuth) fromCache(resource string) (allowed bool, found bool) { +func (a *apikeyAuth) fromCache(resource es.Resource) (allowed bool, found bool) { privileges := a.cache.get(id(a.key, resource)) if privileges == nil { return @@ -114,43 +106,28 @@ func (a *apikeyAuth) fromCache(resource string) (allowed bool, found bool) { return } -func (a *apikeyAuth) queryES(resource string) (privileges, error) { - query := buildQuery(PrivilegesAll, resource) - statusCode, body, err := a.esClient.SecurityHasPrivilegesRequest(strings.NewReader(query), - http.Header{headers.Authorization: []string{headers.APIKey + " " + a.key}}) - if err != nil { - return nil, err - } - defer body.Close() - if statusCode != http.StatusOK { - // return nil privileges for queried apps to ensure they are cached - return privileges{}, nil +func (a *apikeyAuth) queryES(resource es.Resource) (es.Perms, error) { + request := es.HasPrivilegesRequest{ + Applications: []es.Application{ + { + Name: Application, + Privileges: a.anyOfPrivileges, + Resources: []es.Resource{resource}, + }, + }, } - - var decodedResponse hasPrivilegesResponse - if err := json.NewDecoder(body).Decode(&decodedResponse); err != nil { + info, err := es.HasPrivileges(a.esClient, request, a.key) + if err != nil { return nil, err } - if resources, ok := decodedResponse.Applications[application]; ok { + if resources, ok := info.Application[Application]; ok { if privileges, ok := resources[resource]; ok { return privileges, nil } } - return privileges{}, nil -} - -func buildQuery(privileges []string, resource string) string { - var b strings.Builder - b.WriteString(`{"application":[{"application":"`) - b.WriteString(application) - b.WriteString(`","privileges":["`) - b.WriteString(strings.Join(privileges, sep)) - b.WriteString(`"],"resources":"`) - b.WriteString(resource) - b.WriteString(`"}]}`) - return b.String() + return es.Perms{}, nil } -func id(apiKey, resource string) string { - return apiKey + "_" + resource +func id(apiKey string, resource es.Resource) string { + return apiKey + "_" + fmt.Sprintf("%v", resource) } diff --git a/beater/authorization/apikey_test.go b/beater/authorization/apikey_test.go index 4d9fa19aec3..649e306414c 100644 --- a/beater/authorization/apikey_test.go +++ b/beater/authorization/apikey_test.go @@ -41,11 +41,11 @@ func TestApikeyBuilder(t *testing.T) { handler2 := tc.builder.forKey(key) // add existing privileges to shared cache - privilegesValid := privileges{} + privilegesValid := elasticsearch.Perms{} for _, p := range PrivilegesAll { - privilegesValid[p] = true + privilegesValid[p.Action] = true } - resource := "service-go" + resource := elasticsearch.Resource("service-go") tc.cache.add(id(key, resource), privilegesValid) // check that cache is actually shared between apiKeyHandlers @@ -85,12 +85,12 @@ func TestAPIKey_AuthorizedFor(t *testing.T) { tc.setup(t) key := "" handler := tc.builder.forKey(key) - resourceValid := "foo" - resourceInvalid := "bar" - resourceMissing := "missing" + resourceValid := elasticsearch.Resource("foo") + resourceInvalid := elasticsearch.Resource("bar") + resourceMissing := elasticsearch.Resource("missing") - tc.cache.add(id(key, resourceValid), privileges{tc.anyOfPrivileges[0]: true}) - tc.cache.add(id(key, resourceInvalid), privileges{tc.anyOfPrivileges[0]: false}) + tc.cache.add(id(key, resourceValid), elasticsearch.Perms{tc.anyOfPrivileges[0]: true}) + tc.cache.add(id(key, resourceInvalid), elasticsearch.Perms{tc.anyOfPrivileges[0]: false}) valid, err := handler.AuthorizedFor(resourceValid) require.NoError(t, err) @@ -163,7 +163,7 @@ type apikeyTestcase struct { transport *estest.Transport client elasticsearch.Client cache *privilegesCache - anyOfPrivileges []string + anyOfPrivileges []elasticsearch.Privilege builder *apikeyBuilder } @@ -174,9 +174,9 @@ func (tc *apikeyTestcase) setup(t *testing.T) { if tc.transport == nil { tc.transport = estest.NewTransport(t, http.StatusOK, map[string]interface{}{ "application": map[string]interface{}{ - application: map[string]privileges{ - "foo": {PrivilegeAgentConfigRead: true, PrivilegeEventWrite: true, PrivilegeSourcemapWrite: false}, - "bar": {PrivilegeAgentConfigRead: true, PrivilegeEventWrite: false}, + "application": map[string]map[string]interface{}{ + "foo": {"agentconfig": true, "event": true, "sourcemap": false}, + "bar": {"agentConfig": true, "event": false}, }}}) } tc.client, err = estest.NewElasticsearchClient(tc.transport) @@ -186,7 +186,7 @@ func (tc *apikeyTestcase) setup(t *testing.T) { tc.cache = newPrivilegesCache(time.Millisecond, 5) } if tc.anyOfPrivileges == nil { - tc.anyOfPrivileges = []string{PrivilegeEventWrite, PrivilegeSourcemapWrite} + tc.anyOfPrivileges = []elasticsearch.Privilege{PrivilegeEventWrite.Action, PrivilegeSourcemapWrite.Action} } tc.builder = newApikeyBuilder(tc.client, tc.cache, tc.anyOfPrivileges) } diff --git a/beater/authorization/bearer.go b/beater/authorization/bearer.go index c3e09aa90dc..92249e2fa02 100644 --- a/beater/authorization/bearer.go +++ b/beater/authorization/bearer.go @@ -19,6 +19,8 @@ package authorization import ( "crypto/subtle" + + "github.com/elastic/apm-server/elasticsearch" ) type bearerBuilder struct { @@ -39,7 +41,7 @@ func (b bearerBuilder) forToken(token string) *bearerAuth { configured: true} } -func (b *bearerAuth) AuthorizedFor(_ string) (bool, error) { +func (b *bearerAuth) AuthorizedFor(_ elasticsearch.Resource) (bool, error) { return b.authorized, nil } diff --git a/beater/authorization/builder.go b/beater/authorization/builder.go index 2a043c77eb6..ad637ab762a 100644 --- a/beater/authorization/builder.go +++ b/beater/authorization/builder.go @@ -37,7 +37,7 @@ type Handler Builder // Authorization interface to be implemented by different auth types type Authorization interface { - AuthorizedFor(string) (bool, error) + AuthorizedFor(_ elasticsearch.Resource) (bool, error) IsAuthorizationConfigured() bool } @@ -46,10 +46,15 @@ const ( ) // NewBuilder creates authorization builder based off of the given information -func NewBuilder(cfg *config.Config) (*Builder, error) { +// if apm-server.api_key is enabled, authorization is granted/denied solely +// based on the request Authorization header +func NewBuilder(cfg config.Config) (*Builder, error) { b := Builder{} b.fallback = AllowAuth{} if cfg.APIKeyConfig.IsEnabled() { + // do not use username+password for API Key requests + cfg.APIKeyConfig.ESConfig.Username = "" + cfg.APIKeyConfig.ESConfig.Password = "" client, err := elasticsearch.NewClient(cfg.APIKeyConfig.ESConfig) if err != nil { return nil, err @@ -57,7 +62,7 @@ func NewBuilder(cfg *config.Config) (*Builder, error) { size := cfg.APIKeyConfig.LimitMin * cacheTimeoutMinute cache := newPrivilegesCache(cacheTimeoutMinute*time.Minute, size) - b.apikey = newApikeyBuilder(client, cache, []string{}) + b.apikey = newApikeyBuilder(client, cache, []elasticsearch.Privilege{}) b.fallback = DenyAuth{} } if cfg.SecretToken != "" { @@ -69,12 +74,12 @@ func NewBuilder(cfg *config.Config) (*Builder, error) { } // ForPrivilege creates an authorization Handler checking for this privilege -func (b *Builder) ForPrivilege(privilege string) *Handler { - return b.ForAnyOfPrivileges([]string{privilege}) +func (b *Builder) ForPrivilege(privilege elasticsearch.Privilege) *Handler { + return b.ForAnyOfPrivileges(privilege) } // ForAnyOfPrivileges creates an authorization Handler checking for any of the provided privileges -func (b *Builder) ForAnyOfPrivileges(privileges []string) *Handler { +func (b *Builder) ForAnyOfPrivileges(privileges ...elasticsearch.Privilege) *Handler { handler := Handler{bearer: b.bearer, fallback: b.fallback} if b.apikey != nil { handler.apikey = newApikeyBuilder(b.apikey.esClient, b.apikey.cache, privileges) diff --git a/beater/authorization/builder_test.go b/beater/authorization/builder_test.go index 0a896e85558..edf4d57db9d 100644 --- a/beater/authorization/builder_test.go +++ b/beater/authorization/builder_test.go @@ -41,8 +41,9 @@ func TestBuilder(t *testing.T) { } { setup := func() *Builder { - cfg := config.DefaultConfig("9.9.9") - + rawCfg := config.DefaultConfig("9.9.9") + cfg, err := config.Setup(nil, rawCfg, nil) + require.NoError(t, err) if tc.withBearer { cfg.SecretToken = "xvz" } @@ -51,7 +52,7 @@ func TestBuilder(t *testing.T) { Enabled: true, LimitMin: 100, ESConfig: elasticsearch.DefaultConfig()} } - builder, err := NewBuilder(cfg) + builder, err := NewBuilder(*cfg) require.NoError(t, err) return builder } @@ -68,12 +69,12 @@ func TestBuilder(t *testing.T) { t.Run("ForPrivilege"+name, func(t *testing.T) { builder := setup() - h := builder.ForPrivilege(PrivilegeSourcemapWrite) + h := builder.ForPrivilege(PrivilegeSourcemapWrite.Action) assert.Equal(t, builder.bearer, h.bearer) assert.Equal(t, builder.fallback, h.fallback) if tc.withApikey { - assert.Equal(t, []string{}, builder.apikey.anyOfPrivileges) - assert.Equal(t, []string{PrivilegeSourcemapWrite}, h.apikey.anyOfPrivileges) + assert.Equal(t, []elasticsearch.Privilege{}, builder.apikey.anyOfPrivileges) + assert.Equal(t, []elasticsearch.Privilege{PrivilegeSourcemapWrite.Action}, h.apikey.anyOfPrivileges) assert.Equal(t, builder.apikey.esClient, h.apikey.esClient) assert.Equal(t, builder.apikey.cache, h.apikey.cache) } @@ -81,7 +82,7 @@ func TestBuilder(t *testing.T) { t.Run("AuthorizationFor"+name, func(t *testing.T) { builder := setup() - h := builder.ForPrivilege(PrivilegeSourcemapWrite) + h := builder.ForPrivilege(PrivilegeSourcemapWrite.Action) auth := h.AuthorizationFor("ApiKey", "") if tc.withApikey { assert.IsType(t, &apikeyAuth{}, auth) diff --git a/beater/authorization/deny.go b/beater/authorization/deny.go index c9ec2ef2cb0..985226cb51a 100644 --- a/beater/authorization/deny.go +++ b/beater/authorization/deny.go @@ -17,11 +17,13 @@ package authorization +import "github.com/elastic/apm-server/elasticsearch" + // DenyAuth implements the Authorization interface. It denies all authorization requests. type DenyAuth struct{} // AuthorizedFor always returns false -func (DenyAuth) AuthorizedFor(_ string) (bool, error) { +func (DenyAuth) AuthorizedFor(_ elasticsearch.Resource) (bool, error) { return false, nil } diff --git a/beater/authorization/privilege.go b/beater/authorization/privilege.go index 7cc874e0bea..eea676ebc9c 100644 --- a/beater/authorization/privilege.go +++ b/beater/authorization/privilege.go @@ -20,22 +20,18 @@ package authorization import ( "time" - "github.com/patrickmn/go-cache" -) + es "github.com/elastic/apm-server/elasticsearch" -//Privileges -const ( - PrivilegeAgentConfigRead = "config_agent:read" - PrivilegeEventWrite = "event:write" - PrivilegeSourcemapWrite = "sourcemap:write" + "github.com/patrickmn/go-cache" ) var ( - //PrivilegesAll returns all available privileges - PrivilegesAll = []string{ - PrivilegeAgentConfigRead, - PrivilegeEventWrite, - PrivilegeSourcemapWrite} + PrivilegeAgentConfigRead = es.NewPrivilege("agentConfig", "config_agent:read") + PrivilegeEventWrite = es.NewPrivilege("event", "event:write") + PrivilegeSourcemapWrite = es.NewPrivilege("sourcemap", "sourcemap:write") + PrivilegesAll = []es.NamedPrivilege{PrivilegeAgentConfigRead, PrivilegeEventWrite, PrivilegeSourcemapWrite} + // ActionAny can't be used for querying + ActionAny = es.Privilege("*") ) type privilegesCache struct { @@ -43,8 +39,6 @@ type privilegesCache struct { size int } -type privileges map[string]bool - func newPrivilegesCache(expiration time.Duration, size int) *privilegesCache { return &privilegesCache{cache: cache.New(expiration, cleanupInterval), size: size} } @@ -53,13 +47,13 @@ func (c *privilegesCache) isFull() bool { return c.cache.ItemCount() >= c.size } -func (c *privilegesCache) get(id string) privileges { +func (c *privilegesCache) get(id string) es.Perms { if val, exists := c.cache.Get(id); exists { - return val.(privileges) + return val.(es.Perms) } return nil } -func (c *privilegesCache) add(id string, privileges privileges) { +func (c *privilegesCache) add(id string, privileges es.Perms) { c.cache.SetDefault(id, privileges) } diff --git a/beater/authorization/privilege_test.go b/beater/authorization/privilege_test.go index 81484d3df1b..f5849adff08 100644 --- a/beater/authorization/privilege_test.go +++ b/beater/authorization/privilege_test.go @@ -21,6 +21,8 @@ import ( "testing" "time" + "github.com/elastic/apm-server/elasticsearch" + "github.com/stretchr/testify/assert" ) @@ -29,16 +31,16 @@ func TestPrivilegesCache(t *testing.T) { cache := newPrivilegesCache(time.Millisecond, n) assert.False(t, cache.isFull()) for i := 0; i < n-1; i++ { - cache.add(string(i), privileges{}) + cache.add(string(i), elasticsearch.Perms{}) assert.False(t, cache.isFull()) } - cache.add("oneMore", privileges{}) + cache.add("oneMore", elasticsearch.Perms{}) assert.True(t, cache.isFull()) assert.NotNil(t, cache.get("oneMore")) time.Sleep(time.Millisecond) assert.Nil(t, cache.get("oneMore")) - p := privileges{"a": true, "b": false} + p := elasticsearch.Perms{"a": true, "b": false} cache.add("id1", p) assert.Equal(t, p, cache.get("id1")) assert.Nil(t, cache.get("oneMore")) diff --git a/beater/beater.go b/beater/beater.go index d6cba45ee10..0504083c325 100644 --- a/beater/beater.go +++ b/beater/beater.go @@ -95,6 +95,7 @@ func New(b *beat.Beat, ucfg *common.Config) (beat.Beater, error) { if isElasticsearchOutput(b) { esOutputCfg = b.Config.Output.Config() } + beaterConfig, err := config.NewConfig(b.Info.Version, ucfg, esOutputCfg) if err != nil { return nil, err diff --git a/beater/config/api_key.go b/beater/config/api_key.go index 0c26c743493..3e4ed840589 100644 --- a/beater/config/api_key.go +++ b/beater/config/api_key.go @@ -53,9 +53,7 @@ func (c *APIKeyConfig) setup(log *logp.Logger, outputESCfg *common.Config) error if err := outputESCfg.Unpack(c.ESConfig); err != nil { return errors.Wrap(err, "unpacking Elasticsearch config into API key config") } - // do not use username+password for API Key requests - c.ESConfig.Username = "" - c.ESConfig.Password = "" + return nil } diff --git a/beater/config/config.go b/beater/config/config.go index 98cce679dcd..4321c3932d5 100644 --- a/beater/config/config.go +++ b/beater/config/config.go @@ -68,6 +68,8 @@ type Config struct { Pipeline string } +type RawConfig Config + // ExpvarConfig holds config information about exposing expvar type ExpvarConfig struct { Enabled *bool `config:"enabled"` @@ -84,9 +86,8 @@ type Cache struct { Expiration time.Duration `config:"expiration"` } -// NewConfig creates a Config struct based on the default config and the given input params -func NewConfig(version string, ucfg *common.Config, outputESCfg *common.Config) (*Config, error) { - logger := logp.NewLogger(logs.Config) +// NewRawConfig unpacks the ucfg data +func NewRawConfig(version string, ucfg *common.Config) (*RawConfig, error) { c := DefaultConfig(version) if ucfg.HasField("ssl") { @@ -104,6 +105,16 @@ func NewConfig(version string, ucfg *common.Config, outputESCfg *common.Config) if err := ucfg.Unpack(c); err != nil { return nil, errors.Wrap(err, "Error processing configuration") } + return c, nil +} + +// Setup contains ad-hoc logic for apm-server configuration +func Setup(rawConfig *RawConfig, outputESCfg *common.Config) (*Config, error) { + logger := logp.NewLogger(logs.Config) + if rawConfig == nil { + return nil, nil + } + c := Config(*rawConfig) if float64(int(c.AgentConfig.Cache.Expiration.Seconds())) != c.AgentConfig.Cache.Expiration.Seconds() { return nil, errors.New(msgInvalidConfigAgentCfg) @@ -125,7 +136,15 @@ func NewConfig(version string, ucfg *common.Config, outputESCfg *common.Config) return nil, err } - return c, nil + return &c, nil +} + +func NewConfig(version string, ucfg *common.Config, outputESCfg *common.Config) (*Config, error) { + raw, err := NewRawConfig(version, ucfg) + if err != nil { + return nil, err + } + return Setup(raw, outputESCfg) } // IsEnabled indicates whether expvar is enabled or not @@ -134,8 +153,8 @@ func (c *ExpvarConfig) IsEnabled() bool { } // DefaultConfig returns a config with default settings for `apm-server` config options. -func DefaultConfig(beatVersion string) *Config { - return &Config{ +func DefaultConfig(beatVersion string) *RawConfig { + return &RawConfig{ Host: net.JoinHostPort("localhost", DefaultPort), MaxHeaderSize: 1 * 1024 * 1024, // 1mb MaxConnections: 0, // unlimited diff --git a/beater/config/config_test.go b/beater/config/config_test.go index 867b411ef8b..cc79951fc2a 100644 --- a/beater/config/config_test.go +++ b/beater/config/config_test.go @@ -37,14 +37,15 @@ import ( func Test_UnpackConfig(t *testing.T) { falsy, truthy := false, true version := "8.0.0" - + defaultRawConfig := DefaultConfig(version) + defaultConfig, _ := Setup(nil, defaultRawConfig, nil) tests := map[string]struct { inpCfg map[string]interface{} outCfg *Config }{ "default config": { inpCfg: map[string]interface{}{}, - outCfg: DefaultConfig(version), + outCfg: defaultConfig, }, "overwrite default": { inpCfg: map[string]interface{}{ @@ -239,7 +240,8 @@ func Test_UnpackConfig(t *testing.T) { inpCfg, err := common.NewConfigFrom(test.inpCfg) assert.NoError(t, err) - cfg, err := NewConfig(version, inpCfg, nil) + rawCfg, err := NewRawConfig(version, inpCfg) + cfg, err := Setup(nil, rawCfg, err) require.NoError(t, err) require.NotNil(t, cfg) assert.Equal(t, test.outCfg, cfg) @@ -320,7 +322,7 @@ func TestTLSSettings(t *testing.T) { ucfgCfg, err := common.NewConfigFrom(tc.config) require.NoError(t, err) - cfg, err := NewConfig("9.9.9", ucfgCfg, nil) + cfg, err := NewRawConfig("9.9.9", ucfgCfg) require.NoError(t, err) assert.Equal(t, tc.tls.ClientAuth, cfg.TLS.ClientAuth) }) @@ -346,7 +348,7 @@ func TestTLSSettings(t *testing.T) { ucfgCfg, err := common.NewConfigFrom(tc.config) require.NoError(t, err) - cfg, err := NewConfig("9.9.9", ucfgCfg, nil) + cfg, err := NewRawConfig("9.9.9", ucfgCfg) require.NoError(t, err) assert.Equal(t, tc.tls.VerificationMode, cfg.TLS.VerificationMode) }) @@ -378,22 +380,25 @@ func TestTLSSettings(t *testing.T) { func TestAgentConfig(t *testing.T) { t.Run("InvalidValueTooSmall", func(t *testing.T) { - cfg, err := NewConfig("9.9.9", - common.MustNewConfigFrom(map[string]string{"agent.config.cache.expiration": "123ms"}), nil) + rawCfg, err := NewRawConfig("9.9.9", + common.MustNewConfigFrom(map[string]string{"agent.config.cache.expiration": "123ms"})) + cfg, err := Setup(nil, rawCfg, err) require.Error(t, err) assert.Nil(t, cfg) }) t.Run("InvalidUnit", func(t *testing.T) { - cfg, err := NewConfig("9.9.9", - common.MustNewConfigFrom(map[string]string{"agent.config.cache.expiration": "1230ms"}), nil) + rawCfg, err := NewRawConfig("9.9.9", + common.MustNewConfigFrom(map[string]string{"agent.config.cache.expiration": "1230ms"})) + cfg, err := Setup(nil, rawCfg, err) require.Error(t, err) assert.Nil(t, cfg) }) t.Run("Valid", func(t *testing.T) { - cfg, err := NewConfig("9.9.9", - common.MustNewConfigFrom(map[string]string{"agent.config.cache.expiration": "123000ms"}), nil) + rawCfg, err := NewRawConfig("9.9.9", + common.MustNewConfigFrom(map[string]string{"agent.config.cache.expiration": "123000ms"})) + cfg, err := Setup(nil, rawCfg, err) require.NoError(t, err) assert.Equal(t, time.Second*123, cfg.AgentConfig.Cache.Expiration) }) @@ -405,14 +410,16 @@ func TestNewConfig_ESConfig(t *testing.T) { require.NoError(t, err) // no es config given - cfg, err := NewConfig(version, ucfg, nil) + rawCfg, err := NewRawConfig(version, ucfg) + cfg, err := Setup(nil, rawCfg, err) require.NoError(t, err) assert.Nil(t, cfg.RumConfig.SourceMapping.ESConfig) assert.Equal(t, elasticsearch.DefaultConfig(), cfg.APIKeyConfig.ESConfig) // with es config outputESCfg := common.MustNewConfigFrom(`{"hosts":["192.0.0.168:9200"]}`) - cfg, err = NewConfig(version, ucfg, outputESCfg) + rawCfg, err = NewRawConfig(version, ucfg) + cfg, err = Setup(outputESCfg, rawCfg, err) require.NoError(t, err) assert.NotNil(t, cfg.RumConfig.SourceMapping.ESConfig) assert.Equal(t, []string{"192.0.0.168:9200"}, []string(cfg.RumConfig.SourceMapping.ESConfig.Hosts)) diff --git a/beater/middleware/authorization_middleware.go b/beater/middleware/authorization_middleware.go index a694cfa56d0..75a5a5751e9 100644 --- a/beater/middleware/authorization_middleware.go +++ b/beater/middleware/authorization_middleware.go @@ -28,13 +28,12 @@ import ( // AuthorizationMiddleware returns a Middleware to only let authorized requests pass through func AuthorizationMiddleware(auth *authorization.Handler, apply bool) Middleware { - resource := authorization.DefaultResource return func(h request.Handler) (request.Handler, error) { return func(c *request.Context) { c.Authorization = auth.AuthorizationFor(fetchAuthHeader(c.Request)) if apply { - authorized, err := c.Authorization.AuthorizedFor(resource) + authorized, err := c.Authorization.AuthorizedFor(authorization.ResourceInternal) if !authorized { c.Result.SetDeniedAuthorization(err) c.Write() diff --git a/cmd/apikey.go b/cmd/apikey.go new file mode 100644 index 00000000000..26e3d134ea6 --- /dev/null +++ b/cmd/apikey.go @@ -0,0 +1,326 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package cmd + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "math" + "os" + "strings" + "time" + + "github.com/pkg/errors" + + "github.com/elastic/apm-server/beater/config" + "github.com/elastic/apm-server/beater/headers" + + auth "github.com/elastic/apm-server/beater/authorization" + es "github.com/elastic/apm-server/elasticsearch" +) + +// creates an API Key with the given privileges, *AND* all the privileges modeled in apm-server +// we need to ensure forward-compatibility, for which future privileges must be created here and +// during server startup because we don't know if customers will run this command +func createApiKeyWithPrivileges(client es.Client, apikeyName, expiry string, privileges []es.Privilege, asJson bool) error { + var privilegesRequest = make(es.CreatePrivilegesRequest) + event := auth.PrivilegeEventWrite + agentConfig := auth.PrivilegeAgentConfigRead + sourcemap := auth.PrivilegeSourcemapWrite + privilegesRequest[auth.Application] = map[es.PrivilegeName]es.Actions{ + agentConfig.Name: {[]es.Privilege{agentConfig.Action}}, + event.Name: {[]es.Privilege{event.Action}}, + sourcemap.Name: {[]es.Privilege{sourcemap.Action}}, + } + + privilegesCreated, err := es.CreatePrivileges(client, privilegesRequest) + + if err != nil { + return printErr(err, + `Error creating privileges for APM Server, do you have the "manage_cluster" security privilege?`, + asJson) + } + + printText, printJson := printers(asJson) + for privilege, result := range privilegesCreated[auth.Application] { + if result.Created { + printText("Security privilege \"%v\" created", privilege) + } + } + + apikeyRequest := es.CreateApiKeyRequest{ + Name: apikeyName, + RoleDescriptors: es.RoleDescriptor{ + auth.Application: es.Applications{ + Applications: []es.Application{ + { + Name: auth.Application, + Privileges: privileges, + Resources: []es.Resource{auth.ResourceAny}, + }, + }, + }, + }, + } + if expiry != "" { + apikeyRequest.Expiration = &expiry + } + + apikey, err := es.CreateAPIKey(client, apikeyRequest) + if err != nil { + return printErr(err, fmt.Sprintf( + `Error creating the API Key %s, do you have the "manage_cluster" security privilege?`, apikeyName), + asJson) + } + credentials := base64.StdEncoding.EncodeToString([]byte(apikey.Id + ":" + apikey.Key)) + apikey.Credentials = &credentials + printText("API Key created:") + printText("") + printText("Name ........... %s", apikey.Name) + printText("Expiration ..... %s", humanTime(apikey.ExpirationMs)) + printText("Id ............. %s", apikey.Id) + printText("API Key ........ %s (won't be shown again)", apikey.Key) + printText(`Credentials .... %s (use it as "Authorization: ApiKey " header to communicate with APM Server, won't be shown again)`, + credentials) + + return printJson(struct { + es.CreateApiKeyResponse + Privileges es.CreatePrivilegesResponse `json:"created_privileges,omitempty"` + }{ + CreateApiKeyResponse: apikey, + Privileges: privilegesCreated, + }) +} + +func getApiKey(client es.Client, id, name *string, validOnly, asJson bool) error { + if id != nil { + name = nil + } else if id == nil && name == nil { + return printErr(errors.New("could not query Elasticsearch"), + `either "id" or "name" are required`, + asJson) + } + request := es.GetApiKeyRequest{ + ApiKeyQuery: es.ApiKeyQuery{ + Id: id, + Name: name, + }, + } + + apikeys, err := es.GetAPIKeys(client, request) + if err != nil { + return printErr(err, + `Error retrieving API Key(s) for APM Server, do you have the "manage_cluster" security privilege?`, + asJson) + } + + transform := es.GetApiKeyResponse{ApiKeys: make([]es.ApiKeyResponse, 0)} + printText, printJson := printers(asJson) + for _, apikey := range apikeys.ApiKeys { + expiry := humanTime(apikey.ExpirationMs) + if validOnly && (apikey.Invalidated || expiry == "expired") { + continue + } + creation := time.Unix(apikey.Creation/1000, 0).Format("2006-02-01 15:04") + printText("Username ....... %s", apikey.Username) + printText("Api Key Name ... %s", apikey.Name) + printText("Id ............. %s", apikey.Id) + printText("Creation ....... %s", creation) + printText("Invalidated .... %t", apikey.Invalidated) + printText("Expiration ..... %s", expiry) + printText("") + transform.ApiKeys = append(transform.ApiKeys, apikey) + } + printText("%d API Keys found", len(transform.ApiKeys)) + return printJson(transform) +} + +func invalidateApiKey(client es.Client, id, name *string, deletePrivileges, asJson bool) error { + if id != nil { + name = nil + } else if id == nil && name == nil { + return printErr(errors.New("could not query Elasticsearch"), + `either "id" or "name" are required`, + asJson) + } + invalidateKeysRequest := es.InvalidateApiKeyRequest{ + ApiKeyQuery: es.ApiKeyQuery{ + Id: id, + Name: name, + }, + } + + invalidation, err := es.InvalidateAPIKey(client, invalidateKeysRequest) + if err != nil { + return printErr(err, + `Error invalidating API Key(s), do you have the "manage_cluster" security privilege?`, + asJson) + } + printText, printJson := printers(asJson) + out := struct { + es.InvalidateApiKeyResponse + Privileges []es.DeletePrivilegeResponse `json:"deleted_privileges,omitempty"` + }{ + InvalidateApiKeyResponse: invalidation, + Privileges: make([]es.DeletePrivilegeResponse, 0), + } + printText("Invalidated keys ... %s", strings.Join(invalidation.Invalidated, ", ")) + printText("Error count ........ %d", invalidation.ErrorCount) + + for _, privilege := range auth.PrivilegesAll { + if !deletePrivileges { + break + } + deletePrivilegesRequest := es.DeletePrivilegeRequest{ + Application: auth.Application, + Privilege: privilege.Name, + } + + deletion, err := es.DeletePrivileges(client, deletePrivilegesRequest) + if err != nil { + continue + } + if _, ok := deletion[auth.Application]; !ok { + continue + } + if result, ok := deletion[auth.Application][privilege.Name]; ok && result.Found { + printText("Deleted privilege \"%v\"", privilege) + } + out.Privileges = append(out.Privileges, deletion) + } + return printJson(out) +} + +func verifyApiKey(config *config.Config, privileges []es.Privilege, credentials string, asJson bool) error { + builder, err := auth.NewBuilder(*config) + perms := make(es.Perms) + + printText, printJson := printers(asJson) + + for _, privilege := range privileges { + if err != nil { + break + } + var authorized bool + authorized, err = builder. + ForPrivilege(privilege). + AuthorizationFor(headers.APIKey, credentials). + AuthorizedFor(auth.ResourceInternal) + perms[privilege] = authorized + printText("Authorized for %s...: %s\n", humanPrivilege(privilege), humanBool(authorized)) + } + + if err != nil { + return printErr(err, "could not verify credentials, please check your Elasticsearch connection", asJson) + } + return printJson(perms) +} + +func humanBool(b bool) string { + if b { + return "Yes" + } + return "No" +} + +func humanPrivilege(privilege es.Privilege) string { + switch privilege { + case auth.ActionAny: + return fmt.Sprintf("all privileges (\"%v\")", privilege) + default: + return fmt.Sprintf("privilege \"%v\"", privilege) + } +} + +func humanTime(millis *int64) string { + if millis == nil { + return fmt.Sprint("never") + } + seconds := time.Until(time.Unix(*millis/1000, 0)).Seconds() + if seconds < 0 { + return fmt.Sprintf("expired") + } + minutes := math.Round(seconds / 60) + if minutes < 2 { + return fmt.Sprintf("%.0f seconds", seconds) + } + hours := math.Round(minutes / 60) + if hours < 2 { + return fmt.Sprintf("%.0f minutes", minutes) + } + days := math.Round(hours / 24) + if days < 2 { + return fmt.Sprintf("%.0f hours", hours) + } + years := math.Round(days / 365) + if years < 2 { + return fmt.Sprintf("%.0f days", days) + } + return fmt.Sprintf("%.0f years", years) +} + +// returns 2 printers, one for text and one for JSON +// one of them will be a noop based on the boolean argument +func printers(b bool) (func(string, ...interface{}), func(interface{}) error) { + var w1 io.Writer = os.Stdout + var w2 = ioutil.Discard + if b { + w1 = ioutil.Discard + w2 = os.Stdout + } + return func(f string, i ...interface{}) { + fmt.Fprintf(w1, f, i...) + fmt.Fprintln(w1) + }, func(i interface{}) error { + data, err := json.MarshalIndent(i, "", "\t") + fmt.Fprintln(w2, string(data)) + // conform the interface + return errors.Wrap(err, fmt.Sprintf("%v+", i)) + } +} + +// prints an Elasticsearch error to stderr, with some additional contextual information as a hint +func printErr(err error, help string, asJson bool) error { + if asJson { + var data []byte + var m map[string]interface{} + e := json.Unmarshal([]byte(err.Error()), &m) + if e == nil { + // err.Error() has JSON shape, likely coming from Elasticsearch + m["help"] = help + data, _ = json.MarshalIndent(m, "", "\t") + } else { + // err.Error() is a bare string, likely coming from apm-server + data, _ = json.MarshalIndent(struct { + Error string `json:"error"` + Help string `json:"help"` + }{ + Error: err.Error(), + Help: help, + }, "", "\t") + } + fmt.Fprintln(os.Stderr, string(data)) + } else { + fmt.Fprintln(os.Stderr, help) + fmt.Fprintln(os.Stderr, err.Error()) + } + return errors.Wrap(err, help) +} diff --git a/cmd/root.go b/cmd/root.go index 57871de4124..05211d48e46 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -20,6 +20,13 @@ package cmd import ( "fmt" + "github.com/spf13/cobra" + + auth "github.com/elastic/apm-server/beater/authorization" + + "github.com/elastic/apm-server/beater/config" + es "github.com/elastic/apm-server/elasticsearch" + "github.com/elastic/beats/libbeat/cfgfile" "github.com/spf13/pflag" @@ -43,7 +50,11 @@ const IdxPattern = "apm" // RootCmd for running apm-server. // This is the command that is used if no other command is specified. // Running `apm-server run` or `apm-server` is identical. -var RootCmd *cmd.BeatsRootCmd +type ApmCmd struct { + *cmd.BeatsRootCmd +} + +var RootCmd = ApmCmd{} func init() { overrides := common.MustNewConfigFrom(map[string]interface{}{ @@ -92,8 +103,8 @@ func init() { }, }, } - RootCmd = cmd.GenRootCmdWithSettings(beater.New, settings) - + RootCmd = ApmCmd{cmd.GenRootCmdWithSettings(beater.New, settings)} + RootCmd.AddCommand(genApikeyCmd(settings)) for _, cmd := range RootCmd.ExportCmd.Commands() { // remove `dashboard` from `export` commands @@ -126,3 +137,210 @@ func init() { setup.Flags().Bool(cmd.PipelineKey, false, "Setup ingest pipelines") } + +func genApikeyCmd(settings instance.Settings) *cobra.Command { + + var client es.Client + apmConfig, err := bootstrap(settings) + if err == nil { + client, err = es.NewClient(apmConfig.APIKeyConfig.ESConfig) + } + + short := "Manage API Keys for communication between APM agents and server" + apikeyCmd := cobra.Command{ + Use: "apikey", + Short: short, + Long: short + `. +Most operations require the "manage_security" cluster privilege. Ensure to configure "apm-server.api_key.*" or +"output.elasticsearch.*" appropriately. APM Server will create security privileges for the "apm" application; +you can freely query them. If you modify or delete apm privileges, APM Server might reject all requests. +If an invalid argument is passed, nothing will be printed. +Check the Elastic Security API documentation for details.`, + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + return err + }, + } + + apikeyCmd.AddCommand( + createApikeyCmd(client), + invalidateApikeyCmd(client), + getApikeysCmd(client), + verifyApikeyCmd(apmConfig), + ) + + return &apikeyCmd +} + +func createApikeyCmd(client es.Client) *cobra.Command { + var keyName, expiration string + var ingest, sourcemap, agentConfig, json bool + short := "Create an API Key with the specified privilege(s)" + create := &cobra.Command{ + Use: "create", + Short: short, + Long: short + `. +If no privilege(s) are specified, the API Key will be valid for all. +Requires the "manage_security" cluster privilege in Elasticsearch.`, + // always need to return error for possible scripts checking the exit code, + // but printing the error must be done inside + RunE: func(cmd *cobra.Command, args []string) error { + privileges := booleansToPrivileges(ingest, sourcemap, agentConfig) + return createApiKeyWithPrivileges(client, keyName, expiration, privileges, json) + }, + // these are needed to not break JSON formatting + // this has the caveat that if an invalid argument is passed, the command won't return anything + SilenceUsage: true, + SilenceErrors: true, + } + create.Flags().StringVar(&keyName, "name", "apm-key", "API Key name") + create.Flags().StringVar(&expiration, "expiration", "", + `expiration for the key, eg. "1d" (default never)`) + create.Flags().BoolVar(&ingest, "ingest", false, + fmt.Sprintf("give the %v privilege to this key, required for ingesting events", auth.PrivilegeEventWrite)) + create.Flags().BoolVar(&sourcemap, "sourcemap", false, + fmt.Sprintf("give the %v privilege to this key, required for uploading sourcemaps", + auth.PrivilegeSourcemapWrite)) + create.Flags().BoolVar(&agentConfig, "agent-config", false, + fmt.Sprintf("give the %v privilege to this key, required for agents to read configuration remotely", + auth.PrivilegeAgentConfigRead)) + create.Flags().BoolVar(&json, "json", false, + "prints the output of this command as JSON") + // this actually means "preserve sorting given in code" and not reorder them alphabetically + create.Flags().SortFlags = false + return create +} + +func invalidateApikeyCmd(client es.Client) *cobra.Command { + var id, name string + var purge, json bool + short := "Invalidate API Key(s) by Id or Name" + invalidate := &cobra.Command{ + Use: "invalidate", + Short: short, + Long: short + `. +If both "id" and "name" are supplied, only "id" will be used. +If neither of them are, an error will be returned. +Requires the "manage_security" cluster privilege in Elasticsearch.`, + RunE: func(cmd *cobra.Command, args []string) error { + return invalidateApiKey(client, &id, &name, purge, json) + }, + SilenceErrors: true, + SilenceUsage: true, + } + invalidate.Flags().StringVar(&id, "id", "", "id of the API Key to delete") + invalidate.Flags().StringVar(&name, "name", "", + "name of the API Key(s) to delete (several might match)") + invalidate.Flags().BoolVar(&purge, "purge", false, + "also remove all privileges created and used by APM Server") + invalidate.Flags().BoolVar(&json, "json", false, + "prints the output of this command as JSON") + invalidate.Flags().SortFlags = false + return invalidate +} + +func getApikeysCmd(client es.Client) *cobra.Command { + var id, name string + var validOnly, json bool + short := "Query API Key(s) by Id or Name" + info := &cobra.Command{ + Use: "info", + Short: short, + Long: short + `. +If both "id" and "name" are supplied, only "id" will be used. +If neither of them are, an error will be returned. +Requires the "manage_security" cluster privilege in Elasticsearch.`, + RunE: func(cmd *cobra.Command, args []string) error { + return getApiKey(client, &id, &name, validOnly, json) + }, + SilenceErrors: true, + SilenceUsage: true, + } + info.Flags().StringVar(&id, "id", "", "id of the API Key to query") + info.Flags().StringVar(&name, "name", "", + "name of the API Key(s) to query (several might match)") + info.Flags().BoolVar(&validOnly, "valid-only", false, + "only return valid API Keys (not expired or invalidated)") + info.Flags().BoolVar(&json, "json", false, + "prints the output of this command as JSON") + info.Flags().SortFlags = false + return info +} + +func verifyApikeyCmd(config *config.Config) *cobra.Command { + var credentials string + var ingest, sourcemap, agentConfig, json bool + short := `Check if a "credentials" string has the given privilege(s)` + long := short + `. +If no privilege(s) are specified, the credentials will be queried for all.` + verify := &cobra.Command{ + Use: "verify", + Short: short, + Long: long, + RunE: func(cmd *cobra.Command, args []string) error { + privileges := booleansToPrivileges(ingest, sourcemap, agentConfig) + return verifyApiKey(config, privileges, credentials, json) + }, + SilenceUsage: true, + SilenceErrors: true, + } + verify.Flags().StringVar(&credentials, "credentials", "", `credentials for which check privileges`) + verify.Flags().BoolVar(&ingest, "ingest", false, + fmt.Sprintf("ask for the %v privilege, required for ingesting events", auth.PrivilegeEventWrite)) + verify.Flags().BoolVar(&sourcemap, "sourcemap", false, + fmt.Sprintf("ask for the %v privilege, required for uploading sourcemaps", + auth.PrivilegeSourcemapWrite)) + verify.Flags().BoolVar(&agentConfig, "agent-config", false, + fmt.Sprintf("ask for the %v privilege, required for agents to read configuration remotely", + auth.PrivilegeAgentConfigRead)) + verify.Flags().BoolVar(&json, "json", false, + "prints the output of this command as JSON") + verify.Flags().SortFlags = false + + return verify +} + +// created the beat, instantiate configuration, and so on +// apm-server.api_key.enabled is implicitly true +func bootstrap(settings instance.Settings) (*config.Config, error) { + beat, err := instance.NewBeat(settings.Name, settings.IndexPrefix, settings.Version) + if err != nil { + return nil, err + } + err = beat.InitWithSettings(settings) + if err != nil { + return nil, err + } + + outCfg := beat.Config.Output + apmRawConfig, err := config.NewRawConfig(settings.Version, beat.RawConfig) + if apmRawConfig == nil { + return nil, err + } + if apmRawConfig.APIKeyConfig == nil { + apmRawConfig.APIKeyConfig = &config.APIKeyConfig{} + } + apmRawConfig.APIKeyConfig.Enabled = true + + return config.Setup(apmRawConfig, outCfg.Config()) +} + +// if all are false, returns any ("*") +// this is because Elasticsearch requires at least 1 privilege for most queries, +// so "*" acts as default +func booleansToPrivileges(ingest, sourcemap, agentConfig bool) []es.Privilege { + privileges := make([]es.Privilege, 0) + if ingest { + privileges = append(privileges, auth.PrivilegeEventWrite.Action) + } + if sourcemap { + privileges = append(privileges, auth.PrivilegeSourcemapWrite.Action) + } + if agentConfig { + privileges = append(privileges, auth.PrivilegeAgentConfigRead.Action) + } + any := ingest || sourcemap || agentConfig + if !any { + privileges = append(privileges, auth.ActionAny) + } + return privileges +} diff --git a/elasticsearch/client.go b/elasticsearch/client.go index eefd4abbf9f..1c920137272 100644 --- a/elasticsearch/client.go +++ b/elasticsearch/client.go @@ -18,81 +18,88 @@ package elasticsearch import ( + "bytes" "context" + "encoding/json" + "errors" "io" + "io/ioutil" "net/http" + "net/url" + "strings" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/version" v7 "github.com/elastic/go-elasticsearch/v7" - v7esapi "github.com/elastic/go-elasticsearch/v7/esapi" v8 "github.com/elastic/go-elasticsearch/v8" - v8esapi "github.com/elastic/go-elasticsearch/v8/esapi" ) // Client is an interface designed to abstract away version differences between elasticsearch clients type Client interface { + // TODO: deprecate // Search performs a query against the given index with the given body Search(index string, body io.Reader) (int, io.ReadCloser, error) - SecurityHasPrivilegesRequest(body io.Reader, header http.Header) (int, io.ReadCloser, error) + // Makes a request with application/json Content-Type and Accept headers by default + // pass/overwrite headers with "key: value" format + JSONRequest(method, path string, body interface{}, headers ...string) JSONResponse } type clientV8 struct { - client *v8.Client + c *v8.Client } // Search satisfies the Client interface for version 8 -func (c clientV8) Search(index string, body io.Reader) (int, io.ReadCloser, error) { - return v8Response(c.client.Search( - c.client.Search.WithContext(context.Background()), - c.client.Search.WithIndex(index), - c.client.Search.WithBody(body), - c.client.Search.WithTrackTotalHits(true), - c.client.Search.WithPretty(), - )) -} - -func (c clientV8) SecurityHasPrivilegesRequest(body io.Reader, header http.Header) (int, io.ReadCloser, error) { - hasPrivileges := v8esapi.SecurityHasPrivilegesRequest{Body: body, Header: header} - return v8Response(hasPrivileges.Do(context.Background(), c.client)) -} - -func v8Response(response *v8esapi.Response, err error) (int, io.ReadCloser, error) { +func (v8 clientV8) Search(index string, body io.Reader) (int, io.ReadCloser, error) { + response, err := v8.c.Search( + v8.c.Search.WithContext(context.Background()), + v8.c.Search.WithIndex(index), + v8.c.Search.WithBody(body), + v8.c.Search.WithTrackTotalHits(true), + v8.c.Search.WithPretty(), + ) if err != nil { return 0, nil, err } return response.StatusCode, response.Body, nil } -type clientV7 struct { - client *v7.Client -} - -// Search satisfies the Client interface for version 7 -func (c clientV7) Search(index string, body io.Reader) (int, io.ReadCloser, error) { - return v7Response(c.client.Search( - c.client.Search.WithContext(context.Background()), - c.client.Search.WithIndex(index), - c.client.Search.WithBody(body), - c.client.Search.WithTrackTotalHits(true), - c.client.Search.WithPretty(), - )) +func (v8 clientV8) JSONRequest(method, path string, body interface{}, headers ...string) JSONResponse { + req, err := makeRequest(method, path, body, headers...) + if err != nil { + return JSONResponse{nil, err} + } + return parseResponse(v8.c.Perform(req)) } -func (c clientV7) SecurityHasPrivilegesRequest(body io.Reader, header http.Header) (int, io.ReadCloser, error) { - hasPrivileges := v7esapi.SecurityHasPrivilegesRequest{Body: body, Header: header} - return v7Response(hasPrivileges.Do(context.Background(), c.client)) +type clientV7 struct { + c *v7.Client } -func v7Response(response *v7esapi.Response, err error) (int, io.ReadCloser, error) { +// Search satisfies the Client interface for version 7 +func (v7 clientV7) Search(index string, body io.Reader) (int, io.ReadCloser, error) { + response, err := v7.c.Search( + v7.c.Search.WithContext(context.Background()), + v7.c.Search.WithIndex(index), + v7.c.Search.WithBody(body), + v7.c.Search.WithTrackTotalHits(true), + v7.c.Search.WithPretty(), + ) if err != nil { return 0, nil, err } return response.StatusCode, response.Body, nil } +func (v7 clientV7) JSONRequest(method, path string, body interface{}, headers ...string) JSONResponse { + req, err := makeRequest(method, path, body, headers...) + if err != nil { + return JSONResponse{nil, err} + } + return parseResponse(v7.c.Perform(req)) +} + // NewClient parses the given config and returns a version-aware client as an interface func NewClient(config *Config) (Client, error) { if config == nil { @@ -135,3 +142,65 @@ func newV8Client(apikey, user, pwd string, addresses []string, transport http.Ro Transport: transport, }) } + +type JSONResponse struct { + content io.ReadCloser + err error +} + +func (r JSONResponse) DecodeTo(i interface{}) error { + if r.err != nil { + return r.err + } + bs, err := ioutil.ReadAll(r.content) + if err != nil { + return err + } + err = json.Unmarshal(bs, i) + return err +} + +// each header has the format "key: value" +func makeRequest(method, path string, body interface{}, headers ...string) (*http.Request, error) { + header := http.Header{ + "Content-Type": []string{"application/json"}, + "Accept": []string{"application/json"}, + } + for _, h := range headers { + kv := strings.Split(h, ":") + if len(kv) == 2 { + header[kv[0]] = strings.Split(kv[1], ",") + } + } + u, _ := url.Parse(path) + req := &http.Request{ + Method: method, + URL: u, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Header: header, + } + bs, err := json.Marshal(body) + if err != nil { + return nil, err + } + if body != nil { + req.Body = ioutil.NopCloser(bytes.NewReader(bs)) + req.ContentLength = int64(len(bs)) + } + return req, nil +} + +func parseResponse(resp *http.Response, err error) JSONResponse { + if err != nil { + return JSONResponse{nil, err} + } + body := resp.Body + if resp.StatusCode >= http.StatusMultipleChoices { + buf := new(bytes.Buffer) + buf.ReadFrom(body) + return JSONResponse{nil, errors.New(buf.String())} + } + return JSONResponse{body, nil} +} diff --git a/elasticsearch/security_api.go b/elasticsearch/security_api.go new file mode 100644 index 00000000000..2b9d0dbe369 --- /dev/null +++ b/elasticsearch/security_api.go @@ -0,0 +1,220 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package elasticsearch + +import ( + "fmt" + "net/http" + "net/url" + "strconv" +) + +// requires manage_security cluster privilege +func CreateAPIKey(client Client, apikeyReq CreateApiKeyRequest) (CreateApiKeyResponse, error) { + response := client.JSONRequest(http.MethodPut, "/_security/api_key", apikeyReq) + + var apikey CreateApiKeyResponse + err := response.DecodeTo(&apikey) + return apikey, err +} + +// requires manage_security cluster privilege +func GetAPIKeys(client Client, apikeyReq GetApiKeyRequest) (GetApiKeyResponse, error) { + u := url.URL{Path: "/_security/api_key"} + params := url.Values{} + params.Set("owner", strconv.FormatBool(apikeyReq.Owner)) + if apikeyReq.Id != nil { + params.Set("id", *apikeyReq.Id) + } else if apikeyReq.Name != nil { + params.Set("name", *apikeyReq.Name) + } + u.RawQuery = params.Encode() + + response := client.JSONRequest(http.MethodGet, u.String(), nil) + + var apikey GetApiKeyResponse + err := response.DecodeTo(&apikey) + return apikey, err +} + +// requires manage_security cluster privilege +func CreatePrivileges(client Client, privilegesReq CreatePrivilegesRequest) (CreatePrivilegesResponse, error) { + response := client.JSONRequest(http.MethodPut, "/_security/privilege", privilegesReq) + + var privileges CreatePrivilegesResponse + err := response.DecodeTo(&privileges) + return privileges, err +} + +// requires manage_security cluster privilege +func InvalidateAPIKey(client Client, apikeyReq InvalidateApiKeyRequest) (InvalidateApiKeyResponse, error) { + response := client.JSONRequest(http.MethodDelete, "/_security/api_key", apikeyReq) + + var confirmation InvalidateApiKeyResponse + err := response.DecodeTo(&confirmation) + return confirmation, err +} + +func DeletePrivileges(client Client, privilegesReq DeletePrivilegeRequest) (DeletePrivilegeResponse, error) { + // requires manage_security cluster privilege + path := fmt.Sprintf("/_security/privilege/%v/%v", privilegesReq.Application, privilegesReq.Privilege) + response := client.JSONRequest(http.MethodDelete, path, nil) + + var confirmation DeletePrivilegeResponse + err := response.DecodeTo(&confirmation) + return confirmation, err +} + +func HasPrivileges(client Client, privileges HasPrivilegesRequest, credentials string) (HasPrivilegesResponse, error) { + h := fmt.Sprintf("Authorization: ApiKey %s", credentials) + response := client.JSONRequest(http.MethodGet, "/_security/user/_has_privileges", privileges, h) + + var info HasPrivilegesResponse + err := response.DecodeTo(&info) + return info, err +} + +type CreateApiKeyRequest struct { + Name string `json:"name"` + Expiration *string `json:"expiration,omitempty"` + RoleDescriptors RoleDescriptor `json:"role_descriptors"` +} + +type CreateApiKeyResponse struct { + ApiKey + Key string `json:"api_key"` +} + +type GetApiKeyRequest struct { + ApiKeyQuery + Owner bool `json:"owner"` +} + +type GetApiKeyResponse struct { + ApiKeys []ApiKeyResponse `json:"api_keys"` +} + +type CreatePrivilegesRequest map[AppName]PrivilegeGroup + +type CreatePrivilegesResponse map[AppName]PrivilegeResponse + +type HasPrivilegesRequest struct { + // can't reuse the `Applications` type because here the JSON attribute must be singular + Applications []Application `json:"application"` +} +type HasPrivilegesResponse struct { + Username string `json:"username"` + HasAll bool `json:"has_all_requested"` + Application map[AppName]PrivilegesPerResource `json:"application"` +} + +type InvalidateApiKeyRequest struct { + ApiKeyQuery +} + +type InvalidateApiKeyResponse struct { + Invalidated []string `json:"invalidated_api_keys"` + ErrorCount int `json:"error_count"` +} + +type DeletePrivilegeRequest struct { + Application AppName `json:"application"` + Privilege PrivilegeName `json:"privilege"` +} + +//noinspection GoRedundantParens +type DeletePrivilegeResponse map[AppName](map[PrivilegeName]DeleteResponse) + +type RoleDescriptor map[AppName]Applications + +type Applications struct { + Applications []Application `json:"applications"` +} + +type Application struct { + Name AppName `json:"application"` + Privileges []Privilege `json:"privileges"` + Resources []Resource `json:"resources"` +} + +type ApiKeyResponse struct { + ApiKey + Creation int64 `json:"creation"` + Invalidated bool `json:"invalidated"` + Username string `json:"username"` +} + +type ApiKeyQuery struct { + // normally the Elasticsearch API will require either Id or Name, but not both + Id *string `json:"id,omitempty"` + Name *string `json:"name,omitempty"` +} + +type ApiKey struct { + Id string `json:"id"` + Name string `json:"name"` + ExpirationMs *int64 `json:"expiration,omitempty"` + // This attribute does not come from Elasticsearch, but is filled in by APM Server + Credentials *string `json:"credentials,omitempty"` +} + +type PrivilegeResponse map[Privilege]PutResponse + +type PrivilegeGroup map[PrivilegeName]Actions + +type Perms map[Privilege]bool + +type PrivilegesPerResource map[Resource]Perms + +type Actions struct { + Actions []Privilege `json:"actions"` +} + +type PutResponse struct { + Created bool `json:"created"` +} + +type DeleteResponse struct { + Found bool `json:"found"` +} + +type AppName string + +type Resource string + +// in Elasticsearch a "privilege" represents both an "action" that a user might/might not have authorization to +// perform; and a tuple consisting of a name and an action +// for differentiation, we call the tuple NamedPrivilege +// in apm-server, each name is associated with one action, but that needs not to be the case (see PrivilegeGroup) +type NamedPrivilege struct { + Name PrivilegeName + Action Privilege +} + +// sometimes referred in Elasticsearch documentation as "action" +// we keep the name "privilege" because is more informative +type Privilege string + +type PrivilegeName string + +func NewPrivilege(name, action string) NamedPrivilege { + return NamedPrivilege{ + Name: PrivilegeName(name), + Action: Privilege(action), + } +} diff --git a/x-pack/apm-server/cmd/root.go b/x-pack/apm-server/cmd/root.go index c85dc435118..0ebf14e3e99 100644 --- a/x-pack/apm-server/cmd/root.go +++ b/x-pack/apm-server/cmd/root.go @@ -14,5 +14,5 @@ import ( var RootCmd = cmd.RootCmd func init() { - xpackcmd.AddXPack(RootCmd, cmd.Name) + xpackcmd.AddXPack(RootCmd.BeatsRootCmd, cmd.Name) } From 3214f4206a65fa084a7aa14351bb1fdfa72325ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20=C3=81lvarez?= Date: Mon, 16 Dec 2019 16:58:58 +0100 Subject: [PATCH 02/36] Fix some tests --- beater/api/mux_config_agent_test.go | 6 ++-- beater/api/mux_intake_backend_test.go | 6 ++-- beater/api/mux_intake_rum_test.go | 14 +++++--- beater/api/mux_root_test.go | 3 +- beater/api/mux_sourcemap_handler_test.go | 9 +++-- beater/api/mux_test.go | 5 +-- beater/api/root/handler_test.go | 2 +- beater/authorization/apikey_test.go | 12 +++---- beater/authorization/builder_test.go | 4 +-- beater/config/api_key_test.go | 2 ++ beater/config/config_test.go | 34 ++++++++----------- .../authorization_middleware_test.go | 4 +-- beater/onboarding_test.go | 5 +-- 13 files changed, 59 insertions(+), 47 deletions(-) diff --git a/beater/api/mux_config_agent_test.go b/beater/api/mux_config_agent_test.go index 01e147dc35b..fc599ccc21d 100644 --- a/beater/api/mux_config_agent_test.go +++ b/beater/api/mux_config_agent_test.go @@ -57,8 +57,10 @@ func TestConfigAgentHandler_AuthorizationMiddleware(t *testing.T) { } func TestConfigAgentHandler_KillSwitchMiddleware(t *testing.T) { + cfg, err := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) + require.NoError(t, err) t.Run("Off", func(t *testing.T) { - rec, err := requestToMuxerWithPattern(config.DefaultConfig(beatertest.MockBeatVersion()), AgentConfigPath) + rec, err := requestToMuxerWithPattern(cfg, AgentConfigPath) require.NoError(t, err) require.Equal(t, http.StatusForbidden, rec.Code) approvals.AssertApproveResult(t, approvalPathConfigAgent(t.Name()), rec.Body.Bytes()) @@ -98,7 +100,7 @@ func TestConfigAgentHandler_MonitoringMiddleware(t *testing.T) { } func configEnabledConfigAgent() *config.Config { - cfg := config.DefaultConfig(beatertest.MockBeatVersion()) + cfg, _ := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) cfg.Kibana = common.MustNewConfigFrom(map[string]interface{}{"enabled": "true", "host": "localhost:foo"}) return cfg } diff --git a/beater/api/mux_intake_backend_test.go b/beater/api/mux_intake_backend_test.go index e8ad5f71fa6..f7ee139fcdf 100644 --- a/beater/api/mux_intake_backend_test.go +++ b/beater/api/mux_intake_backend_test.go @@ -35,7 +35,8 @@ import ( func TestIntakeBackendHandler_AuthorizationMiddleware(t *testing.T) { t.Run("Unauthorized", func(t *testing.T) { - cfg := config.DefaultConfig(beatertest.MockBeatVersion()) + cfg, err := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) + require.NoError(t, err) cfg.SecretToken = "1234" rec, err := requestToMuxerWithPattern(cfg, IntakePath) require.NoError(t, err) @@ -45,7 +46,8 @@ func TestIntakeBackendHandler_AuthorizationMiddleware(t *testing.T) { }) t.Run("Authorized", func(t *testing.T) { - cfg := config.DefaultConfig(beatertest.MockBeatVersion()) + cfg, err := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) + require.NoError(t, err) cfg.SecretToken = "1234" h := map[string]string{headers.Authorization: "Bearer 1234"} rec, err := requestToMuxerWithHeader(cfg, IntakePath, http.MethodGet, h) diff --git a/beater/api/mux_intake_rum_test.go b/beater/api/mux_intake_rum_test.go index 0359c5d02b6..880eb0c6d82 100644 --- a/beater/api/mux_intake_rum_test.go +++ b/beater/api/mux_intake_rum_test.go @@ -77,7 +77,9 @@ func TestRUMHandler_NoAuthorizationRequired(t *testing.T) { func TestRUMHandler_KillSwitchMiddleware(t *testing.T) { t.Run("OffRum", func(t *testing.T) { - rec, err := requestToMuxerWithPattern(config.DefaultConfig(beatertest.MockBeatVersion()), IntakeRUMPath) + cfg, err := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) + require.NoError(t, err) + rec, err := requestToMuxerWithPattern(cfg, IntakeRUMPath) require.NoError(t, err) assert.Equal(t, http.StatusForbidden, rec.Code) approvals.AssertApproveResult(t, approvalPathIntakeRUM(t.Name()), rec.Body.Bytes()) @@ -104,7 +106,9 @@ func TestRUMHandler_CORSMiddleware(t *testing.T) { } func TestIntakeRUMHandler_PanicMiddleware(t *testing.T) { - h, err := rumIntakeHandler(config.DefaultConfig(beatertest.MockBeatVersion()), nil, beatertest.NilReporter) + cfg, err := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) + require.NoError(t, err) + h, err := rumIntakeHandler(cfg, nil, beatertest.NilReporter) require.NoError(t, err) rec := &beatertest.WriterPanicOnce{} c := &request.Context{} @@ -115,7 +119,9 @@ func TestIntakeRUMHandler_PanicMiddleware(t *testing.T) { } func TestRumHandler_MonitoringMiddleware(t *testing.T) { - h, err := rumIntakeHandler(config.DefaultConfig(beatertest.MockBeatVersion()), nil, beatertest.NilReporter) + cfg, err := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) + require.NoError(t, err) + h, err := rumIntakeHandler(cfg, nil, beatertest.NilReporter) require.NoError(t, err) c, _ := beatertest.ContextWithResponseRecorder(http.MethodPost, "/") // send GET request resulting in 403 Forbidden error @@ -130,7 +136,7 @@ func TestRumHandler_MonitoringMiddleware(t *testing.T) { } func cfgEnabledRUM() *config.Config { - cfg := config.DefaultConfig(beatertest.MockBeatVersion()) + cfg, _ := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) t := true cfg.RumConfig.Enabled = &t return cfg diff --git a/beater/api/mux_root_test.go b/beater/api/mux_root_test.go index 61c0a918139..4f5f65d747d 100644 --- a/beater/api/mux_root_test.go +++ b/beater/api/mux_root_test.go @@ -34,7 +34,8 @@ import ( ) func TestRootHandler_AuthorizationMiddleware(t *testing.T) { - cfg := config.DefaultConfig(beatertest.MockBeatVersion()) + cfg, err := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) + require.NoError(t, err) cfg.SecretToken = "1234" t.Run("No auth", func(t *testing.T) { diff --git a/beater/api/mux_sourcemap_handler_test.go b/beater/api/mux_sourcemap_handler_test.go index ffe7f5ab010..c6433a656ec 100644 --- a/beater/api/mux_sourcemap_handler_test.go +++ b/beater/api/mux_sourcemap_handler_test.go @@ -56,18 +56,21 @@ func TestSourcemapHandler_AuthorizationMiddleware(t *testing.T) { func TestSourcemapHandler_KillSwitchMiddleware(t *testing.T) { t.Run("OffRum", func(t *testing.T) { - rec, err := requestToMuxerWithPattern(config.DefaultConfig(beatertest.MockBeatVersion()), AssetSourcemapPath) + cfg, err := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) + require.NoError(t, err) + rec, err := requestToMuxerWithPattern(cfg, AssetSourcemapPath) require.NoError(t, err) require.Equal(t, http.StatusForbidden, rec.Code) approvals.AssertApproveResult(t, approvalPathAsset(t.Name()), rec.Body.Bytes()) }) t.Run("OffSourcemap", func(t *testing.T) { - cfg := config.DefaultConfig(beatertest.MockBeatVersion()) + cfg, err := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) + require.NoError(t, err) rum := true cfg.RumConfig.Enabled = &rum cfg.RumConfig.SourceMapping.Enabled = new(bool) - rec, err := requestToMuxerWithPattern(config.DefaultConfig(beatertest.MockBeatVersion()), AssetSourcemapPath) + rec, err := requestToMuxerWithPattern(cfg, AssetSourcemapPath) require.NoError(t, err) require.Equal(t, http.StatusForbidden, rec.Code) approvals.AssertApproveResult(t, approvalPathAsset(t.Name()), rec.Body.Bytes()) diff --git a/beater/api/mux_test.go b/beater/api/mux_test.go index f4d9ed38a37..6be2fe06059 100644 --- a/beater/api/mux_test.go +++ b/beater/api/mux_test.go @@ -55,8 +55,9 @@ func requestToMuxer(cfg *config.Config, r *http.Request) (*httptest.ResponseReco } func testHandler(t *testing.T, fn func(*config.Config, *authorization.Builder, publish.Reporter) (request.Handler, error)) request.Handler { - cfg := config.DefaultConfig(beatertest.MockBeatVersion()) - builder, err := authorization.NewBuilder(cfg) + cfg, err := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) + require.NoError(t, err) + builder, err := authorization.NewBuilder(*cfg) require.NoError(t, err) h, err := fn(cfg, builder, beatertest.NilReporter) require.NoError(t, err) diff --git a/beater/api/root/handler_test.go b/beater/api/root/handler_test.go index 631ae5c64aa..1926375d934 100644 --- a/beater/api/root/handler_test.go +++ b/beater/api/root/handler_test.go @@ -62,7 +62,7 @@ func TestRootHandler(t *testing.T) { t.Run("authorized", func(t *testing.T) { c, w := beatertest.ContextWithResponseRecorder(http.MethodGet, "/") - builder, err := authorization.NewBuilder(&config.Config{SecretToken: "abc"}) + builder, err := authorization.NewBuilder(config.Config{SecretToken: "abc"}) require.NoError(t, err) c.Authorization = builder.ForPrivilege("").AuthorizationFor("Bearer", "abc") Handler()(c) diff --git a/beater/authorization/apikey_test.go b/beater/authorization/apikey_test.go index 649e306414c..43a58b6898e 100644 --- a/beater/authorization/apikey_test.go +++ b/beater/authorization/apikey_test.go @@ -143,9 +143,9 @@ func TestAPIKey_AuthorizedFor(t *testing.T) { handler := tc.builder.forKey("12a3") valid, err := handler.AuthorizedFor("xyz") - require.NoError(t, err) + require.Error(t, err) assert.False(t, valid) - assert.Equal(t, 1, tc.cache.cache.ItemCount()) + assert.Equal(t, 0, tc.cache.cache.ItemCount()) }) t.Run("decode error from ES", func(t *testing.T) { @@ -174,16 +174,16 @@ func (tc *apikeyTestcase) setup(t *testing.T) { if tc.transport == nil { tc.transport = estest.NewTransport(t, http.StatusOK, map[string]interface{}{ "application": map[string]interface{}{ - "application": map[string]map[string]interface{}{ - "foo": {"agentconfig": true, "event": true, "sourcemap": false}, - "bar": {"agentConfig": true, "event": false}, + "apm": map[string]map[string]interface{}{ + "foo": {"config_agent:read": true, "event:write": true, "sourcemap:write": false}, + "bar": {"config_agent:read": true, "event:write": false}, }}}) } tc.client, err = estest.NewElasticsearchClient(tc.transport) require.NoError(t, err) } if tc.cache == nil { - tc.cache = newPrivilegesCache(time.Millisecond, 5) + tc.cache = newPrivilegesCache(time.Minute, 5) } if tc.anyOfPrivileges == nil { tc.anyOfPrivileges = []elasticsearch.Privilege{PrivilegeEventWrite.Action, PrivilegeSourcemapWrite.Action} diff --git a/beater/authorization/builder_test.go b/beater/authorization/builder_test.go index edf4d57db9d..1bff434d091 100644 --- a/beater/authorization/builder_test.go +++ b/beater/authorization/builder_test.go @@ -41,9 +41,9 @@ func TestBuilder(t *testing.T) { } { setup := func() *Builder { - rawCfg := config.DefaultConfig("9.9.9") - cfg, err := config.Setup(nil, rawCfg, nil) + cfg, err := config.Setup(config.DefaultConfig("9.9.9"), nil) require.NoError(t, err) + if tc.withBearer { cfg.SecretToken = "xvz" } diff --git a/beater/config/api_key_test.go b/beater/config/api_key_test.go index 18dca17f752..112904732fa 100644 --- a/beater/config/api_key_test.go +++ b/beater/config/api_key_test.go @@ -77,6 +77,8 @@ func TestAPIKeyConfig_ESConfig(t *testing.T) { LimitMin: 20, ESConfig: &elasticsearch.Config{ Timeout: 5 * time.Second, + Username: "foo", + Password: "bar", Protocol: "http", Hosts: elasticsearch.Hosts{"192.0.0.168:9200"}}}, }, diff --git a/beater/config/config_test.go b/beater/config/config_test.go index cc79951fc2a..9477d33586a 100644 --- a/beater/config/config_test.go +++ b/beater/config/config_test.go @@ -37,15 +37,15 @@ import ( func Test_UnpackConfig(t *testing.T) { falsy, truthy := false, true version := "8.0.0" - defaultRawConfig := DefaultConfig(version) - defaultConfig, _ := Setup(nil, defaultRawConfig, nil) + cfg, err := Setup(DefaultConfig(version), nil) + require.NoError(t, err) tests := map[string]struct { inpCfg map[string]interface{} outCfg *Config }{ "default config": { inpCfg: map[string]interface{}{}, - outCfg: defaultConfig, + outCfg: cfg, }, "overwrite default": { inpCfg: map[string]interface{}{ @@ -240,8 +240,7 @@ func Test_UnpackConfig(t *testing.T) { inpCfg, err := common.NewConfigFrom(test.inpCfg) assert.NoError(t, err) - rawCfg, err := NewRawConfig(version, inpCfg) - cfg, err := Setup(nil, rawCfg, err) + cfg, err := NewConfig(version, inpCfg, nil) require.NoError(t, err) require.NotNil(t, cfg) assert.Equal(t, test.outCfg, cfg) @@ -322,7 +321,7 @@ func TestTLSSettings(t *testing.T) { ucfgCfg, err := common.NewConfigFrom(tc.config) require.NoError(t, err) - cfg, err := NewRawConfig("9.9.9", ucfgCfg) + cfg, err := NewConfig("9.9.9", ucfgCfg, nil) require.NoError(t, err) assert.Equal(t, tc.tls.ClientAuth, cfg.TLS.ClientAuth) }) @@ -348,7 +347,7 @@ func TestTLSSettings(t *testing.T) { ucfgCfg, err := common.NewConfigFrom(tc.config) require.NoError(t, err) - cfg, err := NewRawConfig("9.9.9", ucfgCfg) + cfg, err := NewConfig("9.9.9", ucfgCfg, nil) require.NoError(t, err) assert.Equal(t, tc.tls.VerificationMode, cfg.TLS.VerificationMode) }) @@ -380,25 +379,22 @@ func TestTLSSettings(t *testing.T) { func TestAgentConfig(t *testing.T) { t.Run("InvalidValueTooSmall", func(t *testing.T) { - rawCfg, err := NewRawConfig("9.9.9", - common.MustNewConfigFrom(map[string]string{"agent.config.cache.expiration": "123ms"})) - cfg, err := Setup(nil, rawCfg, err) + cfg, err := NewConfig("9.9.9", + common.MustNewConfigFrom(map[string]string{"agent.config.cache.expiration": "123ms"}), nil) require.Error(t, err) assert.Nil(t, cfg) }) t.Run("InvalidUnit", func(t *testing.T) { - rawCfg, err := NewRawConfig("9.9.9", - common.MustNewConfigFrom(map[string]string{"agent.config.cache.expiration": "1230ms"})) - cfg, err := Setup(nil, rawCfg, err) + cfg, err := NewConfig("9.9.9", + common.MustNewConfigFrom(map[string]string{"agent.config.cache.expiration": "1230ms"}), nil) require.Error(t, err) assert.Nil(t, cfg) }) t.Run("Valid", func(t *testing.T) { - rawCfg, err := NewRawConfig("9.9.9", - common.MustNewConfigFrom(map[string]string{"agent.config.cache.expiration": "123000ms"})) - cfg, err := Setup(nil, rawCfg, err) + cfg, err := NewConfig("9.9.9", + common.MustNewConfigFrom(map[string]string{"agent.config.cache.expiration": "123000ms"}), nil) require.NoError(t, err) assert.Equal(t, time.Second*123, cfg.AgentConfig.Cache.Expiration) }) @@ -410,16 +406,14 @@ func TestNewConfig_ESConfig(t *testing.T) { require.NoError(t, err) // no es config given - rawCfg, err := NewRawConfig(version, ucfg) - cfg, err := Setup(nil, rawCfg, err) + cfg, err := NewConfig(version, ucfg, nil) require.NoError(t, err) assert.Nil(t, cfg.RumConfig.SourceMapping.ESConfig) assert.Equal(t, elasticsearch.DefaultConfig(), cfg.APIKeyConfig.ESConfig) // with es config outputESCfg := common.MustNewConfigFrom(`{"hosts":["192.0.0.168:9200"]}`) - rawCfg, err = NewRawConfig(version, ucfg) - cfg, err = Setup(outputESCfg, rawCfg, err) + cfg, err = NewConfig(version, ucfg, outputESCfg) require.NoError(t, err) assert.NotNil(t, cfg.RumConfig.SourceMapping.ESConfig) assert.Equal(t, []string{"192.0.0.168:9200"}, []string(cfg.RumConfig.SourceMapping.ESConfig.Hosts)) diff --git a/beater/middleware/authorization_middleware_test.go b/beater/middleware/authorization_middleware_test.go index 2fad8f141ee..60a25c32cf6 100644 --- a/beater/middleware/authorization_middleware_test.go +++ b/beater/middleware/authorization_middleware_test.go @@ -49,9 +49,9 @@ func TestAuthorizationMiddleware(t *testing.T) { if tc.header != "" { c.Request.Header.Set(headers.Authorization, tc.header) } - builder, err := authorization.NewBuilder(&config.Config{SecretToken: token}) + builder, err := authorization.NewBuilder(config.Config{SecretToken: token}) require.NoError(t, err) - return builder.ForAnyOfPrivileges(authorization.PrivilegesAll), c, rec + return builder.ForAnyOfPrivileges(authorization.ActionAny), c, rec } t.Run(name+"secured apply", func(t *testing.T) { diff --git a/beater/onboarding_test.go b/beater/onboarding_test.go index 68b06ce5f68..ea9b909a3c3 100644 --- a/beater/onboarding_test.go +++ b/beater/onboarding_test.go @@ -33,12 +33,13 @@ import ( ) func TestNotifyUpServerDown(t *testing.T) { - config := config.DefaultConfig("7.0.0") + config, err := config.Setup(config.DefaultConfig("7.0.0"), nil) + require.NoError(t, err) var saved beat.Event var publisher = func(e beat.Event) { saved = e } lis, err := net.Listen("tcp", "localhost:0") - assert.NoError(t, err) + require.NoError(t, err) defer lis.Close() config.Host = lis.Addr().String() From 1d7f20469653b6b33679f3b24f897f318762ddf6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20=C3=81lvarez?= Date: Wed, 18 Dec 2019 10:52:30 +0100 Subject: [PATCH 03/36] Some fixes --- cmd/apikey.go | 23 +++++++++++++++++------ elasticsearch/config.go | 2 +- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/cmd/apikey.go b/cmd/apikey.go index 26e3d134ea6..5b7bfe62245 100644 --- a/cmd/apikey.go +++ b/cmd/apikey.go @@ -111,9 +111,9 @@ func createApiKeyWithPrivileges(client es.Client, apikeyName, expiry string, pri } func getApiKey(client es.Client, id, name *string, validOnly, asJson bool) error { - if id != nil { + if isSet(id) { name = nil - } else if id == nil && name == nil { + } else if !(isSet(id) || isSet(name)) { return printErr(errors.New("could not query Elasticsearch"), `either "id" or "name" are required`, asJson) @@ -154,9 +154,9 @@ func getApiKey(client es.Client, id, name *string, validOnly, asJson bool) error } func invalidateApiKey(client es.Client, id, name *string, deletePrivileges, asJson bool) error { - if id != nil { + if isSet(id) { name = nil - } else if id == nil && name == nil { + } else if !(isSet(id) || isSet(name)) { return printErr(errors.New("could not query Elasticsearch"), `either "id" or "name" are required`, asJson) @@ -210,22 +210,29 @@ func invalidateApiKey(client es.Client, id, name *string, deletePrivileges, asJs } func verifyApiKey(config *config.Config, privileges []es.Privilege, credentials string, asJson bool) error { - builder, err := auth.NewBuilder(*config) perms := make(es.Perms) printText, printJson := printers(asJson) + var err error for _, privilege := range privileges { + var builder *auth.Builder + builder, err := auth.NewBuilder(*config) if err != nil { break } + var authorized bool authorized, err = builder. ForPrivilege(privilege). AuthorizationFor(headers.APIKey, credentials). AuthorizedFor(auth.ResourceInternal) + if err != nil { + break + } + perms[privilege] = authorized - printText("Authorized for %s...: %s\n", humanPrivilege(privilege), humanBool(authorized)) + printText("Authorized for %s...: %s", humanPrivilege(privilege), humanBool(authorized)) } if err != nil { @@ -324,3 +331,7 @@ func printErr(err error, help string, asJson bool) error { } return errors.Wrap(err, help) } + +func isSet(s *string) bool { + return s != nil && *s != "" +} diff --git a/elasticsearch/config.go b/elasticsearch/config.go index d0ef6102755..c433437e530 100644 --- a/elasticsearch/config.go +++ b/elasticsearch/config.go @@ -44,7 +44,7 @@ var ( // Config holds all configurable fields that are used to create a Client type Config struct { - Hosts Hosts `config:"hosts" validate:"required"` + Hosts Hosts `config:"hosts"` Protocol string `config:"protocol"` Path string `config:"path"` ProxyURL string `config:"proxy_url"` From 2420c35f7baa364bc68f2f082a8d3432d7879a3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20=C3=81lvarez?= Date: Wed, 18 Dec 2019 15:53:51 +0100 Subject: [PATCH 04/36] Revert all the config stuff --- beater/api/mux_config_agent_test.go | 6 ++--- beater/api/mux_intake_backend_test.go | 6 ++--- beater/api/mux_intake_rum_test.go | 14 +++-------- beater/api/mux_root_test.go | 3 +-- beater/api/mux_sourcemap_handler_test.go | 9 +++---- beater/api/mux_test.go | 3 +-- beater/authorization/builder_test.go | 3 +-- beater/config/config.go | 31 +++++------------------- beater/config/config_test.go | 5 ++-- beater/onboarding_test.go | 5 ++-- cmd/root.go | 11 +-------- 11 files changed, 25 insertions(+), 71 deletions(-) diff --git a/beater/api/mux_config_agent_test.go b/beater/api/mux_config_agent_test.go index fc599ccc21d..01e147dc35b 100644 --- a/beater/api/mux_config_agent_test.go +++ b/beater/api/mux_config_agent_test.go @@ -57,10 +57,8 @@ func TestConfigAgentHandler_AuthorizationMiddleware(t *testing.T) { } func TestConfigAgentHandler_KillSwitchMiddleware(t *testing.T) { - cfg, err := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) - require.NoError(t, err) t.Run("Off", func(t *testing.T) { - rec, err := requestToMuxerWithPattern(cfg, AgentConfigPath) + rec, err := requestToMuxerWithPattern(config.DefaultConfig(beatertest.MockBeatVersion()), AgentConfigPath) require.NoError(t, err) require.Equal(t, http.StatusForbidden, rec.Code) approvals.AssertApproveResult(t, approvalPathConfigAgent(t.Name()), rec.Body.Bytes()) @@ -100,7 +98,7 @@ func TestConfigAgentHandler_MonitoringMiddleware(t *testing.T) { } func configEnabledConfigAgent() *config.Config { - cfg, _ := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) + cfg := config.DefaultConfig(beatertest.MockBeatVersion()) cfg.Kibana = common.MustNewConfigFrom(map[string]interface{}{"enabled": "true", "host": "localhost:foo"}) return cfg } diff --git a/beater/api/mux_intake_backend_test.go b/beater/api/mux_intake_backend_test.go index f7ee139fcdf..e8ad5f71fa6 100644 --- a/beater/api/mux_intake_backend_test.go +++ b/beater/api/mux_intake_backend_test.go @@ -35,8 +35,7 @@ import ( func TestIntakeBackendHandler_AuthorizationMiddleware(t *testing.T) { t.Run("Unauthorized", func(t *testing.T) { - cfg, err := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) - require.NoError(t, err) + cfg := config.DefaultConfig(beatertest.MockBeatVersion()) cfg.SecretToken = "1234" rec, err := requestToMuxerWithPattern(cfg, IntakePath) require.NoError(t, err) @@ -46,8 +45,7 @@ func TestIntakeBackendHandler_AuthorizationMiddleware(t *testing.T) { }) t.Run("Authorized", func(t *testing.T) { - cfg, err := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) - require.NoError(t, err) + cfg := config.DefaultConfig(beatertest.MockBeatVersion()) cfg.SecretToken = "1234" h := map[string]string{headers.Authorization: "Bearer 1234"} rec, err := requestToMuxerWithHeader(cfg, IntakePath, http.MethodGet, h) diff --git a/beater/api/mux_intake_rum_test.go b/beater/api/mux_intake_rum_test.go index 880eb0c6d82..0359c5d02b6 100644 --- a/beater/api/mux_intake_rum_test.go +++ b/beater/api/mux_intake_rum_test.go @@ -77,9 +77,7 @@ func TestRUMHandler_NoAuthorizationRequired(t *testing.T) { func TestRUMHandler_KillSwitchMiddleware(t *testing.T) { t.Run("OffRum", func(t *testing.T) { - cfg, err := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) - require.NoError(t, err) - rec, err := requestToMuxerWithPattern(cfg, IntakeRUMPath) + rec, err := requestToMuxerWithPattern(config.DefaultConfig(beatertest.MockBeatVersion()), IntakeRUMPath) require.NoError(t, err) assert.Equal(t, http.StatusForbidden, rec.Code) approvals.AssertApproveResult(t, approvalPathIntakeRUM(t.Name()), rec.Body.Bytes()) @@ -106,9 +104,7 @@ func TestRUMHandler_CORSMiddleware(t *testing.T) { } func TestIntakeRUMHandler_PanicMiddleware(t *testing.T) { - cfg, err := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) - require.NoError(t, err) - h, err := rumIntakeHandler(cfg, nil, beatertest.NilReporter) + h, err := rumIntakeHandler(config.DefaultConfig(beatertest.MockBeatVersion()), nil, beatertest.NilReporter) require.NoError(t, err) rec := &beatertest.WriterPanicOnce{} c := &request.Context{} @@ -119,9 +115,7 @@ func TestIntakeRUMHandler_PanicMiddleware(t *testing.T) { } func TestRumHandler_MonitoringMiddleware(t *testing.T) { - cfg, err := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) - require.NoError(t, err) - h, err := rumIntakeHandler(cfg, nil, beatertest.NilReporter) + h, err := rumIntakeHandler(config.DefaultConfig(beatertest.MockBeatVersion()), nil, beatertest.NilReporter) require.NoError(t, err) c, _ := beatertest.ContextWithResponseRecorder(http.MethodPost, "/") // send GET request resulting in 403 Forbidden error @@ -136,7 +130,7 @@ func TestRumHandler_MonitoringMiddleware(t *testing.T) { } func cfgEnabledRUM() *config.Config { - cfg, _ := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) + cfg := config.DefaultConfig(beatertest.MockBeatVersion()) t := true cfg.RumConfig.Enabled = &t return cfg diff --git a/beater/api/mux_root_test.go b/beater/api/mux_root_test.go index 4f5f65d747d..61c0a918139 100644 --- a/beater/api/mux_root_test.go +++ b/beater/api/mux_root_test.go @@ -34,8 +34,7 @@ import ( ) func TestRootHandler_AuthorizationMiddleware(t *testing.T) { - cfg, err := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) - require.NoError(t, err) + cfg := config.DefaultConfig(beatertest.MockBeatVersion()) cfg.SecretToken = "1234" t.Run("No auth", func(t *testing.T) { diff --git a/beater/api/mux_sourcemap_handler_test.go b/beater/api/mux_sourcemap_handler_test.go index c6433a656ec..ffe7f5ab010 100644 --- a/beater/api/mux_sourcemap_handler_test.go +++ b/beater/api/mux_sourcemap_handler_test.go @@ -56,21 +56,18 @@ func TestSourcemapHandler_AuthorizationMiddleware(t *testing.T) { func TestSourcemapHandler_KillSwitchMiddleware(t *testing.T) { t.Run("OffRum", func(t *testing.T) { - cfg, err := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) - require.NoError(t, err) - rec, err := requestToMuxerWithPattern(cfg, AssetSourcemapPath) + rec, err := requestToMuxerWithPattern(config.DefaultConfig(beatertest.MockBeatVersion()), AssetSourcemapPath) require.NoError(t, err) require.Equal(t, http.StatusForbidden, rec.Code) approvals.AssertApproveResult(t, approvalPathAsset(t.Name()), rec.Body.Bytes()) }) t.Run("OffSourcemap", func(t *testing.T) { - cfg, err := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) - require.NoError(t, err) + cfg := config.DefaultConfig(beatertest.MockBeatVersion()) rum := true cfg.RumConfig.Enabled = &rum cfg.RumConfig.SourceMapping.Enabled = new(bool) - rec, err := requestToMuxerWithPattern(cfg, AssetSourcemapPath) + rec, err := requestToMuxerWithPattern(config.DefaultConfig(beatertest.MockBeatVersion()), AssetSourcemapPath) require.NoError(t, err) require.Equal(t, http.StatusForbidden, rec.Code) approvals.AssertApproveResult(t, approvalPathAsset(t.Name()), rec.Body.Bytes()) diff --git a/beater/api/mux_test.go b/beater/api/mux_test.go index 6be2fe06059..00f89bc8a42 100644 --- a/beater/api/mux_test.go +++ b/beater/api/mux_test.go @@ -55,8 +55,7 @@ func requestToMuxer(cfg *config.Config, r *http.Request) (*httptest.ResponseReco } func testHandler(t *testing.T, fn func(*config.Config, *authorization.Builder, publish.Reporter) (request.Handler, error)) request.Handler { - cfg, err := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) - require.NoError(t, err) + cfg := config.DefaultConfig(beatertest.MockBeatVersion()) builder, err := authorization.NewBuilder(*cfg) require.NoError(t, err) h, err := fn(cfg, builder, beatertest.NilReporter) diff --git a/beater/authorization/builder_test.go b/beater/authorization/builder_test.go index 1bff434d091..f665d11a585 100644 --- a/beater/authorization/builder_test.go +++ b/beater/authorization/builder_test.go @@ -41,8 +41,7 @@ func TestBuilder(t *testing.T) { } { setup := func() *Builder { - cfg, err := config.Setup(config.DefaultConfig("9.9.9"), nil) - require.NoError(t, err) + cfg := config.DefaultConfig("9.9.9") if tc.withBearer { cfg.SecretToken = "xvz" diff --git a/beater/config/config.go b/beater/config/config.go index 4321c3932d5..98cce679dcd 100644 --- a/beater/config/config.go +++ b/beater/config/config.go @@ -68,8 +68,6 @@ type Config struct { Pipeline string } -type RawConfig Config - // ExpvarConfig holds config information about exposing expvar type ExpvarConfig struct { Enabled *bool `config:"enabled"` @@ -86,8 +84,9 @@ type Cache struct { Expiration time.Duration `config:"expiration"` } -// NewRawConfig unpacks the ucfg data -func NewRawConfig(version string, ucfg *common.Config) (*RawConfig, error) { +// NewConfig creates a Config struct based on the default config and the given input params +func NewConfig(version string, ucfg *common.Config, outputESCfg *common.Config) (*Config, error) { + logger := logp.NewLogger(logs.Config) c := DefaultConfig(version) if ucfg.HasField("ssl") { @@ -105,16 +104,6 @@ func NewRawConfig(version string, ucfg *common.Config) (*RawConfig, error) { if err := ucfg.Unpack(c); err != nil { return nil, errors.Wrap(err, "Error processing configuration") } - return c, nil -} - -// Setup contains ad-hoc logic for apm-server configuration -func Setup(rawConfig *RawConfig, outputESCfg *common.Config) (*Config, error) { - logger := logp.NewLogger(logs.Config) - if rawConfig == nil { - return nil, nil - } - c := Config(*rawConfig) if float64(int(c.AgentConfig.Cache.Expiration.Seconds())) != c.AgentConfig.Cache.Expiration.Seconds() { return nil, errors.New(msgInvalidConfigAgentCfg) @@ -136,15 +125,7 @@ func Setup(rawConfig *RawConfig, outputESCfg *common.Config) (*Config, error) { return nil, err } - return &c, nil -} - -func NewConfig(version string, ucfg *common.Config, outputESCfg *common.Config) (*Config, error) { - raw, err := NewRawConfig(version, ucfg) - if err != nil { - return nil, err - } - return Setup(raw, outputESCfg) + return c, nil } // IsEnabled indicates whether expvar is enabled or not @@ -153,8 +134,8 @@ func (c *ExpvarConfig) IsEnabled() bool { } // DefaultConfig returns a config with default settings for `apm-server` config options. -func DefaultConfig(beatVersion string) *RawConfig { - return &RawConfig{ +func DefaultConfig(beatVersion string) *Config { + return &Config{ Host: net.JoinHostPort("localhost", DefaultPort), MaxHeaderSize: 1 * 1024 * 1024, // 1mb MaxConnections: 0, // unlimited diff --git a/beater/config/config_test.go b/beater/config/config_test.go index 9477d33586a..867b411ef8b 100644 --- a/beater/config/config_test.go +++ b/beater/config/config_test.go @@ -37,15 +37,14 @@ import ( func Test_UnpackConfig(t *testing.T) { falsy, truthy := false, true version := "8.0.0" - cfg, err := Setup(DefaultConfig(version), nil) - require.NoError(t, err) + tests := map[string]struct { inpCfg map[string]interface{} outCfg *Config }{ "default config": { inpCfg: map[string]interface{}{}, - outCfg: cfg, + outCfg: DefaultConfig(version), }, "overwrite default": { inpCfg: map[string]interface{}{ diff --git a/beater/onboarding_test.go b/beater/onboarding_test.go index ea9b909a3c3..68b06ce5f68 100644 --- a/beater/onboarding_test.go +++ b/beater/onboarding_test.go @@ -33,13 +33,12 @@ import ( ) func TestNotifyUpServerDown(t *testing.T) { - config, err := config.Setup(config.DefaultConfig("7.0.0"), nil) - require.NoError(t, err) + config := config.DefaultConfig("7.0.0") var saved beat.Event var publisher = func(e beat.Event) { saved = e } lis, err := net.Listen("tcp", "localhost:0") - require.NoError(t, err) + assert.NoError(t, err) defer lis.Close() config.Host = lis.Addr().String() diff --git a/cmd/root.go b/cmd/root.go index 05211d48e46..7c992508061 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -312,16 +312,7 @@ func bootstrap(settings instance.Settings) (*config.Config, error) { } outCfg := beat.Config.Output - apmRawConfig, err := config.NewRawConfig(settings.Version, beat.RawConfig) - if apmRawConfig == nil { - return nil, err - } - if apmRawConfig.APIKeyConfig == nil { - apmRawConfig.APIKeyConfig = &config.APIKeyConfig{} - } - apmRawConfig.APIKeyConfig.Enabled = true - - return config.Setup(apmRawConfig, outCfg.Config()) + return config.NewConfig(settings.Version, beat.RawConfig, outCfg.Config()) } // if all are false, returns any ("*") From 4fcbd9da7101b71b66d69252bb7613658c0b6bdb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20=C3=81lvarez?= Date: Thu, 5 Dec 2019 13:28:06 +0100 Subject: [PATCH 05/36] Add apikey subcommand --- _meta/beat.yml | 5 +- apm-server.docker.yml | 5 +- apm-server.yml | 5 +- beater/api/mux.go | 12 +- beater/authorization/allow.go | 4 +- beater/authorization/apikey.go | 89 ++--- beater/authorization/apikey_test.go | 26 +- beater/authorization/bearer.go | 4 +- beater/authorization/builder.go | 17 +- beater/authorization/builder_test.go | 15 +- beater/authorization/deny.go | 4 +- beater/authorization/privilege.go | 28 +- beater/authorization/privilege_test.go | 8 +- beater/beater.go | 1 + beater/config/api_key.go | 4 +- beater/config/config.go | 31 +- beater/config/config_test.go | 33 +- beater/middleware/authorization_middleware.go | 3 +- cmd/apikey.go | 326 ++++++++++++++++++ cmd/root.go | 224 +++++++++++- elasticsearch/client.go | 143 ++++++-- elasticsearch/security_api.go | 220 ++++++++++++ x-pack/apm-server/cmd/root.go | 2 +- 23 files changed, 1030 insertions(+), 179 deletions(-) create mode 100644 cmd/apikey.go create mode 100644 elasticsearch/security_api.go diff --git a/_meta/beat.yml b/_meta/beat.yml index d9511fb8917..bd87dea8c2e 100644 --- a/_meta/beat.yml +++ b/_meta/beat.yml @@ -153,7 +153,10 @@ apm-server: #protocol: "http" - # Optional HTTP Path. + #username: "elastic" + #password: "changeme" + + # Optional HTTP Path. #path: "" # Proxy server url. diff --git a/apm-server.docker.yml b/apm-server.docker.yml index ebca44674f4..419b940f95a 100644 --- a/apm-server.docker.yml +++ b/apm-server.docker.yml @@ -153,7 +153,10 @@ apm-server: #protocol: "http" - # Optional HTTP Path. + #username: "elastic" + #password: "changeme" + + # Optional HTTP Path. #path: "" # Proxy server url. diff --git a/apm-server.yml b/apm-server.yml index 8dff2ae9554..c367b5f940a 100644 --- a/apm-server.yml +++ b/apm-server.yml @@ -153,7 +153,10 @@ apm-server: #protocol: "http" - # Optional HTTP Path. + #username: "elastic" + #password: "changeme" + + # Optional HTTP Path. #path: "" # Proxy server url. diff --git a/beater/api/mux.go b/beater/api/mux.go index 1c42ed933e3..f52e2eb4c77 100644 --- a/beater/api/mux.go +++ b/beater/api/mux.go @@ -83,7 +83,7 @@ func NewMux(beaterConfig *config.Config, report publish.Reporter) (*http.ServeMu mux := http.NewServeMux() logger := logp.NewLogger(logs.Handler) - auth, err := authorization.NewBuilder(beaterConfig) + auth, err := authorization.NewBuilder(*beaterConfig) if err != nil { return nil, err } @@ -125,7 +125,7 @@ func NewMux(beaterConfig *config.Config, report publish.Reporter) (*http.ServeMu func profileHandler(cfg *config.Config, builder *authorization.Builder, reporter publish.Reporter) (request.Handler, error) { h := profile.Handler(systemMetadataDecoder(cfg, emptyDecoder), transform.Config{}, reporter) - authHandler := builder.ForPrivilege(authorization.PrivilegeEventWrite) + authHandler := builder.ForPrivilege(authorization.PrivilegeEventWrite.Action) return middleware.Wrap(h, backendMiddleware(cfg, authHandler, profile.MonitoringMap)...) } @@ -137,7 +137,7 @@ func backendIntakeHandler(cfg *config.Config, builder *authorization.Builder, re MaxEventSize: cfg.MaxEventSize, }, reporter) - authHandler := builder.ForPrivilege(authorization.PrivilegeEventWrite) + authHandler := builder.ForPrivilege(authorization.PrivilegeEventWrite.Action) return middleware.Wrap(h, backendMiddleware(cfg, authHandler, intake.MonitoringMap)...) } @@ -162,12 +162,12 @@ func sourcemapHandler(cfg *config.Config, builder *authorization.Builder, report return nil, err } h := sourcemap.Handler(systemMetadataDecoder(cfg, decoder.DecodeSourcemapFormData), psourcemap.Processor, *tcfg, reporter) - authHandler := builder.ForPrivilege(authorization.PrivilegeSourcemapWrite) + authHandler := builder.ForPrivilege(authorization.PrivilegeSourcemapWrite.Action) return middleware.Wrap(h, sourcemapMiddleware(cfg, authHandler)...) } func backendAgentConfigHandler(cfg *config.Config, builder *authorization.Builder, _ publish.Reporter) (request.Handler, error) { - authHandler := builder.ForPrivilege(authorization.PrivilegeAgentConfigRead) + authHandler := builder.ForPrivilege(authorization.PrivilegeAgentConfigRead.Action) return agentConfigHandler(cfg, authHandler, backendMiddleware) } @@ -193,7 +193,7 @@ func agentConfigHandler(cfg *config.Config, authHandler *authorization.Handler, func rootHandler(cfg *config.Config, builder *authorization.Builder, _ publish.Reporter) (request.Handler, error) { return middleware.Wrap(root.Handler(), - rootMiddleware(cfg, builder.ForAnyOfPrivileges(authorization.PrivilegesAll))...) + rootMiddleware(cfg, builder.ForAnyOfPrivileges(authorization.ActionAny))...) } func apmMiddleware(m map[request.ResultID]*monitoring.Int) []middleware.Middleware { diff --git a/beater/authorization/allow.go b/beater/authorization/allow.go index 1a3bba19aa0..56ea3460296 100644 --- a/beater/authorization/allow.go +++ b/beater/authorization/allow.go @@ -17,11 +17,13 @@ package authorization +import "github.com/elastic/apm-server/elasticsearch" + // AllowAuth implements the Authorization interface. It allows all authorization requests. type AllowAuth struct{} // AuthorizedFor always returns true -func (AllowAuth) AuthorizedFor(_ string) (bool, error) { +func (AllowAuth) AuthorizedFor(_ elasticsearch.Resource) (bool, error) { return true, nil } diff --git a/beater/authorization/apikey.go b/beater/authorization/apikey.go index 997f409d9f8..d05010f9865 100644 --- a/beater/authorization/apikey.go +++ b/beater/authorization/apikey.go @@ -18,43 +18,39 @@ package authorization import ( - "encoding/json" - "net/http" - "strings" + "fmt" "time" "github.com/pkg/errors" - "github.com/elastic/apm-server/beater/headers" - "github.com/elastic/apm-server/elasticsearch" + es "github.com/elastic/apm-server/elasticsearch" ) -const ( - //DefaultResource for apm backend enabled API Keys - DefaultResource = "-" +const cleanupInterval = 60 * time.Second - application = "apm" - sep = `","` - - cleanupInterval = 60 * time.Second +var ( + // Constant mapped to the "application" field for the Elasticsearch security API + // This identifies privileges and keys created for APM + Application = es.AppName("apm") + // Only valid for first authorization of a request. + // The API Key needs to grant privileges to additional resources for successful processing of requests. + ResourceInternal = es.Resource("-") + ResourceAny = es.Resource("*") ) type apikeyBuilder struct { - esClient elasticsearch.Client + esClient es.Client cache *privilegesCache - anyOfPrivileges []string + anyOfPrivileges []es.Privilege } type apikeyAuth struct { *apikeyBuilder + // key is base64(id:apiKey) key string } -type hasPrivilegesResponse struct { - Applications map[string]map[string]privileges `json:"application"` -} - -func newApikeyBuilder(client elasticsearch.Client, cache *privilegesCache, anyOfPrivileges []string) *apikeyBuilder { +func newApikeyBuilder(client es.Client, cache *privilegesCache, anyOfPrivileges []es.Privilege) *apikeyBuilder { return &apikeyBuilder{client, cache, anyOfPrivileges} } @@ -69,12 +65,8 @@ func (a *apikeyAuth) IsAuthorizationConfigured() bool { // AuthorizedFor checks if the configured api key is authorized. // An api key is considered to be authorized when the api key has the configured privileges for the requested resource. -// Privileges are fetched from Elasticsearch and then cached in a global cache. -func (a *apikeyAuth) AuthorizedFor(resource string) (bool, error) { - if resource == "" { - resource = DefaultResource - } - +// PrivilegeGroup are fetched from Elasticsearch and then cached in a global cache. +func (a *apikeyAuth) AuthorizedFor(resource es.Resource) (bool, error) { //fetch from cache if allowed, found := a.fromCache(resource); found { return allowed, nil @@ -98,7 +90,7 @@ func (a *apikeyAuth) AuthorizedFor(resource string) (bool, error) { return allowed, nil } -func (a *apikeyAuth) fromCache(resource string) (allowed bool, found bool) { +func (a *apikeyAuth) fromCache(resource es.Resource) (allowed bool, found bool) { privileges := a.cache.get(id(a.key, resource)) if privileges == nil { return @@ -114,43 +106,28 @@ func (a *apikeyAuth) fromCache(resource string) (allowed bool, found bool) { return } -func (a *apikeyAuth) queryES(resource string) (privileges, error) { - query := buildQuery(PrivilegesAll, resource) - statusCode, body, err := a.esClient.SecurityHasPrivilegesRequest(strings.NewReader(query), - http.Header{headers.Authorization: []string{headers.APIKey + " " + a.key}}) - if err != nil { - return nil, err - } - defer body.Close() - if statusCode != http.StatusOK { - // return nil privileges for queried apps to ensure they are cached - return privileges{}, nil +func (a *apikeyAuth) queryES(resource es.Resource) (es.Perms, error) { + request := es.HasPrivilegesRequest{ + Applications: []es.Application{ + { + Name: Application, + Privileges: a.anyOfPrivileges, + Resources: []es.Resource{resource}, + }, + }, } - - var decodedResponse hasPrivilegesResponse - if err := json.NewDecoder(body).Decode(&decodedResponse); err != nil { + info, err := es.HasPrivileges(a.esClient, request, a.key) + if err != nil { return nil, err } - if resources, ok := decodedResponse.Applications[application]; ok { + if resources, ok := info.Application[Application]; ok { if privileges, ok := resources[resource]; ok { return privileges, nil } } - return privileges{}, nil -} - -func buildQuery(privileges []string, resource string) string { - var b strings.Builder - b.WriteString(`{"application":[{"application":"`) - b.WriteString(application) - b.WriteString(`","privileges":["`) - b.WriteString(strings.Join(privileges, sep)) - b.WriteString(`"],"resources":"`) - b.WriteString(resource) - b.WriteString(`"}]}`) - return b.String() + return es.Perms{}, nil } -func id(apiKey, resource string) string { - return apiKey + "_" + resource +func id(apiKey string, resource es.Resource) string { + return apiKey + "_" + fmt.Sprintf("%v", resource) } diff --git a/beater/authorization/apikey_test.go b/beater/authorization/apikey_test.go index 4d9fa19aec3..649e306414c 100644 --- a/beater/authorization/apikey_test.go +++ b/beater/authorization/apikey_test.go @@ -41,11 +41,11 @@ func TestApikeyBuilder(t *testing.T) { handler2 := tc.builder.forKey(key) // add existing privileges to shared cache - privilegesValid := privileges{} + privilegesValid := elasticsearch.Perms{} for _, p := range PrivilegesAll { - privilegesValid[p] = true + privilegesValid[p.Action] = true } - resource := "service-go" + resource := elasticsearch.Resource("service-go") tc.cache.add(id(key, resource), privilegesValid) // check that cache is actually shared between apiKeyHandlers @@ -85,12 +85,12 @@ func TestAPIKey_AuthorizedFor(t *testing.T) { tc.setup(t) key := "" handler := tc.builder.forKey(key) - resourceValid := "foo" - resourceInvalid := "bar" - resourceMissing := "missing" + resourceValid := elasticsearch.Resource("foo") + resourceInvalid := elasticsearch.Resource("bar") + resourceMissing := elasticsearch.Resource("missing") - tc.cache.add(id(key, resourceValid), privileges{tc.anyOfPrivileges[0]: true}) - tc.cache.add(id(key, resourceInvalid), privileges{tc.anyOfPrivileges[0]: false}) + tc.cache.add(id(key, resourceValid), elasticsearch.Perms{tc.anyOfPrivileges[0]: true}) + tc.cache.add(id(key, resourceInvalid), elasticsearch.Perms{tc.anyOfPrivileges[0]: false}) valid, err := handler.AuthorizedFor(resourceValid) require.NoError(t, err) @@ -163,7 +163,7 @@ type apikeyTestcase struct { transport *estest.Transport client elasticsearch.Client cache *privilegesCache - anyOfPrivileges []string + anyOfPrivileges []elasticsearch.Privilege builder *apikeyBuilder } @@ -174,9 +174,9 @@ func (tc *apikeyTestcase) setup(t *testing.T) { if tc.transport == nil { tc.transport = estest.NewTransport(t, http.StatusOK, map[string]interface{}{ "application": map[string]interface{}{ - application: map[string]privileges{ - "foo": {PrivilegeAgentConfigRead: true, PrivilegeEventWrite: true, PrivilegeSourcemapWrite: false}, - "bar": {PrivilegeAgentConfigRead: true, PrivilegeEventWrite: false}, + "application": map[string]map[string]interface{}{ + "foo": {"agentconfig": true, "event": true, "sourcemap": false}, + "bar": {"agentConfig": true, "event": false}, }}}) } tc.client, err = estest.NewElasticsearchClient(tc.transport) @@ -186,7 +186,7 @@ func (tc *apikeyTestcase) setup(t *testing.T) { tc.cache = newPrivilegesCache(time.Millisecond, 5) } if tc.anyOfPrivileges == nil { - tc.anyOfPrivileges = []string{PrivilegeEventWrite, PrivilegeSourcemapWrite} + tc.anyOfPrivileges = []elasticsearch.Privilege{PrivilegeEventWrite.Action, PrivilegeSourcemapWrite.Action} } tc.builder = newApikeyBuilder(tc.client, tc.cache, tc.anyOfPrivileges) } diff --git a/beater/authorization/bearer.go b/beater/authorization/bearer.go index c3e09aa90dc..92249e2fa02 100644 --- a/beater/authorization/bearer.go +++ b/beater/authorization/bearer.go @@ -19,6 +19,8 @@ package authorization import ( "crypto/subtle" + + "github.com/elastic/apm-server/elasticsearch" ) type bearerBuilder struct { @@ -39,7 +41,7 @@ func (b bearerBuilder) forToken(token string) *bearerAuth { configured: true} } -func (b *bearerAuth) AuthorizedFor(_ string) (bool, error) { +func (b *bearerAuth) AuthorizedFor(_ elasticsearch.Resource) (bool, error) { return b.authorized, nil } diff --git a/beater/authorization/builder.go b/beater/authorization/builder.go index 2a043c77eb6..ad637ab762a 100644 --- a/beater/authorization/builder.go +++ b/beater/authorization/builder.go @@ -37,7 +37,7 @@ type Handler Builder // Authorization interface to be implemented by different auth types type Authorization interface { - AuthorizedFor(string) (bool, error) + AuthorizedFor(_ elasticsearch.Resource) (bool, error) IsAuthorizationConfigured() bool } @@ -46,10 +46,15 @@ const ( ) // NewBuilder creates authorization builder based off of the given information -func NewBuilder(cfg *config.Config) (*Builder, error) { +// if apm-server.api_key is enabled, authorization is granted/denied solely +// based on the request Authorization header +func NewBuilder(cfg config.Config) (*Builder, error) { b := Builder{} b.fallback = AllowAuth{} if cfg.APIKeyConfig.IsEnabled() { + // do not use username+password for API Key requests + cfg.APIKeyConfig.ESConfig.Username = "" + cfg.APIKeyConfig.ESConfig.Password = "" client, err := elasticsearch.NewClient(cfg.APIKeyConfig.ESConfig) if err != nil { return nil, err @@ -57,7 +62,7 @@ func NewBuilder(cfg *config.Config) (*Builder, error) { size := cfg.APIKeyConfig.LimitMin * cacheTimeoutMinute cache := newPrivilegesCache(cacheTimeoutMinute*time.Minute, size) - b.apikey = newApikeyBuilder(client, cache, []string{}) + b.apikey = newApikeyBuilder(client, cache, []elasticsearch.Privilege{}) b.fallback = DenyAuth{} } if cfg.SecretToken != "" { @@ -69,12 +74,12 @@ func NewBuilder(cfg *config.Config) (*Builder, error) { } // ForPrivilege creates an authorization Handler checking for this privilege -func (b *Builder) ForPrivilege(privilege string) *Handler { - return b.ForAnyOfPrivileges([]string{privilege}) +func (b *Builder) ForPrivilege(privilege elasticsearch.Privilege) *Handler { + return b.ForAnyOfPrivileges(privilege) } // ForAnyOfPrivileges creates an authorization Handler checking for any of the provided privileges -func (b *Builder) ForAnyOfPrivileges(privileges []string) *Handler { +func (b *Builder) ForAnyOfPrivileges(privileges ...elasticsearch.Privilege) *Handler { handler := Handler{bearer: b.bearer, fallback: b.fallback} if b.apikey != nil { handler.apikey = newApikeyBuilder(b.apikey.esClient, b.apikey.cache, privileges) diff --git a/beater/authorization/builder_test.go b/beater/authorization/builder_test.go index 0a896e85558..edf4d57db9d 100644 --- a/beater/authorization/builder_test.go +++ b/beater/authorization/builder_test.go @@ -41,8 +41,9 @@ func TestBuilder(t *testing.T) { } { setup := func() *Builder { - cfg := config.DefaultConfig("9.9.9") - + rawCfg := config.DefaultConfig("9.9.9") + cfg, err := config.Setup(nil, rawCfg, nil) + require.NoError(t, err) if tc.withBearer { cfg.SecretToken = "xvz" } @@ -51,7 +52,7 @@ func TestBuilder(t *testing.T) { Enabled: true, LimitMin: 100, ESConfig: elasticsearch.DefaultConfig()} } - builder, err := NewBuilder(cfg) + builder, err := NewBuilder(*cfg) require.NoError(t, err) return builder } @@ -68,12 +69,12 @@ func TestBuilder(t *testing.T) { t.Run("ForPrivilege"+name, func(t *testing.T) { builder := setup() - h := builder.ForPrivilege(PrivilegeSourcemapWrite) + h := builder.ForPrivilege(PrivilegeSourcemapWrite.Action) assert.Equal(t, builder.bearer, h.bearer) assert.Equal(t, builder.fallback, h.fallback) if tc.withApikey { - assert.Equal(t, []string{}, builder.apikey.anyOfPrivileges) - assert.Equal(t, []string{PrivilegeSourcemapWrite}, h.apikey.anyOfPrivileges) + assert.Equal(t, []elasticsearch.Privilege{}, builder.apikey.anyOfPrivileges) + assert.Equal(t, []elasticsearch.Privilege{PrivilegeSourcemapWrite.Action}, h.apikey.anyOfPrivileges) assert.Equal(t, builder.apikey.esClient, h.apikey.esClient) assert.Equal(t, builder.apikey.cache, h.apikey.cache) } @@ -81,7 +82,7 @@ func TestBuilder(t *testing.T) { t.Run("AuthorizationFor"+name, func(t *testing.T) { builder := setup() - h := builder.ForPrivilege(PrivilegeSourcemapWrite) + h := builder.ForPrivilege(PrivilegeSourcemapWrite.Action) auth := h.AuthorizationFor("ApiKey", "") if tc.withApikey { assert.IsType(t, &apikeyAuth{}, auth) diff --git a/beater/authorization/deny.go b/beater/authorization/deny.go index c9ec2ef2cb0..985226cb51a 100644 --- a/beater/authorization/deny.go +++ b/beater/authorization/deny.go @@ -17,11 +17,13 @@ package authorization +import "github.com/elastic/apm-server/elasticsearch" + // DenyAuth implements the Authorization interface. It denies all authorization requests. type DenyAuth struct{} // AuthorizedFor always returns false -func (DenyAuth) AuthorizedFor(_ string) (bool, error) { +func (DenyAuth) AuthorizedFor(_ elasticsearch.Resource) (bool, error) { return false, nil } diff --git a/beater/authorization/privilege.go b/beater/authorization/privilege.go index 7cc874e0bea..eea676ebc9c 100644 --- a/beater/authorization/privilege.go +++ b/beater/authorization/privilege.go @@ -20,22 +20,18 @@ package authorization import ( "time" - "github.com/patrickmn/go-cache" -) + es "github.com/elastic/apm-server/elasticsearch" -//Privileges -const ( - PrivilegeAgentConfigRead = "config_agent:read" - PrivilegeEventWrite = "event:write" - PrivilegeSourcemapWrite = "sourcemap:write" + "github.com/patrickmn/go-cache" ) var ( - //PrivilegesAll returns all available privileges - PrivilegesAll = []string{ - PrivilegeAgentConfigRead, - PrivilegeEventWrite, - PrivilegeSourcemapWrite} + PrivilegeAgentConfigRead = es.NewPrivilege("agentConfig", "config_agent:read") + PrivilegeEventWrite = es.NewPrivilege("event", "event:write") + PrivilegeSourcemapWrite = es.NewPrivilege("sourcemap", "sourcemap:write") + PrivilegesAll = []es.NamedPrivilege{PrivilegeAgentConfigRead, PrivilegeEventWrite, PrivilegeSourcemapWrite} + // ActionAny can't be used for querying + ActionAny = es.Privilege("*") ) type privilegesCache struct { @@ -43,8 +39,6 @@ type privilegesCache struct { size int } -type privileges map[string]bool - func newPrivilegesCache(expiration time.Duration, size int) *privilegesCache { return &privilegesCache{cache: cache.New(expiration, cleanupInterval), size: size} } @@ -53,13 +47,13 @@ func (c *privilegesCache) isFull() bool { return c.cache.ItemCount() >= c.size } -func (c *privilegesCache) get(id string) privileges { +func (c *privilegesCache) get(id string) es.Perms { if val, exists := c.cache.Get(id); exists { - return val.(privileges) + return val.(es.Perms) } return nil } -func (c *privilegesCache) add(id string, privileges privileges) { +func (c *privilegesCache) add(id string, privileges es.Perms) { c.cache.SetDefault(id, privileges) } diff --git a/beater/authorization/privilege_test.go b/beater/authorization/privilege_test.go index 81484d3df1b..f5849adff08 100644 --- a/beater/authorization/privilege_test.go +++ b/beater/authorization/privilege_test.go @@ -21,6 +21,8 @@ import ( "testing" "time" + "github.com/elastic/apm-server/elasticsearch" + "github.com/stretchr/testify/assert" ) @@ -29,16 +31,16 @@ func TestPrivilegesCache(t *testing.T) { cache := newPrivilegesCache(time.Millisecond, n) assert.False(t, cache.isFull()) for i := 0; i < n-1; i++ { - cache.add(string(i), privileges{}) + cache.add(string(i), elasticsearch.Perms{}) assert.False(t, cache.isFull()) } - cache.add("oneMore", privileges{}) + cache.add("oneMore", elasticsearch.Perms{}) assert.True(t, cache.isFull()) assert.NotNil(t, cache.get("oneMore")) time.Sleep(time.Millisecond) assert.Nil(t, cache.get("oneMore")) - p := privileges{"a": true, "b": false} + p := elasticsearch.Perms{"a": true, "b": false} cache.add("id1", p) assert.Equal(t, p, cache.get("id1")) assert.Nil(t, cache.get("oneMore")) diff --git a/beater/beater.go b/beater/beater.go index d6cba45ee10..0504083c325 100644 --- a/beater/beater.go +++ b/beater/beater.go @@ -95,6 +95,7 @@ func New(b *beat.Beat, ucfg *common.Config) (beat.Beater, error) { if isElasticsearchOutput(b) { esOutputCfg = b.Config.Output.Config() } + beaterConfig, err := config.NewConfig(b.Info.Version, ucfg, esOutputCfg) if err != nil { return nil, err diff --git a/beater/config/api_key.go b/beater/config/api_key.go index 0c26c743493..3e4ed840589 100644 --- a/beater/config/api_key.go +++ b/beater/config/api_key.go @@ -53,9 +53,7 @@ func (c *APIKeyConfig) setup(log *logp.Logger, outputESCfg *common.Config) error if err := outputESCfg.Unpack(c.ESConfig); err != nil { return errors.Wrap(err, "unpacking Elasticsearch config into API key config") } - // do not use username+password for API Key requests - c.ESConfig.Username = "" - c.ESConfig.Password = "" + return nil } diff --git a/beater/config/config.go b/beater/config/config.go index 98cce679dcd..4321c3932d5 100644 --- a/beater/config/config.go +++ b/beater/config/config.go @@ -68,6 +68,8 @@ type Config struct { Pipeline string } +type RawConfig Config + // ExpvarConfig holds config information about exposing expvar type ExpvarConfig struct { Enabled *bool `config:"enabled"` @@ -84,9 +86,8 @@ type Cache struct { Expiration time.Duration `config:"expiration"` } -// NewConfig creates a Config struct based on the default config and the given input params -func NewConfig(version string, ucfg *common.Config, outputESCfg *common.Config) (*Config, error) { - logger := logp.NewLogger(logs.Config) +// NewRawConfig unpacks the ucfg data +func NewRawConfig(version string, ucfg *common.Config) (*RawConfig, error) { c := DefaultConfig(version) if ucfg.HasField("ssl") { @@ -104,6 +105,16 @@ func NewConfig(version string, ucfg *common.Config, outputESCfg *common.Config) if err := ucfg.Unpack(c); err != nil { return nil, errors.Wrap(err, "Error processing configuration") } + return c, nil +} + +// Setup contains ad-hoc logic for apm-server configuration +func Setup(rawConfig *RawConfig, outputESCfg *common.Config) (*Config, error) { + logger := logp.NewLogger(logs.Config) + if rawConfig == nil { + return nil, nil + } + c := Config(*rawConfig) if float64(int(c.AgentConfig.Cache.Expiration.Seconds())) != c.AgentConfig.Cache.Expiration.Seconds() { return nil, errors.New(msgInvalidConfigAgentCfg) @@ -125,7 +136,15 @@ func NewConfig(version string, ucfg *common.Config, outputESCfg *common.Config) return nil, err } - return c, nil + return &c, nil +} + +func NewConfig(version string, ucfg *common.Config, outputESCfg *common.Config) (*Config, error) { + raw, err := NewRawConfig(version, ucfg) + if err != nil { + return nil, err + } + return Setup(raw, outputESCfg) } // IsEnabled indicates whether expvar is enabled or not @@ -134,8 +153,8 @@ func (c *ExpvarConfig) IsEnabled() bool { } // DefaultConfig returns a config with default settings for `apm-server` config options. -func DefaultConfig(beatVersion string) *Config { - return &Config{ +func DefaultConfig(beatVersion string) *RawConfig { + return &RawConfig{ Host: net.JoinHostPort("localhost", DefaultPort), MaxHeaderSize: 1 * 1024 * 1024, // 1mb MaxConnections: 0, // unlimited diff --git a/beater/config/config_test.go b/beater/config/config_test.go index 867b411ef8b..cc79951fc2a 100644 --- a/beater/config/config_test.go +++ b/beater/config/config_test.go @@ -37,14 +37,15 @@ import ( func Test_UnpackConfig(t *testing.T) { falsy, truthy := false, true version := "8.0.0" - + defaultRawConfig := DefaultConfig(version) + defaultConfig, _ := Setup(nil, defaultRawConfig, nil) tests := map[string]struct { inpCfg map[string]interface{} outCfg *Config }{ "default config": { inpCfg: map[string]interface{}{}, - outCfg: DefaultConfig(version), + outCfg: defaultConfig, }, "overwrite default": { inpCfg: map[string]interface{}{ @@ -239,7 +240,8 @@ func Test_UnpackConfig(t *testing.T) { inpCfg, err := common.NewConfigFrom(test.inpCfg) assert.NoError(t, err) - cfg, err := NewConfig(version, inpCfg, nil) + rawCfg, err := NewRawConfig(version, inpCfg) + cfg, err := Setup(nil, rawCfg, err) require.NoError(t, err) require.NotNil(t, cfg) assert.Equal(t, test.outCfg, cfg) @@ -320,7 +322,7 @@ func TestTLSSettings(t *testing.T) { ucfgCfg, err := common.NewConfigFrom(tc.config) require.NoError(t, err) - cfg, err := NewConfig("9.9.9", ucfgCfg, nil) + cfg, err := NewRawConfig("9.9.9", ucfgCfg) require.NoError(t, err) assert.Equal(t, tc.tls.ClientAuth, cfg.TLS.ClientAuth) }) @@ -346,7 +348,7 @@ func TestTLSSettings(t *testing.T) { ucfgCfg, err := common.NewConfigFrom(tc.config) require.NoError(t, err) - cfg, err := NewConfig("9.9.9", ucfgCfg, nil) + cfg, err := NewRawConfig("9.9.9", ucfgCfg) require.NoError(t, err) assert.Equal(t, tc.tls.VerificationMode, cfg.TLS.VerificationMode) }) @@ -378,22 +380,25 @@ func TestTLSSettings(t *testing.T) { func TestAgentConfig(t *testing.T) { t.Run("InvalidValueTooSmall", func(t *testing.T) { - cfg, err := NewConfig("9.9.9", - common.MustNewConfigFrom(map[string]string{"agent.config.cache.expiration": "123ms"}), nil) + rawCfg, err := NewRawConfig("9.9.9", + common.MustNewConfigFrom(map[string]string{"agent.config.cache.expiration": "123ms"})) + cfg, err := Setup(nil, rawCfg, err) require.Error(t, err) assert.Nil(t, cfg) }) t.Run("InvalidUnit", func(t *testing.T) { - cfg, err := NewConfig("9.9.9", - common.MustNewConfigFrom(map[string]string{"agent.config.cache.expiration": "1230ms"}), nil) + rawCfg, err := NewRawConfig("9.9.9", + common.MustNewConfigFrom(map[string]string{"agent.config.cache.expiration": "1230ms"})) + cfg, err := Setup(nil, rawCfg, err) require.Error(t, err) assert.Nil(t, cfg) }) t.Run("Valid", func(t *testing.T) { - cfg, err := NewConfig("9.9.9", - common.MustNewConfigFrom(map[string]string{"agent.config.cache.expiration": "123000ms"}), nil) + rawCfg, err := NewRawConfig("9.9.9", + common.MustNewConfigFrom(map[string]string{"agent.config.cache.expiration": "123000ms"})) + cfg, err := Setup(nil, rawCfg, err) require.NoError(t, err) assert.Equal(t, time.Second*123, cfg.AgentConfig.Cache.Expiration) }) @@ -405,14 +410,16 @@ func TestNewConfig_ESConfig(t *testing.T) { require.NoError(t, err) // no es config given - cfg, err := NewConfig(version, ucfg, nil) + rawCfg, err := NewRawConfig(version, ucfg) + cfg, err := Setup(nil, rawCfg, err) require.NoError(t, err) assert.Nil(t, cfg.RumConfig.SourceMapping.ESConfig) assert.Equal(t, elasticsearch.DefaultConfig(), cfg.APIKeyConfig.ESConfig) // with es config outputESCfg := common.MustNewConfigFrom(`{"hosts":["192.0.0.168:9200"]}`) - cfg, err = NewConfig(version, ucfg, outputESCfg) + rawCfg, err = NewRawConfig(version, ucfg) + cfg, err = Setup(outputESCfg, rawCfg, err) require.NoError(t, err) assert.NotNil(t, cfg.RumConfig.SourceMapping.ESConfig) assert.Equal(t, []string{"192.0.0.168:9200"}, []string(cfg.RumConfig.SourceMapping.ESConfig.Hosts)) diff --git a/beater/middleware/authorization_middleware.go b/beater/middleware/authorization_middleware.go index a694cfa56d0..75a5a5751e9 100644 --- a/beater/middleware/authorization_middleware.go +++ b/beater/middleware/authorization_middleware.go @@ -28,13 +28,12 @@ import ( // AuthorizationMiddleware returns a Middleware to only let authorized requests pass through func AuthorizationMiddleware(auth *authorization.Handler, apply bool) Middleware { - resource := authorization.DefaultResource return func(h request.Handler) (request.Handler, error) { return func(c *request.Context) { c.Authorization = auth.AuthorizationFor(fetchAuthHeader(c.Request)) if apply { - authorized, err := c.Authorization.AuthorizedFor(resource) + authorized, err := c.Authorization.AuthorizedFor(authorization.ResourceInternal) if !authorized { c.Result.SetDeniedAuthorization(err) c.Write() diff --git a/cmd/apikey.go b/cmd/apikey.go new file mode 100644 index 00000000000..26e3d134ea6 --- /dev/null +++ b/cmd/apikey.go @@ -0,0 +1,326 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package cmd + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "math" + "os" + "strings" + "time" + + "github.com/pkg/errors" + + "github.com/elastic/apm-server/beater/config" + "github.com/elastic/apm-server/beater/headers" + + auth "github.com/elastic/apm-server/beater/authorization" + es "github.com/elastic/apm-server/elasticsearch" +) + +// creates an API Key with the given privileges, *AND* all the privileges modeled in apm-server +// we need to ensure forward-compatibility, for which future privileges must be created here and +// during server startup because we don't know if customers will run this command +func createApiKeyWithPrivileges(client es.Client, apikeyName, expiry string, privileges []es.Privilege, asJson bool) error { + var privilegesRequest = make(es.CreatePrivilegesRequest) + event := auth.PrivilegeEventWrite + agentConfig := auth.PrivilegeAgentConfigRead + sourcemap := auth.PrivilegeSourcemapWrite + privilegesRequest[auth.Application] = map[es.PrivilegeName]es.Actions{ + agentConfig.Name: {[]es.Privilege{agentConfig.Action}}, + event.Name: {[]es.Privilege{event.Action}}, + sourcemap.Name: {[]es.Privilege{sourcemap.Action}}, + } + + privilegesCreated, err := es.CreatePrivileges(client, privilegesRequest) + + if err != nil { + return printErr(err, + `Error creating privileges for APM Server, do you have the "manage_cluster" security privilege?`, + asJson) + } + + printText, printJson := printers(asJson) + for privilege, result := range privilegesCreated[auth.Application] { + if result.Created { + printText("Security privilege \"%v\" created", privilege) + } + } + + apikeyRequest := es.CreateApiKeyRequest{ + Name: apikeyName, + RoleDescriptors: es.RoleDescriptor{ + auth.Application: es.Applications{ + Applications: []es.Application{ + { + Name: auth.Application, + Privileges: privileges, + Resources: []es.Resource{auth.ResourceAny}, + }, + }, + }, + }, + } + if expiry != "" { + apikeyRequest.Expiration = &expiry + } + + apikey, err := es.CreateAPIKey(client, apikeyRequest) + if err != nil { + return printErr(err, fmt.Sprintf( + `Error creating the API Key %s, do you have the "manage_cluster" security privilege?`, apikeyName), + asJson) + } + credentials := base64.StdEncoding.EncodeToString([]byte(apikey.Id + ":" + apikey.Key)) + apikey.Credentials = &credentials + printText("API Key created:") + printText("") + printText("Name ........... %s", apikey.Name) + printText("Expiration ..... %s", humanTime(apikey.ExpirationMs)) + printText("Id ............. %s", apikey.Id) + printText("API Key ........ %s (won't be shown again)", apikey.Key) + printText(`Credentials .... %s (use it as "Authorization: ApiKey " header to communicate with APM Server, won't be shown again)`, + credentials) + + return printJson(struct { + es.CreateApiKeyResponse + Privileges es.CreatePrivilegesResponse `json:"created_privileges,omitempty"` + }{ + CreateApiKeyResponse: apikey, + Privileges: privilegesCreated, + }) +} + +func getApiKey(client es.Client, id, name *string, validOnly, asJson bool) error { + if id != nil { + name = nil + } else if id == nil && name == nil { + return printErr(errors.New("could not query Elasticsearch"), + `either "id" or "name" are required`, + asJson) + } + request := es.GetApiKeyRequest{ + ApiKeyQuery: es.ApiKeyQuery{ + Id: id, + Name: name, + }, + } + + apikeys, err := es.GetAPIKeys(client, request) + if err != nil { + return printErr(err, + `Error retrieving API Key(s) for APM Server, do you have the "manage_cluster" security privilege?`, + asJson) + } + + transform := es.GetApiKeyResponse{ApiKeys: make([]es.ApiKeyResponse, 0)} + printText, printJson := printers(asJson) + for _, apikey := range apikeys.ApiKeys { + expiry := humanTime(apikey.ExpirationMs) + if validOnly && (apikey.Invalidated || expiry == "expired") { + continue + } + creation := time.Unix(apikey.Creation/1000, 0).Format("2006-02-01 15:04") + printText("Username ....... %s", apikey.Username) + printText("Api Key Name ... %s", apikey.Name) + printText("Id ............. %s", apikey.Id) + printText("Creation ....... %s", creation) + printText("Invalidated .... %t", apikey.Invalidated) + printText("Expiration ..... %s", expiry) + printText("") + transform.ApiKeys = append(transform.ApiKeys, apikey) + } + printText("%d API Keys found", len(transform.ApiKeys)) + return printJson(transform) +} + +func invalidateApiKey(client es.Client, id, name *string, deletePrivileges, asJson bool) error { + if id != nil { + name = nil + } else if id == nil && name == nil { + return printErr(errors.New("could not query Elasticsearch"), + `either "id" or "name" are required`, + asJson) + } + invalidateKeysRequest := es.InvalidateApiKeyRequest{ + ApiKeyQuery: es.ApiKeyQuery{ + Id: id, + Name: name, + }, + } + + invalidation, err := es.InvalidateAPIKey(client, invalidateKeysRequest) + if err != nil { + return printErr(err, + `Error invalidating API Key(s), do you have the "manage_cluster" security privilege?`, + asJson) + } + printText, printJson := printers(asJson) + out := struct { + es.InvalidateApiKeyResponse + Privileges []es.DeletePrivilegeResponse `json:"deleted_privileges,omitempty"` + }{ + InvalidateApiKeyResponse: invalidation, + Privileges: make([]es.DeletePrivilegeResponse, 0), + } + printText("Invalidated keys ... %s", strings.Join(invalidation.Invalidated, ", ")) + printText("Error count ........ %d", invalidation.ErrorCount) + + for _, privilege := range auth.PrivilegesAll { + if !deletePrivileges { + break + } + deletePrivilegesRequest := es.DeletePrivilegeRequest{ + Application: auth.Application, + Privilege: privilege.Name, + } + + deletion, err := es.DeletePrivileges(client, deletePrivilegesRequest) + if err != nil { + continue + } + if _, ok := deletion[auth.Application]; !ok { + continue + } + if result, ok := deletion[auth.Application][privilege.Name]; ok && result.Found { + printText("Deleted privilege \"%v\"", privilege) + } + out.Privileges = append(out.Privileges, deletion) + } + return printJson(out) +} + +func verifyApiKey(config *config.Config, privileges []es.Privilege, credentials string, asJson bool) error { + builder, err := auth.NewBuilder(*config) + perms := make(es.Perms) + + printText, printJson := printers(asJson) + + for _, privilege := range privileges { + if err != nil { + break + } + var authorized bool + authorized, err = builder. + ForPrivilege(privilege). + AuthorizationFor(headers.APIKey, credentials). + AuthorizedFor(auth.ResourceInternal) + perms[privilege] = authorized + printText("Authorized for %s...: %s\n", humanPrivilege(privilege), humanBool(authorized)) + } + + if err != nil { + return printErr(err, "could not verify credentials, please check your Elasticsearch connection", asJson) + } + return printJson(perms) +} + +func humanBool(b bool) string { + if b { + return "Yes" + } + return "No" +} + +func humanPrivilege(privilege es.Privilege) string { + switch privilege { + case auth.ActionAny: + return fmt.Sprintf("all privileges (\"%v\")", privilege) + default: + return fmt.Sprintf("privilege \"%v\"", privilege) + } +} + +func humanTime(millis *int64) string { + if millis == nil { + return fmt.Sprint("never") + } + seconds := time.Until(time.Unix(*millis/1000, 0)).Seconds() + if seconds < 0 { + return fmt.Sprintf("expired") + } + minutes := math.Round(seconds / 60) + if minutes < 2 { + return fmt.Sprintf("%.0f seconds", seconds) + } + hours := math.Round(minutes / 60) + if hours < 2 { + return fmt.Sprintf("%.0f minutes", minutes) + } + days := math.Round(hours / 24) + if days < 2 { + return fmt.Sprintf("%.0f hours", hours) + } + years := math.Round(days / 365) + if years < 2 { + return fmt.Sprintf("%.0f days", days) + } + return fmt.Sprintf("%.0f years", years) +} + +// returns 2 printers, one for text and one for JSON +// one of them will be a noop based on the boolean argument +func printers(b bool) (func(string, ...interface{}), func(interface{}) error) { + var w1 io.Writer = os.Stdout + var w2 = ioutil.Discard + if b { + w1 = ioutil.Discard + w2 = os.Stdout + } + return func(f string, i ...interface{}) { + fmt.Fprintf(w1, f, i...) + fmt.Fprintln(w1) + }, func(i interface{}) error { + data, err := json.MarshalIndent(i, "", "\t") + fmt.Fprintln(w2, string(data)) + // conform the interface + return errors.Wrap(err, fmt.Sprintf("%v+", i)) + } +} + +// prints an Elasticsearch error to stderr, with some additional contextual information as a hint +func printErr(err error, help string, asJson bool) error { + if asJson { + var data []byte + var m map[string]interface{} + e := json.Unmarshal([]byte(err.Error()), &m) + if e == nil { + // err.Error() has JSON shape, likely coming from Elasticsearch + m["help"] = help + data, _ = json.MarshalIndent(m, "", "\t") + } else { + // err.Error() is a bare string, likely coming from apm-server + data, _ = json.MarshalIndent(struct { + Error string `json:"error"` + Help string `json:"help"` + }{ + Error: err.Error(), + Help: help, + }, "", "\t") + } + fmt.Fprintln(os.Stderr, string(data)) + } else { + fmt.Fprintln(os.Stderr, help) + fmt.Fprintln(os.Stderr, err.Error()) + } + return errors.Wrap(err, help) +} diff --git a/cmd/root.go b/cmd/root.go index 57871de4124..05211d48e46 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -20,6 +20,13 @@ package cmd import ( "fmt" + "github.com/spf13/cobra" + + auth "github.com/elastic/apm-server/beater/authorization" + + "github.com/elastic/apm-server/beater/config" + es "github.com/elastic/apm-server/elasticsearch" + "github.com/elastic/beats/libbeat/cfgfile" "github.com/spf13/pflag" @@ -43,7 +50,11 @@ const IdxPattern = "apm" // RootCmd for running apm-server. // This is the command that is used if no other command is specified. // Running `apm-server run` or `apm-server` is identical. -var RootCmd *cmd.BeatsRootCmd +type ApmCmd struct { + *cmd.BeatsRootCmd +} + +var RootCmd = ApmCmd{} func init() { overrides := common.MustNewConfigFrom(map[string]interface{}{ @@ -92,8 +103,8 @@ func init() { }, }, } - RootCmd = cmd.GenRootCmdWithSettings(beater.New, settings) - + RootCmd = ApmCmd{cmd.GenRootCmdWithSettings(beater.New, settings)} + RootCmd.AddCommand(genApikeyCmd(settings)) for _, cmd := range RootCmd.ExportCmd.Commands() { // remove `dashboard` from `export` commands @@ -126,3 +137,210 @@ func init() { setup.Flags().Bool(cmd.PipelineKey, false, "Setup ingest pipelines") } + +func genApikeyCmd(settings instance.Settings) *cobra.Command { + + var client es.Client + apmConfig, err := bootstrap(settings) + if err == nil { + client, err = es.NewClient(apmConfig.APIKeyConfig.ESConfig) + } + + short := "Manage API Keys for communication between APM agents and server" + apikeyCmd := cobra.Command{ + Use: "apikey", + Short: short, + Long: short + `. +Most operations require the "manage_security" cluster privilege. Ensure to configure "apm-server.api_key.*" or +"output.elasticsearch.*" appropriately. APM Server will create security privileges for the "apm" application; +you can freely query them. If you modify or delete apm privileges, APM Server might reject all requests. +If an invalid argument is passed, nothing will be printed. +Check the Elastic Security API documentation for details.`, + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + return err + }, + } + + apikeyCmd.AddCommand( + createApikeyCmd(client), + invalidateApikeyCmd(client), + getApikeysCmd(client), + verifyApikeyCmd(apmConfig), + ) + + return &apikeyCmd +} + +func createApikeyCmd(client es.Client) *cobra.Command { + var keyName, expiration string + var ingest, sourcemap, agentConfig, json bool + short := "Create an API Key with the specified privilege(s)" + create := &cobra.Command{ + Use: "create", + Short: short, + Long: short + `. +If no privilege(s) are specified, the API Key will be valid for all. +Requires the "manage_security" cluster privilege in Elasticsearch.`, + // always need to return error for possible scripts checking the exit code, + // but printing the error must be done inside + RunE: func(cmd *cobra.Command, args []string) error { + privileges := booleansToPrivileges(ingest, sourcemap, agentConfig) + return createApiKeyWithPrivileges(client, keyName, expiration, privileges, json) + }, + // these are needed to not break JSON formatting + // this has the caveat that if an invalid argument is passed, the command won't return anything + SilenceUsage: true, + SilenceErrors: true, + } + create.Flags().StringVar(&keyName, "name", "apm-key", "API Key name") + create.Flags().StringVar(&expiration, "expiration", "", + `expiration for the key, eg. "1d" (default never)`) + create.Flags().BoolVar(&ingest, "ingest", false, + fmt.Sprintf("give the %v privilege to this key, required for ingesting events", auth.PrivilegeEventWrite)) + create.Flags().BoolVar(&sourcemap, "sourcemap", false, + fmt.Sprintf("give the %v privilege to this key, required for uploading sourcemaps", + auth.PrivilegeSourcemapWrite)) + create.Flags().BoolVar(&agentConfig, "agent-config", false, + fmt.Sprintf("give the %v privilege to this key, required for agents to read configuration remotely", + auth.PrivilegeAgentConfigRead)) + create.Flags().BoolVar(&json, "json", false, + "prints the output of this command as JSON") + // this actually means "preserve sorting given in code" and not reorder them alphabetically + create.Flags().SortFlags = false + return create +} + +func invalidateApikeyCmd(client es.Client) *cobra.Command { + var id, name string + var purge, json bool + short := "Invalidate API Key(s) by Id or Name" + invalidate := &cobra.Command{ + Use: "invalidate", + Short: short, + Long: short + `. +If both "id" and "name" are supplied, only "id" will be used. +If neither of them are, an error will be returned. +Requires the "manage_security" cluster privilege in Elasticsearch.`, + RunE: func(cmd *cobra.Command, args []string) error { + return invalidateApiKey(client, &id, &name, purge, json) + }, + SilenceErrors: true, + SilenceUsage: true, + } + invalidate.Flags().StringVar(&id, "id", "", "id of the API Key to delete") + invalidate.Flags().StringVar(&name, "name", "", + "name of the API Key(s) to delete (several might match)") + invalidate.Flags().BoolVar(&purge, "purge", false, + "also remove all privileges created and used by APM Server") + invalidate.Flags().BoolVar(&json, "json", false, + "prints the output of this command as JSON") + invalidate.Flags().SortFlags = false + return invalidate +} + +func getApikeysCmd(client es.Client) *cobra.Command { + var id, name string + var validOnly, json bool + short := "Query API Key(s) by Id or Name" + info := &cobra.Command{ + Use: "info", + Short: short, + Long: short + `. +If both "id" and "name" are supplied, only "id" will be used. +If neither of them are, an error will be returned. +Requires the "manage_security" cluster privilege in Elasticsearch.`, + RunE: func(cmd *cobra.Command, args []string) error { + return getApiKey(client, &id, &name, validOnly, json) + }, + SilenceErrors: true, + SilenceUsage: true, + } + info.Flags().StringVar(&id, "id", "", "id of the API Key to query") + info.Flags().StringVar(&name, "name", "", + "name of the API Key(s) to query (several might match)") + info.Flags().BoolVar(&validOnly, "valid-only", false, + "only return valid API Keys (not expired or invalidated)") + info.Flags().BoolVar(&json, "json", false, + "prints the output of this command as JSON") + info.Flags().SortFlags = false + return info +} + +func verifyApikeyCmd(config *config.Config) *cobra.Command { + var credentials string + var ingest, sourcemap, agentConfig, json bool + short := `Check if a "credentials" string has the given privilege(s)` + long := short + `. +If no privilege(s) are specified, the credentials will be queried for all.` + verify := &cobra.Command{ + Use: "verify", + Short: short, + Long: long, + RunE: func(cmd *cobra.Command, args []string) error { + privileges := booleansToPrivileges(ingest, sourcemap, agentConfig) + return verifyApiKey(config, privileges, credentials, json) + }, + SilenceUsage: true, + SilenceErrors: true, + } + verify.Flags().StringVar(&credentials, "credentials", "", `credentials for which check privileges`) + verify.Flags().BoolVar(&ingest, "ingest", false, + fmt.Sprintf("ask for the %v privilege, required for ingesting events", auth.PrivilegeEventWrite)) + verify.Flags().BoolVar(&sourcemap, "sourcemap", false, + fmt.Sprintf("ask for the %v privilege, required for uploading sourcemaps", + auth.PrivilegeSourcemapWrite)) + verify.Flags().BoolVar(&agentConfig, "agent-config", false, + fmt.Sprintf("ask for the %v privilege, required for agents to read configuration remotely", + auth.PrivilegeAgentConfigRead)) + verify.Flags().BoolVar(&json, "json", false, + "prints the output of this command as JSON") + verify.Flags().SortFlags = false + + return verify +} + +// created the beat, instantiate configuration, and so on +// apm-server.api_key.enabled is implicitly true +func bootstrap(settings instance.Settings) (*config.Config, error) { + beat, err := instance.NewBeat(settings.Name, settings.IndexPrefix, settings.Version) + if err != nil { + return nil, err + } + err = beat.InitWithSettings(settings) + if err != nil { + return nil, err + } + + outCfg := beat.Config.Output + apmRawConfig, err := config.NewRawConfig(settings.Version, beat.RawConfig) + if apmRawConfig == nil { + return nil, err + } + if apmRawConfig.APIKeyConfig == nil { + apmRawConfig.APIKeyConfig = &config.APIKeyConfig{} + } + apmRawConfig.APIKeyConfig.Enabled = true + + return config.Setup(apmRawConfig, outCfg.Config()) +} + +// if all are false, returns any ("*") +// this is because Elasticsearch requires at least 1 privilege for most queries, +// so "*" acts as default +func booleansToPrivileges(ingest, sourcemap, agentConfig bool) []es.Privilege { + privileges := make([]es.Privilege, 0) + if ingest { + privileges = append(privileges, auth.PrivilegeEventWrite.Action) + } + if sourcemap { + privileges = append(privileges, auth.PrivilegeSourcemapWrite.Action) + } + if agentConfig { + privileges = append(privileges, auth.PrivilegeAgentConfigRead.Action) + } + any := ingest || sourcemap || agentConfig + if !any { + privileges = append(privileges, auth.ActionAny) + } + return privileges +} diff --git a/elasticsearch/client.go b/elasticsearch/client.go index eefd4abbf9f..1c920137272 100644 --- a/elasticsearch/client.go +++ b/elasticsearch/client.go @@ -18,81 +18,88 @@ package elasticsearch import ( + "bytes" "context" + "encoding/json" + "errors" "io" + "io/ioutil" "net/http" + "net/url" + "strings" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/version" v7 "github.com/elastic/go-elasticsearch/v7" - v7esapi "github.com/elastic/go-elasticsearch/v7/esapi" v8 "github.com/elastic/go-elasticsearch/v8" - v8esapi "github.com/elastic/go-elasticsearch/v8/esapi" ) // Client is an interface designed to abstract away version differences between elasticsearch clients type Client interface { + // TODO: deprecate // Search performs a query against the given index with the given body Search(index string, body io.Reader) (int, io.ReadCloser, error) - SecurityHasPrivilegesRequest(body io.Reader, header http.Header) (int, io.ReadCloser, error) + // Makes a request with application/json Content-Type and Accept headers by default + // pass/overwrite headers with "key: value" format + JSONRequest(method, path string, body interface{}, headers ...string) JSONResponse } type clientV8 struct { - client *v8.Client + c *v8.Client } // Search satisfies the Client interface for version 8 -func (c clientV8) Search(index string, body io.Reader) (int, io.ReadCloser, error) { - return v8Response(c.client.Search( - c.client.Search.WithContext(context.Background()), - c.client.Search.WithIndex(index), - c.client.Search.WithBody(body), - c.client.Search.WithTrackTotalHits(true), - c.client.Search.WithPretty(), - )) -} - -func (c clientV8) SecurityHasPrivilegesRequest(body io.Reader, header http.Header) (int, io.ReadCloser, error) { - hasPrivileges := v8esapi.SecurityHasPrivilegesRequest{Body: body, Header: header} - return v8Response(hasPrivileges.Do(context.Background(), c.client)) -} - -func v8Response(response *v8esapi.Response, err error) (int, io.ReadCloser, error) { +func (v8 clientV8) Search(index string, body io.Reader) (int, io.ReadCloser, error) { + response, err := v8.c.Search( + v8.c.Search.WithContext(context.Background()), + v8.c.Search.WithIndex(index), + v8.c.Search.WithBody(body), + v8.c.Search.WithTrackTotalHits(true), + v8.c.Search.WithPretty(), + ) if err != nil { return 0, nil, err } return response.StatusCode, response.Body, nil } -type clientV7 struct { - client *v7.Client -} - -// Search satisfies the Client interface for version 7 -func (c clientV7) Search(index string, body io.Reader) (int, io.ReadCloser, error) { - return v7Response(c.client.Search( - c.client.Search.WithContext(context.Background()), - c.client.Search.WithIndex(index), - c.client.Search.WithBody(body), - c.client.Search.WithTrackTotalHits(true), - c.client.Search.WithPretty(), - )) +func (v8 clientV8) JSONRequest(method, path string, body interface{}, headers ...string) JSONResponse { + req, err := makeRequest(method, path, body, headers...) + if err != nil { + return JSONResponse{nil, err} + } + return parseResponse(v8.c.Perform(req)) } -func (c clientV7) SecurityHasPrivilegesRequest(body io.Reader, header http.Header) (int, io.ReadCloser, error) { - hasPrivileges := v7esapi.SecurityHasPrivilegesRequest{Body: body, Header: header} - return v7Response(hasPrivileges.Do(context.Background(), c.client)) +type clientV7 struct { + c *v7.Client } -func v7Response(response *v7esapi.Response, err error) (int, io.ReadCloser, error) { +// Search satisfies the Client interface for version 7 +func (v7 clientV7) Search(index string, body io.Reader) (int, io.ReadCloser, error) { + response, err := v7.c.Search( + v7.c.Search.WithContext(context.Background()), + v7.c.Search.WithIndex(index), + v7.c.Search.WithBody(body), + v7.c.Search.WithTrackTotalHits(true), + v7.c.Search.WithPretty(), + ) if err != nil { return 0, nil, err } return response.StatusCode, response.Body, nil } +func (v7 clientV7) JSONRequest(method, path string, body interface{}, headers ...string) JSONResponse { + req, err := makeRequest(method, path, body, headers...) + if err != nil { + return JSONResponse{nil, err} + } + return parseResponse(v7.c.Perform(req)) +} + // NewClient parses the given config and returns a version-aware client as an interface func NewClient(config *Config) (Client, error) { if config == nil { @@ -135,3 +142,65 @@ func newV8Client(apikey, user, pwd string, addresses []string, transport http.Ro Transport: transport, }) } + +type JSONResponse struct { + content io.ReadCloser + err error +} + +func (r JSONResponse) DecodeTo(i interface{}) error { + if r.err != nil { + return r.err + } + bs, err := ioutil.ReadAll(r.content) + if err != nil { + return err + } + err = json.Unmarshal(bs, i) + return err +} + +// each header has the format "key: value" +func makeRequest(method, path string, body interface{}, headers ...string) (*http.Request, error) { + header := http.Header{ + "Content-Type": []string{"application/json"}, + "Accept": []string{"application/json"}, + } + for _, h := range headers { + kv := strings.Split(h, ":") + if len(kv) == 2 { + header[kv[0]] = strings.Split(kv[1], ",") + } + } + u, _ := url.Parse(path) + req := &http.Request{ + Method: method, + URL: u, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Header: header, + } + bs, err := json.Marshal(body) + if err != nil { + return nil, err + } + if body != nil { + req.Body = ioutil.NopCloser(bytes.NewReader(bs)) + req.ContentLength = int64(len(bs)) + } + return req, nil +} + +func parseResponse(resp *http.Response, err error) JSONResponse { + if err != nil { + return JSONResponse{nil, err} + } + body := resp.Body + if resp.StatusCode >= http.StatusMultipleChoices { + buf := new(bytes.Buffer) + buf.ReadFrom(body) + return JSONResponse{nil, errors.New(buf.String())} + } + return JSONResponse{body, nil} +} diff --git a/elasticsearch/security_api.go b/elasticsearch/security_api.go new file mode 100644 index 00000000000..2b9d0dbe369 --- /dev/null +++ b/elasticsearch/security_api.go @@ -0,0 +1,220 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package elasticsearch + +import ( + "fmt" + "net/http" + "net/url" + "strconv" +) + +// requires manage_security cluster privilege +func CreateAPIKey(client Client, apikeyReq CreateApiKeyRequest) (CreateApiKeyResponse, error) { + response := client.JSONRequest(http.MethodPut, "/_security/api_key", apikeyReq) + + var apikey CreateApiKeyResponse + err := response.DecodeTo(&apikey) + return apikey, err +} + +// requires manage_security cluster privilege +func GetAPIKeys(client Client, apikeyReq GetApiKeyRequest) (GetApiKeyResponse, error) { + u := url.URL{Path: "/_security/api_key"} + params := url.Values{} + params.Set("owner", strconv.FormatBool(apikeyReq.Owner)) + if apikeyReq.Id != nil { + params.Set("id", *apikeyReq.Id) + } else if apikeyReq.Name != nil { + params.Set("name", *apikeyReq.Name) + } + u.RawQuery = params.Encode() + + response := client.JSONRequest(http.MethodGet, u.String(), nil) + + var apikey GetApiKeyResponse + err := response.DecodeTo(&apikey) + return apikey, err +} + +// requires manage_security cluster privilege +func CreatePrivileges(client Client, privilegesReq CreatePrivilegesRequest) (CreatePrivilegesResponse, error) { + response := client.JSONRequest(http.MethodPut, "/_security/privilege", privilegesReq) + + var privileges CreatePrivilegesResponse + err := response.DecodeTo(&privileges) + return privileges, err +} + +// requires manage_security cluster privilege +func InvalidateAPIKey(client Client, apikeyReq InvalidateApiKeyRequest) (InvalidateApiKeyResponse, error) { + response := client.JSONRequest(http.MethodDelete, "/_security/api_key", apikeyReq) + + var confirmation InvalidateApiKeyResponse + err := response.DecodeTo(&confirmation) + return confirmation, err +} + +func DeletePrivileges(client Client, privilegesReq DeletePrivilegeRequest) (DeletePrivilegeResponse, error) { + // requires manage_security cluster privilege + path := fmt.Sprintf("/_security/privilege/%v/%v", privilegesReq.Application, privilegesReq.Privilege) + response := client.JSONRequest(http.MethodDelete, path, nil) + + var confirmation DeletePrivilegeResponse + err := response.DecodeTo(&confirmation) + return confirmation, err +} + +func HasPrivileges(client Client, privileges HasPrivilegesRequest, credentials string) (HasPrivilegesResponse, error) { + h := fmt.Sprintf("Authorization: ApiKey %s", credentials) + response := client.JSONRequest(http.MethodGet, "/_security/user/_has_privileges", privileges, h) + + var info HasPrivilegesResponse + err := response.DecodeTo(&info) + return info, err +} + +type CreateApiKeyRequest struct { + Name string `json:"name"` + Expiration *string `json:"expiration,omitempty"` + RoleDescriptors RoleDescriptor `json:"role_descriptors"` +} + +type CreateApiKeyResponse struct { + ApiKey + Key string `json:"api_key"` +} + +type GetApiKeyRequest struct { + ApiKeyQuery + Owner bool `json:"owner"` +} + +type GetApiKeyResponse struct { + ApiKeys []ApiKeyResponse `json:"api_keys"` +} + +type CreatePrivilegesRequest map[AppName]PrivilegeGroup + +type CreatePrivilegesResponse map[AppName]PrivilegeResponse + +type HasPrivilegesRequest struct { + // can't reuse the `Applications` type because here the JSON attribute must be singular + Applications []Application `json:"application"` +} +type HasPrivilegesResponse struct { + Username string `json:"username"` + HasAll bool `json:"has_all_requested"` + Application map[AppName]PrivilegesPerResource `json:"application"` +} + +type InvalidateApiKeyRequest struct { + ApiKeyQuery +} + +type InvalidateApiKeyResponse struct { + Invalidated []string `json:"invalidated_api_keys"` + ErrorCount int `json:"error_count"` +} + +type DeletePrivilegeRequest struct { + Application AppName `json:"application"` + Privilege PrivilegeName `json:"privilege"` +} + +//noinspection GoRedundantParens +type DeletePrivilegeResponse map[AppName](map[PrivilegeName]DeleteResponse) + +type RoleDescriptor map[AppName]Applications + +type Applications struct { + Applications []Application `json:"applications"` +} + +type Application struct { + Name AppName `json:"application"` + Privileges []Privilege `json:"privileges"` + Resources []Resource `json:"resources"` +} + +type ApiKeyResponse struct { + ApiKey + Creation int64 `json:"creation"` + Invalidated bool `json:"invalidated"` + Username string `json:"username"` +} + +type ApiKeyQuery struct { + // normally the Elasticsearch API will require either Id or Name, but not both + Id *string `json:"id,omitempty"` + Name *string `json:"name,omitempty"` +} + +type ApiKey struct { + Id string `json:"id"` + Name string `json:"name"` + ExpirationMs *int64 `json:"expiration,omitempty"` + // This attribute does not come from Elasticsearch, but is filled in by APM Server + Credentials *string `json:"credentials,omitempty"` +} + +type PrivilegeResponse map[Privilege]PutResponse + +type PrivilegeGroup map[PrivilegeName]Actions + +type Perms map[Privilege]bool + +type PrivilegesPerResource map[Resource]Perms + +type Actions struct { + Actions []Privilege `json:"actions"` +} + +type PutResponse struct { + Created bool `json:"created"` +} + +type DeleteResponse struct { + Found bool `json:"found"` +} + +type AppName string + +type Resource string + +// in Elasticsearch a "privilege" represents both an "action" that a user might/might not have authorization to +// perform; and a tuple consisting of a name and an action +// for differentiation, we call the tuple NamedPrivilege +// in apm-server, each name is associated with one action, but that needs not to be the case (see PrivilegeGroup) +type NamedPrivilege struct { + Name PrivilegeName + Action Privilege +} + +// sometimes referred in Elasticsearch documentation as "action" +// we keep the name "privilege" because is more informative +type Privilege string + +type PrivilegeName string + +func NewPrivilege(name, action string) NamedPrivilege { + return NamedPrivilege{ + Name: PrivilegeName(name), + Action: Privilege(action), + } +} diff --git a/x-pack/apm-server/cmd/root.go b/x-pack/apm-server/cmd/root.go index c85dc435118..0ebf14e3e99 100644 --- a/x-pack/apm-server/cmd/root.go +++ b/x-pack/apm-server/cmd/root.go @@ -14,5 +14,5 @@ import ( var RootCmd = cmd.RootCmd func init() { - xpackcmd.AddXPack(RootCmd, cmd.Name) + xpackcmd.AddXPack(RootCmd.BeatsRootCmd, cmd.Name) } From 32bbcb6b924ab6423e61c2b1a7a3a03374419b5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20=C3=81lvarez?= Date: Mon, 16 Dec 2019 16:58:58 +0100 Subject: [PATCH 06/36] Fix some tests --- beater/api/mux_config_agent_test.go | 6 ++-- beater/api/mux_intake_backend_test.go | 6 ++-- beater/api/mux_intake_rum_test.go | 14 +++++--- beater/api/mux_root_test.go | 3 +- beater/api/mux_sourcemap_handler_test.go | 9 +++-- beater/api/mux_test.go | 5 +-- beater/api/root/handler_test.go | 2 +- beater/authorization/apikey_test.go | 12 +++---- beater/authorization/builder_test.go | 4 +-- beater/config/api_key_test.go | 2 ++ beater/config/config_test.go | 34 ++++++++----------- .../authorization_middleware_test.go | 4 +-- beater/onboarding_test.go | 5 +-- 13 files changed, 59 insertions(+), 47 deletions(-) diff --git a/beater/api/mux_config_agent_test.go b/beater/api/mux_config_agent_test.go index 01e147dc35b..fc599ccc21d 100644 --- a/beater/api/mux_config_agent_test.go +++ b/beater/api/mux_config_agent_test.go @@ -57,8 +57,10 @@ func TestConfigAgentHandler_AuthorizationMiddleware(t *testing.T) { } func TestConfigAgentHandler_KillSwitchMiddleware(t *testing.T) { + cfg, err := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) + require.NoError(t, err) t.Run("Off", func(t *testing.T) { - rec, err := requestToMuxerWithPattern(config.DefaultConfig(beatertest.MockBeatVersion()), AgentConfigPath) + rec, err := requestToMuxerWithPattern(cfg, AgentConfigPath) require.NoError(t, err) require.Equal(t, http.StatusForbidden, rec.Code) approvals.AssertApproveResult(t, approvalPathConfigAgent(t.Name()), rec.Body.Bytes()) @@ -98,7 +100,7 @@ func TestConfigAgentHandler_MonitoringMiddleware(t *testing.T) { } func configEnabledConfigAgent() *config.Config { - cfg := config.DefaultConfig(beatertest.MockBeatVersion()) + cfg, _ := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) cfg.Kibana = common.MustNewConfigFrom(map[string]interface{}{"enabled": "true", "host": "localhost:foo"}) return cfg } diff --git a/beater/api/mux_intake_backend_test.go b/beater/api/mux_intake_backend_test.go index e8ad5f71fa6..f7ee139fcdf 100644 --- a/beater/api/mux_intake_backend_test.go +++ b/beater/api/mux_intake_backend_test.go @@ -35,7 +35,8 @@ import ( func TestIntakeBackendHandler_AuthorizationMiddleware(t *testing.T) { t.Run("Unauthorized", func(t *testing.T) { - cfg := config.DefaultConfig(beatertest.MockBeatVersion()) + cfg, err := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) + require.NoError(t, err) cfg.SecretToken = "1234" rec, err := requestToMuxerWithPattern(cfg, IntakePath) require.NoError(t, err) @@ -45,7 +46,8 @@ func TestIntakeBackendHandler_AuthorizationMiddleware(t *testing.T) { }) t.Run("Authorized", func(t *testing.T) { - cfg := config.DefaultConfig(beatertest.MockBeatVersion()) + cfg, err := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) + require.NoError(t, err) cfg.SecretToken = "1234" h := map[string]string{headers.Authorization: "Bearer 1234"} rec, err := requestToMuxerWithHeader(cfg, IntakePath, http.MethodGet, h) diff --git a/beater/api/mux_intake_rum_test.go b/beater/api/mux_intake_rum_test.go index 0359c5d02b6..880eb0c6d82 100644 --- a/beater/api/mux_intake_rum_test.go +++ b/beater/api/mux_intake_rum_test.go @@ -77,7 +77,9 @@ func TestRUMHandler_NoAuthorizationRequired(t *testing.T) { func TestRUMHandler_KillSwitchMiddleware(t *testing.T) { t.Run("OffRum", func(t *testing.T) { - rec, err := requestToMuxerWithPattern(config.DefaultConfig(beatertest.MockBeatVersion()), IntakeRUMPath) + cfg, err := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) + require.NoError(t, err) + rec, err := requestToMuxerWithPattern(cfg, IntakeRUMPath) require.NoError(t, err) assert.Equal(t, http.StatusForbidden, rec.Code) approvals.AssertApproveResult(t, approvalPathIntakeRUM(t.Name()), rec.Body.Bytes()) @@ -104,7 +106,9 @@ func TestRUMHandler_CORSMiddleware(t *testing.T) { } func TestIntakeRUMHandler_PanicMiddleware(t *testing.T) { - h, err := rumIntakeHandler(config.DefaultConfig(beatertest.MockBeatVersion()), nil, beatertest.NilReporter) + cfg, err := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) + require.NoError(t, err) + h, err := rumIntakeHandler(cfg, nil, beatertest.NilReporter) require.NoError(t, err) rec := &beatertest.WriterPanicOnce{} c := &request.Context{} @@ -115,7 +119,9 @@ func TestIntakeRUMHandler_PanicMiddleware(t *testing.T) { } func TestRumHandler_MonitoringMiddleware(t *testing.T) { - h, err := rumIntakeHandler(config.DefaultConfig(beatertest.MockBeatVersion()), nil, beatertest.NilReporter) + cfg, err := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) + require.NoError(t, err) + h, err := rumIntakeHandler(cfg, nil, beatertest.NilReporter) require.NoError(t, err) c, _ := beatertest.ContextWithResponseRecorder(http.MethodPost, "/") // send GET request resulting in 403 Forbidden error @@ -130,7 +136,7 @@ func TestRumHandler_MonitoringMiddleware(t *testing.T) { } func cfgEnabledRUM() *config.Config { - cfg := config.DefaultConfig(beatertest.MockBeatVersion()) + cfg, _ := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) t := true cfg.RumConfig.Enabled = &t return cfg diff --git a/beater/api/mux_root_test.go b/beater/api/mux_root_test.go index 61c0a918139..4f5f65d747d 100644 --- a/beater/api/mux_root_test.go +++ b/beater/api/mux_root_test.go @@ -34,7 +34,8 @@ import ( ) func TestRootHandler_AuthorizationMiddleware(t *testing.T) { - cfg := config.DefaultConfig(beatertest.MockBeatVersion()) + cfg, err := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) + require.NoError(t, err) cfg.SecretToken = "1234" t.Run("No auth", func(t *testing.T) { diff --git a/beater/api/mux_sourcemap_handler_test.go b/beater/api/mux_sourcemap_handler_test.go index ffe7f5ab010..c6433a656ec 100644 --- a/beater/api/mux_sourcemap_handler_test.go +++ b/beater/api/mux_sourcemap_handler_test.go @@ -56,18 +56,21 @@ func TestSourcemapHandler_AuthorizationMiddleware(t *testing.T) { func TestSourcemapHandler_KillSwitchMiddleware(t *testing.T) { t.Run("OffRum", func(t *testing.T) { - rec, err := requestToMuxerWithPattern(config.DefaultConfig(beatertest.MockBeatVersion()), AssetSourcemapPath) + cfg, err := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) + require.NoError(t, err) + rec, err := requestToMuxerWithPattern(cfg, AssetSourcemapPath) require.NoError(t, err) require.Equal(t, http.StatusForbidden, rec.Code) approvals.AssertApproveResult(t, approvalPathAsset(t.Name()), rec.Body.Bytes()) }) t.Run("OffSourcemap", func(t *testing.T) { - cfg := config.DefaultConfig(beatertest.MockBeatVersion()) + cfg, err := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) + require.NoError(t, err) rum := true cfg.RumConfig.Enabled = &rum cfg.RumConfig.SourceMapping.Enabled = new(bool) - rec, err := requestToMuxerWithPattern(config.DefaultConfig(beatertest.MockBeatVersion()), AssetSourcemapPath) + rec, err := requestToMuxerWithPattern(cfg, AssetSourcemapPath) require.NoError(t, err) require.Equal(t, http.StatusForbidden, rec.Code) approvals.AssertApproveResult(t, approvalPathAsset(t.Name()), rec.Body.Bytes()) diff --git a/beater/api/mux_test.go b/beater/api/mux_test.go index f4d9ed38a37..6be2fe06059 100644 --- a/beater/api/mux_test.go +++ b/beater/api/mux_test.go @@ -55,8 +55,9 @@ func requestToMuxer(cfg *config.Config, r *http.Request) (*httptest.ResponseReco } func testHandler(t *testing.T, fn func(*config.Config, *authorization.Builder, publish.Reporter) (request.Handler, error)) request.Handler { - cfg := config.DefaultConfig(beatertest.MockBeatVersion()) - builder, err := authorization.NewBuilder(cfg) + cfg, err := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) + require.NoError(t, err) + builder, err := authorization.NewBuilder(*cfg) require.NoError(t, err) h, err := fn(cfg, builder, beatertest.NilReporter) require.NoError(t, err) diff --git a/beater/api/root/handler_test.go b/beater/api/root/handler_test.go index 631ae5c64aa..1926375d934 100644 --- a/beater/api/root/handler_test.go +++ b/beater/api/root/handler_test.go @@ -62,7 +62,7 @@ func TestRootHandler(t *testing.T) { t.Run("authorized", func(t *testing.T) { c, w := beatertest.ContextWithResponseRecorder(http.MethodGet, "/") - builder, err := authorization.NewBuilder(&config.Config{SecretToken: "abc"}) + builder, err := authorization.NewBuilder(config.Config{SecretToken: "abc"}) require.NoError(t, err) c.Authorization = builder.ForPrivilege("").AuthorizationFor("Bearer", "abc") Handler()(c) diff --git a/beater/authorization/apikey_test.go b/beater/authorization/apikey_test.go index 649e306414c..43a58b6898e 100644 --- a/beater/authorization/apikey_test.go +++ b/beater/authorization/apikey_test.go @@ -143,9 +143,9 @@ func TestAPIKey_AuthorizedFor(t *testing.T) { handler := tc.builder.forKey("12a3") valid, err := handler.AuthorizedFor("xyz") - require.NoError(t, err) + require.Error(t, err) assert.False(t, valid) - assert.Equal(t, 1, tc.cache.cache.ItemCount()) + assert.Equal(t, 0, tc.cache.cache.ItemCount()) }) t.Run("decode error from ES", func(t *testing.T) { @@ -174,16 +174,16 @@ func (tc *apikeyTestcase) setup(t *testing.T) { if tc.transport == nil { tc.transport = estest.NewTransport(t, http.StatusOK, map[string]interface{}{ "application": map[string]interface{}{ - "application": map[string]map[string]interface{}{ - "foo": {"agentconfig": true, "event": true, "sourcemap": false}, - "bar": {"agentConfig": true, "event": false}, + "apm": map[string]map[string]interface{}{ + "foo": {"config_agent:read": true, "event:write": true, "sourcemap:write": false}, + "bar": {"config_agent:read": true, "event:write": false}, }}}) } tc.client, err = estest.NewElasticsearchClient(tc.transport) require.NoError(t, err) } if tc.cache == nil { - tc.cache = newPrivilegesCache(time.Millisecond, 5) + tc.cache = newPrivilegesCache(time.Minute, 5) } if tc.anyOfPrivileges == nil { tc.anyOfPrivileges = []elasticsearch.Privilege{PrivilegeEventWrite.Action, PrivilegeSourcemapWrite.Action} diff --git a/beater/authorization/builder_test.go b/beater/authorization/builder_test.go index edf4d57db9d..1bff434d091 100644 --- a/beater/authorization/builder_test.go +++ b/beater/authorization/builder_test.go @@ -41,9 +41,9 @@ func TestBuilder(t *testing.T) { } { setup := func() *Builder { - rawCfg := config.DefaultConfig("9.9.9") - cfg, err := config.Setup(nil, rawCfg, nil) + cfg, err := config.Setup(config.DefaultConfig("9.9.9"), nil) require.NoError(t, err) + if tc.withBearer { cfg.SecretToken = "xvz" } diff --git a/beater/config/api_key_test.go b/beater/config/api_key_test.go index 18dca17f752..112904732fa 100644 --- a/beater/config/api_key_test.go +++ b/beater/config/api_key_test.go @@ -77,6 +77,8 @@ func TestAPIKeyConfig_ESConfig(t *testing.T) { LimitMin: 20, ESConfig: &elasticsearch.Config{ Timeout: 5 * time.Second, + Username: "foo", + Password: "bar", Protocol: "http", Hosts: elasticsearch.Hosts{"192.0.0.168:9200"}}}, }, diff --git a/beater/config/config_test.go b/beater/config/config_test.go index cc79951fc2a..9477d33586a 100644 --- a/beater/config/config_test.go +++ b/beater/config/config_test.go @@ -37,15 +37,15 @@ import ( func Test_UnpackConfig(t *testing.T) { falsy, truthy := false, true version := "8.0.0" - defaultRawConfig := DefaultConfig(version) - defaultConfig, _ := Setup(nil, defaultRawConfig, nil) + cfg, err := Setup(DefaultConfig(version), nil) + require.NoError(t, err) tests := map[string]struct { inpCfg map[string]interface{} outCfg *Config }{ "default config": { inpCfg: map[string]interface{}{}, - outCfg: defaultConfig, + outCfg: cfg, }, "overwrite default": { inpCfg: map[string]interface{}{ @@ -240,8 +240,7 @@ func Test_UnpackConfig(t *testing.T) { inpCfg, err := common.NewConfigFrom(test.inpCfg) assert.NoError(t, err) - rawCfg, err := NewRawConfig(version, inpCfg) - cfg, err := Setup(nil, rawCfg, err) + cfg, err := NewConfig(version, inpCfg, nil) require.NoError(t, err) require.NotNil(t, cfg) assert.Equal(t, test.outCfg, cfg) @@ -322,7 +321,7 @@ func TestTLSSettings(t *testing.T) { ucfgCfg, err := common.NewConfigFrom(tc.config) require.NoError(t, err) - cfg, err := NewRawConfig("9.9.9", ucfgCfg) + cfg, err := NewConfig("9.9.9", ucfgCfg, nil) require.NoError(t, err) assert.Equal(t, tc.tls.ClientAuth, cfg.TLS.ClientAuth) }) @@ -348,7 +347,7 @@ func TestTLSSettings(t *testing.T) { ucfgCfg, err := common.NewConfigFrom(tc.config) require.NoError(t, err) - cfg, err := NewRawConfig("9.9.9", ucfgCfg) + cfg, err := NewConfig("9.9.9", ucfgCfg, nil) require.NoError(t, err) assert.Equal(t, tc.tls.VerificationMode, cfg.TLS.VerificationMode) }) @@ -380,25 +379,22 @@ func TestTLSSettings(t *testing.T) { func TestAgentConfig(t *testing.T) { t.Run("InvalidValueTooSmall", func(t *testing.T) { - rawCfg, err := NewRawConfig("9.9.9", - common.MustNewConfigFrom(map[string]string{"agent.config.cache.expiration": "123ms"})) - cfg, err := Setup(nil, rawCfg, err) + cfg, err := NewConfig("9.9.9", + common.MustNewConfigFrom(map[string]string{"agent.config.cache.expiration": "123ms"}), nil) require.Error(t, err) assert.Nil(t, cfg) }) t.Run("InvalidUnit", func(t *testing.T) { - rawCfg, err := NewRawConfig("9.9.9", - common.MustNewConfigFrom(map[string]string{"agent.config.cache.expiration": "1230ms"})) - cfg, err := Setup(nil, rawCfg, err) + cfg, err := NewConfig("9.9.9", + common.MustNewConfigFrom(map[string]string{"agent.config.cache.expiration": "1230ms"}), nil) require.Error(t, err) assert.Nil(t, cfg) }) t.Run("Valid", func(t *testing.T) { - rawCfg, err := NewRawConfig("9.9.9", - common.MustNewConfigFrom(map[string]string{"agent.config.cache.expiration": "123000ms"})) - cfg, err := Setup(nil, rawCfg, err) + cfg, err := NewConfig("9.9.9", + common.MustNewConfigFrom(map[string]string{"agent.config.cache.expiration": "123000ms"}), nil) require.NoError(t, err) assert.Equal(t, time.Second*123, cfg.AgentConfig.Cache.Expiration) }) @@ -410,16 +406,14 @@ func TestNewConfig_ESConfig(t *testing.T) { require.NoError(t, err) // no es config given - rawCfg, err := NewRawConfig(version, ucfg) - cfg, err := Setup(nil, rawCfg, err) + cfg, err := NewConfig(version, ucfg, nil) require.NoError(t, err) assert.Nil(t, cfg.RumConfig.SourceMapping.ESConfig) assert.Equal(t, elasticsearch.DefaultConfig(), cfg.APIKeyConfig.ESConfig) // with es config outputESCfg := common.MustNewConfigFrom(`{"hosts":["192.0.0.168:9200"]}`) - rawCfg, err = NewRawConfig(version, ucfg) - cfg, err = Setup(outputESCfg, rawCfg, err) + cfg, err = NewConfig(version, ucfg, outputESCfg) require.NoError(t, err) assert.NotNil(t, cfg.RumConfig.SourceMapping.ESConfig) assert.Equal(t, []string{"192.0.0.168:9200"}, []string(cfg.RumConfig.SourceMapping.ESConfig.Hosts)) diff --git a/beater/middleware/authorization_middleware_test.go b/beater/middleware/authorization_middleware_test.go index 2fad8f141ee..60a25c32cf6 100644 --- a/beater/middleware/authorization_middleware_test.go +++ b/beater/middleware/authorization_middleware_test.go @@ -49,9 +49,9 @@ func TestAuthorizationMiddleware(t *testing.T) { if tc.header != "" { c.Request.Header.Set(headers.Authorization, tc.header) } - builder, err := authorization.NewBuilder(&config.Config{SecretToken: token}) + builder, err := authorization.NewBuilder(config.Config{SecretToken: token}) require.NoError(t, err) - return builder.ForAnyOfPrivileges(authorization.PrivilegesAll), c, rec + return builder.ForAnyOfPrivileges(authorization.ActionAny), c, rec } t.Run(name+"secured apply", func(t *testing.T) { diff --git a/beater/onboarding_test.go b/beater/onboarding_test.go index 68b06ce5f68..ea9b909a3c3 100644 --- a/beater/onboarding_test.go +++ b/beater/onboarding_test.go @@ -33,12 +33,13 @@ import ( ) func TestNotifyUpServerDown(t *testing.T) { - config := config.DefaultConfig("7.0.0") + config, err := config.Setup(config.DefaultConfig("7.0.0"), nil) + require.NoError(t, err) var saved beat.Event var publisher = func(e beat.Event) { saved = e } lis, err := net.Listen("tcp", "localhost:0") - assert.NoError(t, err) + require.NoError(t, err) defer lis.Close() config.Host = lis.Addr().String() From 2aaa7462363cc36403ef05f10d01a21f26538c33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20=C3=81lvarez?= Date: Wed, 18 Dec 2019 10:52:30 +0100 Subject: [PATCH 07/36] Some fixes --- cmd/apikey.go | 23 +++++++++++++++++------ elasticsearch/config.go | 2 +- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/cmd/apikey.go b/cmd/apikey.go index 26e3d134ea6..5b7bfe62245 100644 --- a/cmd/apikey.go +++ b/cmd/apikey.go @@ -111,9 +111,9 @@ func createApiKeyWithPrivileges(client es.Client, apikeyName, expiry string, pri } func getApiKey(client es.Client, id, name *string, validOnly, asJson bool) error { - if id != nil { + if isSet(id) { name = nil - } else if id == nil && name == nil { + } else if !(isSet(id) || isSet(name)) { return printErr(errors.New("could not query Elasticsearch"), `either "id" or "name" are required`, asJson) @@ -154,9 +154,9 @@ func getApiKey(client es.Client, id, name *string, validOnly, asJson bool) error } func invalidateApiKey(client es.Client, id, name *string, deletePrivileges, asJson bool) error { - if id != nil { + if isSet(id) { name = nil - } else if id == nil && name == nil { + } else if !(isSet(id) || isSet(name)) { return printErr(errors.New("could not query Elasticsearch"), `either "id" or "name" are required`, asJson) @@ -210,22 +210,29 @@ func invalidateApiKey(client es.Client, id, name *string, deletePrivileges, asJs } func verifyApiKey(config *config.Config, privileges []es.Privilege, credentials string, asJson bool) error { - builder, err := auth.NewBuilder(*config) perms := make(es.Perms) printText, printJson := printers(asJson) + var err error for _, privilege := range privileges { + var builder *auth.Builder + builder, err := auth.NewBuilder(*config) if err != nil { break } + var authorized bool authorized, err = builder. ForPrivilege(privilege). AuthorizationFor(headers.APIKey, credentials). AuthorizedFor(auth.ResourceInternal) + if err != nil { + break + } + perms[privilege] = authorized - printText("Authorized for %s...: %s\n", humanPrivilege(privilege), humanBool(authorized)) + printText("Authorized for %s...: %s", humanPrivilege(privilege), humanBool(authorized)) } if err != nil { @@ -324,3 +331,7 @@ func printErr(err error, help string, asJson bool) error { } return errors.Wrap(err, help) } + +func isSet(s *string) bool { + return s != nil && *s != "" +} diff --git a/elasticsearch/config.go b/elasticsearch/config.go index d0ef6102755..c433437e530 100644 --- a/elasticsearch/config.go +++ b/elasticsearch/config.go @@ -44,7 +44,7 @@ var ( // Config holds all configurable fields that are used to create a Client type Config struct { - Hosts Hosts `config:"hosts" validate:"required"` + Hosts Hosts `config:"hosts"` Protocol string `config:"protocol"` Path string `config:"path"` ProxyURL string `config:"proxy_url"` From 1c692eb485bbd3ab1111c437c0c2cb1422d16647 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20=C3=81lvarez?= Date: Wed, 18 Dec 2019 15:53:51 +0100 Subject: [PATCH 08/36] Revert all the config stuff --- beater/api/mux_config_agent_test.go | 6 ++--- beater/api/mux_intake_backend_test.go | 6 ++--- beater/api/mux_intake_rum_test.go | 14 +++-------- beater/api/mux_root_test.go | 3 +-- beater/api/mux_sourcemap_handler_test.go | 9 +++---- beater/api/mux_test.go | 3 +-- beater/authorization/builder_test.go | 3 +-- beater/config/config.go | 31 +++++------------------- beater/config/config_test.go | 5 ++-- beater/onboarding_test.go | 5 ++-- cmd/root.go | 11 +-------- 11 files changed, 25 insertions(+), 71 deletions(-) diff --git a/beater/api/mux_config_agent_test.go b/beater/api/mux_config_agent_test.go index fc599ccc21d..01e147dc35b 100644 --- a/beater/api/mux_config_agent_test.go +++ b/beater/api/mux_config_agent_test.go @@ -57,10 +57,8 @@ func TestConfigAgentHandler_AuthorizationMiddleware(t *testing.T) { } func TestConfigAgentHandler_KillSwitchMiddleware(t *testing.T) { - cfg, err := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) - require.NoError(t, err) t.Run("Off", func(t *testing.T) { - rec, err := requestToMuxerWithPattern(cfg, AgentConfigPath) + rec, err := requestToMuxerWithPattern(config.DefaultConfig(beatertest.MockBeatVersion()), AgentConfigPath) require.NoError(t, err) require.Equal(t, http.StatusForbidden, rec.Code) approvals.AssertApproveResult(t, approvalPathConfigAgent(t.Name()), rec.Body.Bytes()) @@ -100,7 +98,7 @@ func TestConfigAgentHandler_MonitoringMiddleware(t *testing.T) { } func configEnabledConfigAgent() *config.Config { - cfg, _ := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) + cfg := config.DefaultConfig(beatertest.MockBeatVersion()) cfg.Kibana = common.MustNewConfigFrom(map[string]interface{}{"enabled": "true", "host": "localhost:foo"}) return cfg } diff --git a/beater/api/mux_intake_backend_test.go b/beater/api/mux_intake_backend_test.go index f7ee139fcdf..e8ad5f71fa6 100644 --- a/beater/api/mux_intake_backend_test.go +++ b/beater/api/mux_intake_backend_test.go @@ -35,8 +35,7 @@ import ( func TestIntakeBackendHandler_AuthorizationMiddleware(t *testing.T) { t.Run("Unauthorized", func(t *testing.T) { - cfg, err := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) - require.NoError(t, err) + cfg := config.DefaultConfig(beatertest.MockBeatVersion()) cfg.SecretToken = "1234" rec, err := requestToMuxerWithPattern(cfg, IntakePath) require.NoError(t, err) @@ -46,8 +45,7 @@ func TestIntakeBackendHandler_AuthorizationMiddleware(t *testing.T) { }) t.Run("Authorized", func(t *testing.T) { - cfg, err := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) - require.NoError(t, err) + cfg := config.DefaultConfig(beatertest.MockBeatVersion()) cfg.SecretToken = "1234" h := map[string]string{headers.Authorization: "Bearer 1234"} rec, err := requestToMuxerWithHeader(cfg, IntakePath, http.MethodGet, h) diff --git a/beater/api/mux_intake_rum_test.go b/beater/api/mux_intake_rum_test.go index 880eb0c6d82..0359c5d02b6 100644 --- a/beater/api/mux_intake_rum_test.go +++ b/beater/api/mux_intake_rum_test.go @@ -77,9 +77,7 @@ func TestRUMHandler_NoAuthorizationRequired(t *testing.T) { func TestRUMHandler_KillSwitchMiddleware(t *testing.T) { t.Run("OffRum", func(t *testing.T) { - cfg, err := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) - require.NoError(t, err) - rec, err := requestToMuxerWithPattern(cfg, IntakeRUMPath) + rec, err := requestToMuxerWithPattern(config.DefaultConfig(beatertest.MockBeatVersion()), IntakeRUMPath) require.NoError(t, err) assert.Equal(t, http.StatusForbidden, rec.Code) approvals.AssertApproveResult(t, approvalPathIntakeRUM(t.Name()), rec.Body.Bytes()) @@ -106,9 +104,7 @@ func TestRUMHandler_CORSMiddleware(t *testing.T) { } func TestIntakeRUMHandler_PanicMiddleware(t *testing.T) { - cfg, err := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) - require.NoError(t, err) - h, err := rumIntakeHandler(cfg, nil, beatertest.NilReporter) + h, err := rumIntakeHandler(config.DefaultConfig(beatertest.MockBeatVersion()), nil, beatertest.NilReporter) require.NoError(t, err) rec := &beatertest.WriterPanicOnce{} c := &request.Context{} @@ -119,9 +115,7 @@ func TestIntakeRUMHandler_PanicMiddleware(t *testing.T) { } func TestRumHandler_MonitoringMiddleware(t *testing.T) { - cfg, err := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) - require.NoError(t, err) - h, err := rumIntakeHandler(cfg, nil, beatertest.NilReporter) + h, err := rumIntakeHandler(config.DefaultConfig(beatertest.MockBeatVersion()), nil, beatertest.NilReporter) require.NoError(t, err) c, _ := beatertest.ContextWithResponseRecorder(http.MethodPost, "/") // send GET request resulting in 403 Forbidden error @@ -136,7 +130,7 @@ func TestRumHandler_MonitoringMiddleware(t *testing.T) { } func cfgEnabledRUM() *config.Config { - cfg, _ := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) + cfg := config.DefaultConfig(beatertest.MockBeatVersion()) t := true cfg.RumConfig.Enabled = &t return cfg diff --git a/beater/api/mux_root_test.go b/beater/api/mux_root_test.go index 4f5f65d747d..61c0a918139 100644 --- a/beater/api/mux_root_test.go +++ b/beater/api/mux_root_test.go @@ -34,8 +34,7 @@ import ( ) func TestRootHandler_AuthorizationMiddleware(t *testing.T) { - cfg, err := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) - require.NoError(t, err) + cfg := config.DefaultConfig(beatertest.MockBeatVersion()) cfg.SecretToken = "1234" t.Run("No auth", func(t *testing.T) { diff --git a/beater/api/mux_sourcemap_handler_test.go b/beater/api/mux_sourcemap_handler_test.go index c6433a656ec..ffe7f5ab010 100644 --- a/beater/api/mux_sourcemap_handler_test.go +++ b/beater/api/mux_sourcemap_handler_test.go @@ -56,21 +56,18 @@ func TestSourcemapHandler_AuthorizationMiddleware(t *testing.T) { func TestSourcemapHandler_KillSwitchMiddleware(t *testing.T) { t.Run("OffRum", func(t *testing.T) { - cfg, err := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) - require.NoError(t, err) - rec, err := requestToMuxerWithPattern(cfg, AssetSourcemapPath) + rec, err := requestToMuxerWithPattern(config.DefaultConfig(beatertest.MockBeatVersion()), AssetSourcemapPath) require.NoError(t, err) require.Equal(t, http.StatusForbidden, rec.Code) approvals.AssertApproveResult(t, approvalPathAsset(t.Name()), rec.Body.Bytes()) }) t.Run("OffSourcemap", func(t *testing.T) { - cfg, err := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) - require.NoError(t, err) + cfg := config.DefaultConfig(beatertest.MockBeatVersion()) rum := true cfg.RumConfig.Enabled = &rum cfg.RumConfig.SourceMapping.Enabled = new(bool) - rec, err := requestToMuxerWithPattern(cfg, AssetSourcemapPath) + rec, err := requestToMuxerWithPattern(config.DefaultConfig(beatertest.MockBeatVersion()), AssetSourcemapPath) require.NoError(t, err) require.Equal(t, http.StatusForbidden, rec.Code) approvals.AssertApproveResult(t, approvalPathAsset(t.Name()), rec.Body.Bytes()) diff --git a/beater/api/mux_test.go b/beater/api/mux_test.go index 6be2fe06059..00f89bc8a42 100644 --- a/beater/api/mux_test.go +++ b/beater/api/mux_test.go @@ -55,8 +55,7 @@ func requestToMuxer(cfg *config.Config, r *http.Request) (*httptest.ResponseReco } func testHandler(t *testing.T, fn func(*config.Config, *authorization.Builder, publish.Reporter) (request.Handler, error)) request.Handler { - cfg, err := config.Setup(config.DefaultConfig(beatertest.MockBeatVersion()), nil) - require.NoError(t, err) + cfg := config.DefaultConfig(beatertest.MockBeatVersion()) builder, err := authorization.NewBuilder(*cfg) require.NoError(t, err) h, err := fn(cfg, builder, beatertest.NilReporter) diff --git a/beater/authorization/builder_test.go b/beater/authorization/builder_test.go index 1bff434d091..f665d11a585 100644 --- a/beater/authorization/builder_test.go +++ b/beater/authorization/builder_test.go @@ -41,8 +41,7 @@ func TestBuilder(t *testing.T) { } { setup := func() *Builder { - cfg, err := config.Setup(config.DefaultConfig("9.9.9"), nil) - require.NoError(t, err) + cfg := config.DefaultConfig("9.9.9") if tc.withBearer { cfg.SecretToken = "xvz" diff --git a/beater/config/config.go b/beater/config/config.go index 4321c3932d5..98cce679dcd 100644 --- a/beater/config/config.go +++ b/beater/config/config.go @@ -68,8 +68,6 @@ type Config struct { Pipeline string } -type RawConfig Config - // ExpvarConfig holds config information about exposing expvar type ExpvarConfig struct { Enabled *bool `config:"enabled"` @@ -86,8 +84,9 @@ type Cache struct { Expiration time.Duration `config:"expiration"` } -// NewRawConfig unpacks the ucfg data -func NewRawConfig(version string, ucfg *common.Config) (*RawConfig, error) { +// NewConfig creates a Config struct based on the default config and the given input params +func NewConfig(version string, ucfg *common.Config, outputESCfg *common.Config) (*Config, error) { + logger := logp.NewLogger(logs.Config) c := DefaultConfig(version) if ucfg.HasField("ssl") { @@ -105,16 +104,6 @@ func NewRawConfig(version string, ucfg *common.Config) (*RawConfig, error) { if err := ucfg.Unpack(c); err != nil { return nil, errors.Wrap(err, "Error processing configuration") } - return c, nil -} - -// Setup contains ad-hoc logic for apm-server configuration -func Setup(rawConfig *RawConfig, outputESCfg *common.Config) (*Config, error) { - logger := logp.NewLogger(logs.Config) - if rawConfig == nil { - return nil, nil - } - c := Config(*rawConfig) if float64(int(c.AgentConfig.Cache.Expiration.Seconds())) != c.AgentConfig.Cache.Expiration.Seconds() { return nil, errors.New(msgInvalidConfigAgentCfg) @@ -136,15 +125,7 @@ func Setup(rawConfig *RawConfig, outputESCfg *common.Config) (*Config, error) { return nil, err } - return &c, nil -} - -func NewConfig(version string, ucfg *common.Config, outputESCfg *common.Config) (*Config, error) { - raw, err := NewRawConfig(version, ucfg) - if err != nil { - return nil, err - } - return Setup(raw, outputESCfg) + return c, nil } // IsEnabled indicates whether expvar is enabled or not @@ -153,8 +134,8 @@ func (c *ExpvarConfig) IsEnabled() bool { } // DefaultConfig returns a config with default settings for `apm-server` config options. -func DefaultConfig(beatVersion string) *RawConfig { - return &RawConfig{ +func DefaultConfig(beatVersion string) *Config { + return &Config{ Host: net.JoinHostPort("localhost", DefaultPort), MaxHeaderSize: 1 * 1024 * 1024, // 1mb MaxConnections: 0, // unlimited diff --git a/beater/config/config_test.go b/beater/config/config_test.go index 9477d33586a..867b411ef8b 100644 --- a/beater/config/config_test.go +++ b/beater/config/config_test.go @@ -37,15 +37,14 @@ import ( func Test_UnpackConfig(t *testing.T) { falsy, truthy := false, true version := "8.0.0" - cfg, err := Setup(DefaultConfig(version), nil) - require.NoError(t, err) + tests := map[string]struct { inpCfg map[string]interface{} outCfg *Config }{ "default config": { inpCfg: map[string]interface{}{}, - outCfg: cfg, + outCfg: DefaultConfig(version), }, "overwrite default": { inpCfg: map[string]interface{}{ diff --git a/beater/onboarding_test.go b/beater/onboarding_test.go index ea9b909a3c3..68b06ce5f68 100644 --- a/beater/onboarding_test.go +++ b/beater/onboarding_test.go @@ -33,13 +33,12 @@ import ( ) func TestNotifyUpServerDown(t *testing.T) { - config, err := config.Setup(config.DefaultConfig("7.0.0"), nil) - require.NoError(t, err) + config := config.DefaultConfig("7.0.0") var saved beat.Event var publisher = func(e beat.Event) { saved = e } lis, err := net.Listen("tcp", "localhost:0") - require.NoError(t, err) + assert.NoError(t, err) defer lis.Close() config.Host = lis.Addr().String() diff --git a/cmd/root.go b/cmd/root.go index 05211d48e46..7c992508061 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -312,16 +312,7 @@ func bootstrap(settings instance.Settings) (*config.Config, error) { } outCfg := beat.Config.Output - apmRawConfig, err := config.NewRawConfig(settings.Version, beat.RawConfig) - if apmRawConfig == nil { - return nil, err - } - if apmRawConfig.APIKeyConfig == nil { - apmRawConfig.APIKeyConfig = &config.APIKeyConfig{} - } - apmRawConfig.APIKeyConfig.Enabled = true - - return config.Setup(apmRawConfig, outCfg.Config()) + return config.NewConfig(settings.Version, beat.RawConfig, outCfg.Config()) } // if all are false, returns any ("*") From 5defc0319afa428915c90ee9ee4d243accaad739 Mon Sep 17 00:00:00 2001 From: Gil Raphaelli Date: Thu, 19 Dec 2019 16:46:30 -0500 Subject: [PATCH 09/36] key composite literals --- cmd/apikey.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/apikey.go b/cmd/apikey.go index 5b7bfe62245..19411a2e38e 100644 --- a/cmd/apikey.go +++ b/cmd/apikey.go @@ -46,9 +46,9 @@ func createApiKeyWithPrivileges(client es.Client, apikeyName, expiry string, pri agentConfig := auth.PrivilegeAgentConfigRead sourcemap := auth.PrivilegeSourcemapWrite privilegesRequest[auth.Application] = map[es.PrivilegeName]es.Actions{ - agentConfig.Name: {[]es.Privilege{agentConfig.Action}}, - event.Name: {[]es.Privilege{event.Action}}, - sourcemap.Name: {[]es.Privilege{sourcemap.Action}}, + agentConfig.Name: {Actions: []es.Privilege{agentConfig.Action}}, + event.Name: {Actions: []es.Privilege{event.Action}}, + sourcemap.Name: {Actions: []es.Privilege{sourcemap.Action}}, } privilegesCreated, err := es.CreatePrivileges(client, privilegesRequest) From 82a0c2f15203bd591670f36108c68878d69e4238 Mon Sep 17 00:00:00 2001 From: Gil Raphaelli Date: Thu, 19 Dec 2019 18:05:47 -0500 Subject: [PATCH 10/36] some linting --- beater/authorization/apikey.go | 7 +++--- cmd/apikey.go | 42 +++++++++++++++++----------------- cmd/root.go | 8 +++---- 3 files changed, 29 insertions(+), 28 deletions(-) diff --git a/beater/authorization/apikey.go b/beater/authorization/apikey.go index d05010f9865..26138e07df4 100644 --- a/beater/authorization/apikey.go +++ b/beater/authorization/apikey.go @@ -29,12 +29,13 @@ import ( const cleanupInterval = 60 * time.Second var ( - // Constant mapped to the "application" field for the Elasticsearch security API - // This identifies privileges and keys created for APM + // Application is a constant mapped to the "application" field for the Elasticsearch security API + // identifying privileges and keys created for APM Application = es.AppName("apm") - // Only valid for first authorization of a request. + // ResourceInternal is a sentinel valid only for first authorization of a request. // The API Key needs to grant privileges to additional resources for successful processing of requests. ResourceInternal = es.Resource("-") + // ResourceAny matches all resources ResourceAny = es.Resource("*") ) diff --git a/cmd/apikey.go b/cmd/apikey.go index 19411a2e38e..680b626a7d8 100644 --- a/cmd/apikey.go +++ b/cmd/apikey.go @@ -40,7 +40,7 @@ import ( // creates an API Key with the given privileges, *AND* all the privileges modeled in apm-server // we need to ensure forward-compatibility, for which future privileges must be created here and // during server startup because we don't know if customers will run this command -func createApiKeyWithPrivileges(client es.Client, apikeyName, expiry string, privileges []es.Privilege, asJson bool) error { +func createAPIKeyWithPrivileges(client es.Client, apikeyName, expiry string, privileges []es.Privilege, asJSON bool) error { var privilegesRequest = make(es.CreatePrivilegesRequest) event := auth.PrivilegeEventWrite agentConfig := auth.PrivilegeAgentConfigRead @@ -56,10 +56,10 @@ func createApiKeyWithPrivileges(client es.Client, apikeyName, expiry string, pri if err != nil { return printErr(err, `Error creating privileges for APM Server, do you have the "manage_cluster" security privilege?`, - asJson) + asJSON) } - printText, printJson := printers(asJson) + printText, printJSON := printers(asJSON) for privilege, result := range privilegesCreated[auth.Application] { if result.Created { printText("Security privilege \"%v\" created", privilege) @@ -88,7 +88,7 @@ func createApiKeyWithPrivileges(client es.Client, apikeyName, expiry string, pri if err != nil { return printErr(err, fmt.Sprintf( `Error creating the API Key %s, do you have the "manage_cluster" security privilege?`, apikeyName), - asJson) + asJSON) } credentials := base64.StdEncoding.EncodeToString([]byte(apikey.Id + ":" + apikey.Key)) apikey.Credentials = &credentials @@ -101,7 +101,7 @@ func createApiKeyWithPrivileges(client es.Client, apikeyName, expiry string, pri printText(`Credentials .... %s (use it as "Authorization: ApiKey " header to communicate with APM Server, won't be shown again)`, credentials) - return printJson(struct { + return printJSON(struct { es.CreateApiKeyResponse Privileges es.CreatePrivilegesResponse `json:"created_privileges,omitempty"` }{ @@ -110,13 +110,13 @@ func createApiKeyWithPrivileges(client es.Client, apikeyName, expiry string, pri }) } -func getApiKey(client es.Client, id, name *string, validOnly, asJson bool) error { +func getAPIKey(client es.Client, id, name *string, validOnly, asJSON bool) error { if isSet(id) { name = nil } else if !(isSet(id) || isSet(name)) { return printErr(errors.New("could not query Elasticsearch"), `either "id" or "name" are required`, - asJson) + asJSON) } request := es.GetApiKeyRequest{ ApiKeyQuery: es.ApiKeyQuery{ @@ -129,11 +129,11 @@ func getApiKey(client es.Client, id, name *string, validOnly, asJson bool) error if err != nil { return printErr(err, `Error retrieving API Key(s) for APM Server, do you have the "manage_cluster" security privilege?`, - asJson) + asJSON) } transform := es.GetApiKeyResponse{ApiKeys: make([]es.ApiKeyResponse, 0)} - printText, printJson := printers(asJson) + printText, printJSON := printers(asJSON) for _, apikey := range apikeys.ApiKeys { expiry := humanTime(apikey.ExpirationMs) if validOnly && (apikey.Invalidated || expiry == "expired") { @@ -150,16 +150,16 @@ func getApiKey(client es.Client, id, name *string, validOnly, asJson bool) error transform.ApiKeys = append(transform.ApiKeys, apikey) } printText("%d API Keys found", len(transform.ApiKeys)) - return printJson(transform) + return printJSON(transform) } -func invalidateApiKey(client es.Client, id, name *string, deletePrivileges, asJson bool) error { +func invalidateAPIKey(client es.Client, id, name *string, deletePrivileges, asJSON bool) error { if isSet(id) { name = nil } else if !(isSet(id) || isSet(name)) { return printErr(errors.New("could not query Elasticsearch"), `either "id" or "name" are required`, - asJson) + asJSON) } invalidateKeysRequest := es.InvalidateApiKeyRequest{ ApiKeyQuery: es.ApiKeyQuery{ @@ -172,9 +172,9 @@ func invalidateApiKey(client es.Client, id, name *string, deletePrivileges, asJs if err != nil { return printErr(err, `Error invalidating API Key(s), do you have the "manage_cluster" security privilege?`, - asJson) + asJSON) } - printText, printJson := printers(asJson) + printText, printJSON := printers(asJSON) out := struct { es.InvalidateApiKeyResponse Privileges []es.DeletePrivilegeResponse `json:"deleted_privileges,omitempty"` @@ -206,13 +206,13 @@ func invalidateApiKey(client es.Client, id, name *string, deletePrivileges, asJs } out.Privileges = append(out.Privileges, deletion) } - return printJson(out) + return printJSON(out) } -func verifyApiKey(config *config.Config, privileges []es.Privilege, credentials string, asJson bool) error { +func verifyAPIKey(config *config.Config, privileges []es.Privilege, credentials string, asJSON bool) error { perms := make(es.Perms) - printText, printJson := printers(asJson) + printText, printJSON := printers(asJSON) var err error for _, privilege := range privileges { @@ -236,9 +236,9 @@ func verifyApiKey(config *config.Config, privileges []es.Privilege, credentials } if err != nil { - return printErr(err, "could not verify credentials, please check your Elasticsearch connection", asJson) + return printErr(err, "could not verify credentials, please check your Elasticsearch connection", asJSON) } - return printJson(perms) + return printJSON(perms) } func humanBool(b bool) string { @@ -305,8 +305,8 @@ func printers(b bool) (func(string, ...interface{}), func(interface{}) error) { } // prints an Elasticsearch error to stderr, with some additional contextual information as a hint -func printErr(err error, help string, asJson bool) error { - if asJson { +func printErr(err error, help string, asJSON bool) error { + if asJSON { var data []byte var m map[string]interface{} e := json.Unmarshal([]byte(err.Error()), &m) diff --git a/cmd/root.go b/cmd/root.go index 7c992508061..bc38c103a85 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -185,7 +185,7 @@ Requires the "manage_security" cluster privilege in Elasticsearch.`, // but printing the error must be done inside RunE: func(cmd *cobra.Command, args []string) error { privileges := booleansToPrivileges(ingest, sourcemap, agentConfig) - return createApiKeyWithPrivileges(client, keyName, expiration, privileges, json) + return createAPIKeyWithPrivileges(client, keyName, expiration, privileges, json) }, // these are needed to not break JSON formatting // this has the caveat that if an invalid argument is passed, the command won't return anything @@ -222,7 +222,7 @@ If both "id" and "name" are supplied, only "id" will be used. If neither of them are, an error will be returned. Requires the "manage_security" cluster privilege in Elasticsearch.`, RunE: func(cmd *cobra.Command, args []string) error { - return invalidateApiKey(client, &id, &name, purge, json) + return invalidateAPIKey(client, &id, &name, purge, json) }, SilenceErrors: true, SilenceUsage: true, @@ -250,7 +250,7 @@ If both "id" and "name" are supplied, only "id" will be used. If neither of them are, an error will be returned. Requires the "manage_security" cluster privilege in Elasticsearch.`, RunE: func(cmd *cobra.Command, args []string) error { - return getApiKey(client, &id, &name, validOnly, json) + return getAPIKey(client, &id, &name, validOnly, json) }, SilenceErrors: true, SilenceUsage: true, @@ -278,7 +278,7 @@ If no privilege(s) are specified, the credentials will be queried for all.` Long: long, RunE: func(cmd *cobra.Command, args []string) error { privileges := booleansToPrivileges(ingest, sourcemap, agentConfig) - return verifyApiKey(config, privileges, credentials, json) + return verifyAPIKey(config, privileges, credentials, json) }, SilenceUsage: true, SilenceErrors: true, From fbf9280f83fed40be438012e069454d22bd5342b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20=C3=81lvarez?= Date: Fri, 20 Dec 2019 14:26:20 +0100 Subject: [PATCH 11/36] round of review comments --- _meta/beat.yml | 4 +- apm-server.docker.yml | 4 +- apm-server.yml | 2 +- beater/authorization/apikey.go | 10 +- beater/authorization/apikey_test.go | 6 +- beater/authorization/privilege.go | 17 +- beater/authorization/privilege_test.go | 6 +- beater/config/api_key.go | 2 +- beater/config/api_key_test.go | 2 +- beater/config/config.go | 2 +- cmd/apikey.go | 256 ++++++++++++++++++++++++- cmd/root.go | 213 +------------------- elasticsearch/client.go | 68 +++---- elasticsearch/security_api.go | 15 +- x-pack/apm-server/cmd/root.go | 2 +- 15 files changed, 327 insertions(+), 282 deletions(-) diff --git a/_meta/beat.yml b/_meta/beat.yml index bd87dea8c2e..21852877c34 100644 --- a/_meta/beat.yml +++ b/_meta/beat.yml @@ -153,10 +153,12 @@ apm-server: #protocol: "http" + # Username and password are only needed for the apm-server apikey sub-command, and they are ignored otherwise + # See `apm-server apikey --help` for details. #username: "elastic" #password: "changeme" - # Optional HTTP Path. + # Optional HTTP Path. #path: "" # Proxy server url. diff --git a/apm-server.docker.yml b/apm-server.docker.yml index 419b940f95a..cd0705ef3ff 100644 --- a/apm-server.docker.yml +++ b/apm-server.docker.yml @@ -153,10 +153,12 @@ apm-server: #protocol: "http" + # Username and password are only needed for the apm-server apikey sub-command, and they are ignored otherwise + # See `apm-server apikey --help` for details. #username: "elastic" #password: "changeme" - # Optional HTTP Path. + # Optional HTTP Path. #path: "" # Proxy server url. diff --git a/apm-server.yml b/apm-server.yml index c367b5f940a..4841879c3a0 100644 --- a/apm-server.yml +++ b/apm-server.yml @@ -156,7 +156,7 @@ apm-server: #username: "elastic" #password: "changeme" - # Optional HTTP Path. + # Optional HTTP Path. #path: "" # Proxy server url. diff --git a/beater/authorization/apikey.go b/beater/authorization/apikey.go index d05010f9865..b7234cec4f4 100644 --- a/beater/authorization/apikey.go +++ b/beater/authorization/apikey.go @@ -106,12 +106,14 @@ func (a *apikeyAuth) fromCache(resource es.Resource) (allowed bool, found bool) return } -func (a *apikeyAuth) queryES(resource es.Resource) (es.Perms, error) { +func (a *apikeyAuth) queryES(resource es.Resource) (es.Permissions, error) { request := es.HasPrivilegesRequest{ Applications: []es.Application{ { - Name: Application, - Privileges: a.anyOfPrivileges, + Name: Application, + // it is important to query all privilege actions because they are cached by api key+resources + // querying a.anyOfPrivileges would result in an incomplete cache entry + Privileges: ActionsAll(), Resources: []es.Resource{resource}, }, }, @@ -125,7 +127,7 @@ func (a *apikeyAuth) queryES(resource es.Resource) (es.Perms, error) { return privileges, nil } } - return es.Perms{}, nil + return es.Permissions{}, nil } func id(apiKey string, resource es.Resource) string { diff --git a/beater/authorization/apikey_test.go b/beater/authorization/apikey_test.go index 43a58b6898e..62520544f6f 100644 --- a/beater/authorization/apikey_test.go +++ b/beater/authorization/apikey_test.go @@ -41,7 +41,7 @@ func TestApikeyBuilder(t *testing.T) { handler2 := tc.builder.forKey(key) // add existing privileges to shared cache - privilegesValid := elasticsearch.Perms{} + privilegesValid := elasticsearch.Permissions{} for _, p := range PrivilegesAll { privilegesValid[p.Action] = true } @@ -89,8 +89,8 @@ func TestAPIKey_AuthorizedFor(t *testing.T) { resourceInvalid := elasticsearch.Resource("bar") resourceMissing := elasticsearch.Resource("missing") - tc.cache.add(id(key, resourceValid), elasticsearch.Perms{tc.anyOfPrivileges[0]: true}) - tc.cache.add(id(key, resourceInvalid), elasticsearch.Perms{tc.anyOfPrivileges[0]: false}) + tc.cache.add(id(key, resourceValid), elasticsearch.Permissions{tc.anyOfPrivileges[0]: true}) + tc.cache.add(id(key, resourceInvalid), elasticsearch.Permissions{tc.anyOfPrivileges[0]: false}) valid, err := handler.AuthorizedFor(resourceValid) require.NoError(t, err) diff --git a/beater/authorization/privilege.go b/beater/authorization/privilege.go index eea676ebc9c..d51ebeb6cf8 100644 --- a/beater/authorization/privilege.go +++ b/beater/authorization/privilege.go @@ -30,8 +30,15 @@ var ( PrivilegeEventWrite = es.NewPrivilege("event", "event:write") PrivilegeSourcemapWrite = es.NewPrivilege("sourcemap", "sourcemap:write") PrivilegesAll = []es.NamedPrivilege{PrivilegeAgentConfigRead, PrivilegeEventWrite, PrivilegeSourcemapWrite} - // ActionAny can't be used for querying - ActionAny = es.Privilege("*") + // ActionAny can't be used for querying, use ActionsAll instead + ActionAny = es.Privilege("*") + ActionsAll = func() []es.Privilege { + actions := make([]es.Privilege, 0) + for _, privilege := range PrivilegesAll { + actions = append(actions, privilege.Action) + } + return actions + } ) type privilegesCache struct { @@ -47,13 +54,13 @@ func (c *privilegesCache) isFull() bool { return c.cache.ItemCount() >= c.size } -func (c *privilegesCache) get(id string) es.Perms { +func (c *privilegesCache) get(id string) es.Permissions { if val, exists := c.cache.Get(id); exists { - return val.(es.Perms) + return val.(es.Permissions) } return nil } -func (c *privilegesCache) add(id string, privileges es.Perms) { +func (c *privilegesCache) add(id string, privileges es.Permissions) { c.cache.SetDefault(id, privileges) } diff --git a/beater/authorization/privilege_test.go b/beater/authorization/privilege_test.go index f5849adff08..41332b713c0 100644 --- a/beater/authorization/privilege_test.go +++ b/beater/authorization/privilege_test.go @@ -31,16 +31,16 @@ func TestPrivilegesCache(t *testing.T) { cache := newPrivilegesCache(time.Millisecond, n) assert.False(t, cache.isFull()) for i := 0; i < n-1; i++ { - cache.add(string(i), elasticsearch.Perms{}) + cache.add(string(i), elasticsearch.Permissions{}) assert.False(t, cache.isFull()) } - cache.add("oneMore", elasticsearch.Perms{}) + cache.add("oneMore", elasticsearch.Permissions{}) assert.True(t, cache.isFull()) assert.NotNil(t, cache.get("oneMore")) time.Sleep(time.Millisecond) assert.Nil(t, cache.get("oneMore")) - p := elasticsearch.Perms{"a": true, "b": false} + p := elasticsearch.Permissions{"a": true, "b": false} cache.add("id1", p) assert.Equal(t, p, cache.get("id1")) assert.Nil(t, cache.get("oneMore")) diff --git a/beater/config/api_key.go b/beater/config/api_key.go index 3e4ed840589..2a523ad6245 100644 --- a/beater/config/api_key.go +++ b/beater/config/api_key.go @@ -41,7 +41,7 @@ func (c *APIKeyConfig) IsEnabled() bool { return c != nil && c.Enabled } -func (c *APIKeyConfig) setup(log *logp.Logger, outputESCfg *common.Config) error { +func (c *APIKeyConfig) Setup(log *logp.Logger, outputESCfg *common.Config) error { if c == nil || !c.Enabled || c.ESConfig != nil { return nil } diff --git a/beater/config/api_key_test.go b/beater/config/api_key_test.go index 112904732fa..ad995e55a2b 100644 --- a/beater/config/api_key_test.go +++ b/beater/config/api_key_test.go @@ -84,7 +84,7 @@ func TestAPIKeyConfig_ESConfig(t *testing.T) { }, } { t.Run(name, func(t *testing.T) { - err := tc.cfg.setup(logp.NewLogger("api_key"), tc.esCfg) + err := tc.cfg.Setup(logp.NewLogger("api_key"), tc.esCfg) if tc.expectedErr == nil { assert.NoError(t, err) } else { diff --git a/beater/config/config.go b/beater/config/config.go index 98cce679dcd..98766d2f6c4 100644 --- a/beater/config/config.go +++ b/beater/config/config.go @@ -117,7 +117,7 @@ func NewConfig(version string, ucfg *common.Config, outputESCfg *common.Config) return nil, err } - if err := c.APIKeyConfig.setup(logger, outputESCfg); err != nil { + if err := c.APIKeyConfig.Setup(logger, outputESCfg); err != nil { return nil, err } diff --git a/cmd/apikey.go b/cmd/apikey.go index 680b626a7d8..384a9308e7c 100644 --- a/cmd/apikey.go +++ b/cmd/apikey.go @@ -28,6 +28,15 @@ import ( "strings" "time" + logs "github.com/elastic/apm-server/log" + "github.com/elastic/beats/libbeat/logp" + + "github.com/spf13/cobra" + + "github.com/elastic/beats/libbeat/cfgfile" + "github.com/elastic/beats/libbeat/cmd/instance" + "github.com/elastic/beats/libbeat/common" + "github.com/pkg/errors" "github.com/elastic/apm-server/beater/config" @@ -37,6 +46,239 @@ import ( es "github.com/elastic/apm-server/elasticsearch" ) +func genApikeyCmd(settings instance.Settings) *cobra.Command { + + short := "Manage API Keys for communication between APM agents and server" + apikeyCmd := cobra.Command{ + Use: "apikey", + Short: short, + Long: short + `. +Most operations require the "manage_security" cluster privilege. Ensure to configure "apm-server.api_key.*" or +"output.elasticsearch.*" appropriately. APM Server will create security privileges for the "apm" application; +you can freely query them. If you modify or delete apm privileges, APM Server might reject all requests. +If an invalid argument is passed, nothing will be printed. +Check the Elastic Security API documentation for details.`, + } + + apikeyCmd.AddCommand( + createApikeyCmd(settings), + invalidateApikeyCmd(settings), + getApikeysCmd(settings), + verifyApikeyCmd(settings), + ) + return &apikeyCmd +} + +func createApikeyCmd(settings instance.Settings) *cobra.Command { + var keyName, expiration string + var ingest, sourcemap, agentConfig, json bool + short := "Create an API Key with the specified privilege(s)" + create := &cobra.Command{ + Use: "create", + Short: short, + Long: short + `. +If no privilege(s) are specified, the API Key will be valid for all. +Requires the "manage_security" cluster privilege in Elasticsearch.`, + // always need to return error for possible scripts checking the exit code, + // but printing the error must be done inside + RunE: func(cmd *cobra.Command, args []string) error { + client, _, err := bootstrap(settings) + if err != nil { + printErr(err, "is apm-server configured properly and Elasticsearch reachable?", json) + return err + } + privileges := booleansToPrivileges(ingest, sourcemap, agentConfig) + if len(privileges) == 0 { + privileges = []es.Privilege{auth.ActionAny} + } + return createAPIKeyWithPrivileges(client, keyName, expiration, privileges, json) + }, + // these are needed to not break JSON formatting + // this has the caveat that if an invalid argument is passed, the command won't return anything + SilenceUsage: true, + SilenceErrors: true, + } + create.Flags().StringVar(&keyName, "name", "apm-key", "API Key name") + create.Flags().StringVar(&expiration, "expiration", "", + `expiration for the key, eg. "1d" (default never)`) + create.Flags().BoolVar(&ingest, "ingest", false, + fmt.Sprintf("give the %v privilege to this key, required for ingesting events", auth.PrivilegeEventWrite)) + create.Flags().BoolVar(&sourcemap, "sourcemap", false, + fmt.Sprintf("give the %v privilege to this key, required for uploading sourcemaps", + auth.PrivilegeSourcemapWrite)) + create.Flags().BoolVar(&agentConfig, "agent-config", false, + fmt.Sprintf("give the %v privilege to this key, required for agents to read configuration remotely", + auth.PrivilegeAgentConfigRead)) + create.Flags().BoolVar(&json, "json", false, + "prints the output of this command as JSON") + // this actually means "preserve sorting given in code" and not reorder them alphabetically + create.Flags().SortFlags = false + return create +} + +func invalidateApikeyCmd(settings instance.Settings) *cobra.Command { + var id, name string + var purge, json bool + short := "Invalidate API Key(s) by Id or Name" + invalidate := &cobra.Command{ + Use: "invalidate", + Short: short, + Long: short + `. +If both "id" and "name" are supplied, only "id" will be used. +If neither of them are, an error will be returned. +Requires the "manage_security" cluster privilege in Elasticsearch.`, + RunE: func(cmd *cobra.Command, args []string) error { + client, _, err := bootstrap(settings) + if err != nil { + printErr(err, "is apm-server configured properly and Elasticsearch reachable?", json) + return err + } + return invalidateAPIKey(client, &id, &name, purge, json) + }, + SilenceErrors: true, + SilenceUsage: true, + } + invalidate.Flags().StringVar(&id, "id", "", "id of the API Key to delete") + invalidate.Flags().StringVar(&name, "name", "", + "name of the API Key(s) to delete (several might match)") + invalidate.Flags().BoolVar(&purge, "purge", false, + "also remove all privileges created and used by APM Server") + invalidate.Flags().BoolVar(&json, "json", false, + "prints the output of this command as JSON") + invalidate.Flags().SortFlags = false + return invalidate +} + +func getApikeysCmd(settings instance.Settings) *cobra.Command { + var id, name string + var validOnly, json bool + short := "Query API Key(s) by Id or Name" + info := &cobra.Command{ + Use: "info", + Short: short, + Long: short + `. +If both "id" and "name" are supplied, only "id" will be used. +If neither of them are, an error will be returned. +Requires the "manage_security" cluster privilege in Elasticsearch.`, + RunE: func(cmd *cobra.Command, args []string) error { + client, _, err := bootstrap(settings) + if err != nil { + printErr(err, "is apm-server configured properly and Elasticsearch reachable?", json) + return err + } + return getAPIKey(client, &id, &name, validOnly, json) + }, + SilenceErrors: true, + SilenceUsage: true, + } + info.Flags().StringVar(&id, "id", "", "id of the API Key to query") + info.Flags().StringVar(&name, "name", "", + "name of the API Key(s) to query (several might match)") + info.Flags().BoolVar(&validOnly, "valid-only", false, + "only return valid API Keys (not expired or invalidated)") + info.Flags().BoolVar(&json, "json", false, + "prints the output of this command as JSON") + info.Flags().SortFlags = false + return info +} + +func verifyApikeyCmd(settings instance.Settings) *cobra.Command { + var credentials string + var ingest, sourcemap, agentConfig, json bool + short := `Check if a "credentials" string has the given privilege(s)` + long := short + `. +If no privilege(s) are specified, the credentials will be queried for all.` + verify := &cobra.Command{ + Use: "verify", + Short: short, + Long: long, + RunE: func(cmd *cobra.Command, args []string) error { + _, config, err := bootstrap(settings) + if err != nil { + printErr(err, "is apm-server configured properly and Elasticsearch reachable?", json) + return err + } + privileges := booleansToPrivileges(ingest, sourcemap, agentConfig) + if len(privileges) == 0 { + // can't use "*" for querying + privileges = auth.ActionsAll() + } + return verifyAPIKey(config, privileges, credentials, json) + }, + SilenceUsage: true, + SilenceErrors: true, + } + verify.Flags().StringVar(&credentials, "credentials", "", `credentials for which check privileges`) + verify.Flags().BoolVar(&ingest, "ingest", false, + fmt.Sprintf("ask for the %v privilege, required for ingesting events", auth.PrivilegeEventWrite)) + verify.Flags().BoolVar(&sourcemap, "sourcemap", false, + fmt.Sprintf("ask for the %v privilege, required for uploading sourcemaps", + auth.PrivilegeSourcemapWrite)) + verify.Flags().BoolVar(&agentConfig, "agent-config", false, + fmt.Sprintf("ask for the %v privilege, required for agents to read configuration remotely", + auth.PrivilegeAgentConfigRead)) + verify.Flags().BoolVar(&json, "json", false, + "prints the output of this command as JSON") + verify.Flags().SortFlags = false + + return verify +} + +// TODO is there just any other way to do this? +// without the wrapper, YAML settings in "apm-server" are not picked up by ucfg +type ApmConfig struct { + Config *config.Config `config:"apm-server"` +} + +// apm-server.api_key.enabled is implicitly true +func bootstrap(settings instance.Settings) (es.Client, *config.Config, error) { + + settings.ConfigOverrides = append(settings.ConfigOverrides, cfgfile.ConditionalOverride{ + Check: func(_ *common.Config) bool { + return true + }, + Config: common.MustNewConfigFrom(map[string]interface{}{ + "apm-server": map[string]interface{}{ + "api_key": map[string]interface{}{ + "enabled": true, + }, + }, + }), + }) + + beat, err := instance.NewInitializedBeat(settings) + if err != nil { + return nil, nil, err + } + + apm := ApmConfig{config.DefaultConfig(settings.Version)} + err = beat.RawConfig.Unpack(&apm) + if err != nil { + return nil, nil, err + } + + var client es.Client + err = apm.Config.APIKeyConfig.Setup(logp.NewLogger(logs.Config), beat.Config.Output.Config()) + if err == nil { + client, err = es.NewClient(apm.Config.APIKeyConfig.ESConfig) + } + return client, apm.Config, err +} + +func booleansToPrivileges(ingest, sourcemap, agentConfig bool) []es.Privilege { + privileges := make([]es.Privilege, 0) + if ingest { + privileges = append(privileges, auth.PrivilegeEventWrite.Action) + } + if sourcemap { + privileges = append(privileges, auth.PrivilegeSourcemapWrite.Action) + } + if agentConfig { + privileges = append(privileges, auth.PrivilegeAgentConfigRead.Action) + } + return privileges +} + // creates an API Key with the given privileges, *AND* all the privileges modeled in apm-server // we need to ensure forward-compatibility, for which future privileges must be created here and // during server startup because we don't know if customers will run this command @@ -113,7 +355,9 @@ func createAPIKeyWithPrivileges(client es.Client, apikeyName, expiry string, pri func getAPIKey(client es.Client, id, name *string, validOnly, asJSON bool) error { if isSet(id) { name = nil - } else if !(isSet(id) || isSet(name)) { + } else if isSet(name) { + id = nil + } else { return printErr(errors.New("could not query Elasticsearch"), `either "id" or "name" are required`, asJSON) @@ -145,7 +389,9 @@ func getAPIKey(client es.Client, id, name *string, validOnly, asJSON bool) error printText("Id ............. %s", apikey.Id) printText("Creation ....... %s", creation) printText("Invalidated .... %t", apikey.Invalidated) - printText("Expiration ..... %s", expiry) + if !apikey.Invalidated { + printText("Expiration ..... %s", expiry) + } printText("") transform.ApiKeys = append(transform.ApiKeys, apikey) } @@ -156,7 +402,9 @@ func getAPIKey(client es.Client, id, name *string, validOnly, asJSON bool) error func invalidateAPIKey(client es.Client, id, name *string, deletePrivileges, asJSON bool) error { if isSet(id) { name = nil - } else if !(isSet(id) || isSet(name)) { + } else if isSet(name) { + id = nil + } else { return printErr(errors.New("could not query Elasticsearch"), `either "id" or "name" are required`, asJSON) @@ -210,7 +458,7 @@ func invalidateAPIKey(client es.Client, id, name *string, deletePrivileges, asJS } func verifyAPIKey(config *config.Config, privileges []es.Privilege, credentials string, asJSON bool) error { - perms := make(es.Perms) + perms := make(es.Permissions) printText, printJSON := printers(asJSON) var err error diff --git a/cmd/root.go b/cmd/root.go index bc38c103a85..71d343a494a 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -20,13 +20,6 @@ package cmd import ( "fmt" - "github.com/spf13/cobra" - - auth "github.com/elastic/apm-server/beater/authorization" - - "github.com/elastic/apm-server/beater/config" - es "github.com/elastic/apm-server/elasticsearch" - "github.com/elastic/beats/libbeat/cfgfile" "github.com/spf13/pflag" @@ -50,11 +43,7 @@ const IdxPattern = "apm" // RootCmd for running apm-server. // This is the command that is used if no other command is specified. // Running `apm-server run` or `apm-server` is identical. -type ApmCmd struct { - *cmd.BeatsRootCmd -} - -var RootCmd = ApmCmd{} +var RootCmd *cmd.BeatsRootCmd func init() { overrides := common.MustNewConfigFrom(map[string]interface{}{ @@ -103,7 +92,7 @@ func init() { }, }, } - RootCmd = ApmCmd{cmd.GenRootCmdWithSettings(beater.New, settings)} + RootCmd = cmd.GenRootCmdWithSettings(beater.New, settings) RootCmd.AddCommand(genApikeyCmd(settings)) for _, cmd := range RootCmd.ExportCmd.Commands() { @@ -137,201 +126,3 @@ func init() { setup.Flags().Bool(cmd.PipelineKey, false, "Setup ingest pipelines") } - -func genApikeyCmd(settings instance.Settings) *cobra.Command { - - var client es.Client - apmConfig, err := bootstrap(settings) - if err == nil { - client, err = es.NewClient(apmConfig.APIKeyConfig.ESConfig) - } - - short := "Manage API Keys for communication between APM agents and server" - apikeyCmd := cobra.Command{ - Use: "apikey", - Short: short, - Long: short + `. -Most operations require the "manage_security" cluster privilege. Ensure to configure "apm-server.api_key.*" or -"output.elasticsearch.*" appropriately. APM Server will create security privileges for the "apm" application; -you can freely query them. If you modify or delete apm privileges, APM Server might reject all requests. -If an invalid argument is passed, nothing will be printed. -Check the Elastic Security API documentation for details.`, - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - return err - }, - } - - apikeyCmd.AddCommand( - createApikeyCmd(client), - invalidateApikeyCmd(client), - getApikeysCmd(client), - verifyApikeyCmd(apmConfig), - ) - - return &apikeyCmd -} - -func createApikeyCmd(client es.Client) *cobra.Command { - var keyName, expiration string - var ingest, sourcemap, agentConfig, json bool - short := "Create an API Key with the specified privilege(s)" - create := &cobra.Command{ - Use: "create", - Short: short, - Long: short + `. -If no privilege(s) are specified, the API Key will be valid for all. -Requires the "manage_security" cluster privilege in Elasticsearch.`, - // always need to return error for possible scripts checking the exit code, - // but printing the error must be done inside - RunE: func(cmd *cobra.Command, args []string) error { - privileges := booleansToPrivileges(ingest, sourcemap, agentConfig) - return createAPIKeyWithPrivileges(client, keyName, expiration, privileges, json) - }, - // these are needed to not break JSON formatting - // this has the caveat that if an invalid argument is passed, the command won't return anything - SilenceUsage: true, - SilenceErrors: true, - } - create.Flags().StringVar(&keyName, "name", "apm-key", "API Key name") - create.Flags().StringVar(&expiration, "expiration", "", - `expiration for the key, eg. "1d" (default never)`) - create.Flags().BoolVar(&ingest, "ingest", false, - fmt.Sprintf("give the %v privilege to this key, required for ingesting events", auth.PrivilegeEventWrite)) - create.Flags().BoolVar(&sourcemap, "sourcemap", false, - fmt.Sprintf("give the %v privilege to this key, required for uploading sourcemaps", - auth.PrivilegeSourcemapWrite)) - create.Flags().BoolVar(&agentConfig, "agent-config", false, - fmt.Sprintf("give the %v privilege to this key, required for agents to read configuration remotely", - auth.PrivilegeAgentConfigRead)) - create.Flags().BoolVar(&json, "json", false, - "prints the output of this command as JSON") - // this actually means "preserve sorting given in code" and not reorder them alphabetically - create.Flags().SortFlags = false - return create -} - -func invalidateApikeyCmd(client es.Client) *cobra.Command { - var id, name string - var purge, json bool - short := "Invalidate API Key(s) by Id or Name" - invalidate := &cobra.Command{ - Use: "invalidate", - Short: short, - Long: short + `. -If both "id" and "name" are supplied, only "id" will be used. -If neither of them are, an error will be returned. -Requires the "manage_security" cluster privilege in Elasticsearch.`, - RunE: func(cmd *cobra.Command, args []string) error { - return invalidateAPIKey(client, &id, &name, purge, json) - }, - SilenceErrors: true, - SilenceUsage: true, - } - invalidate.Flags().StringVar(&id, "id", "", "id of the API Key to delete") - invalidate.Flags().StringVar(&name, "name", "", - "name of the API Key(s) to delete (several might match)") - invalidate.Flags().BoolVar(&purge, "purge", false, - "also remove all privileges created and used by APM Server") - invalidate.Flags().BoolVar(&json, "json", false, - "prints the output of this command as JSON") - invalidate.Flags().SortFlags = false - return invalidate -} - -func getApikeysCmd(client es.Client) *cobra.Command { - var id, name string - var validOnly, json bool - short := "Query API Key(s) by Id or Name" - info := &cobra.Command{ - Use: "info", - Short: short, - Long: short + `. -If both "id" and "name" are supplied, only "id" will be used. -If neither of them are, an error will be returned. -Requires the "manage_security" cluster privilege in Elasticsearch.`, - RunE: func(cmd *cobra.Command, args []string) error { - return getAPIKey(client, &id, &name, validOnly, json) - }, - SilenceErrors: true, - SilenceUsage: true, - } - info.Flags().StringVar(&id, "id", "", "id of the API Key to query") - info.Flags().StringVar(&name, "name", "", - "name of the API Key(s) to query (several might match)") - info.Flags().BoolVar(&validOnly, "valid-only", false, - "only return valid API Keys (not expired or invalidated)") - info.Flags().BoolVar(&json, "json", false, - "prints the output of this command as JSON") - info.Flags().SortFlags = false - return info -} - -func verifyApikeyCmd(config *config.Config) *cobra.Command { - var credentials string - var ingest, sourcemap, agentConfig, json bool - short := `Check if a "credentials" string has the given privilege(s)` - long := short + `. -If no privilege(s) are specified, the credentials will be queried for all.` - verify := &cobra.Command{ - Use: "verify", - Short: short, - Long: long, - RunE: func(cmd *cobra.Command, args []string) error { - privileges := booleansToPrivileges(ingest, sourcemap, agentConfig) - return verifyAPIKey(config, privileges, credentials, json) - }, - SilenceUsage: true, - SilenceErrors: true, - } - verify.Flags().StringVar(&credentials, "credentials", "", `credentials for which check privileges`) - verify.Flags().BoolVar(&ingest, "ingest", false, - fmt.Sprintf("ask for the %v privilege, required for ingesting events", auth.PrivilegeEventWrite)) - verify.Flags().BoolVar(&sourcemap, "sourcemap", false, - fmt.Sprintf("ask for the %v privilege, required for uploading sourcemaps", - auth.PrivilegeSourcemapWrite)) - verify.Flags().BoolVar(&agentConfig, "agent-config", false, - fmt.Sprintf("ask for the %v privilege, required for agents to read configuration remotely", - auth.PrivilegeAgentConfigRead)) - verify.Flags().BoolVar(&json, "json", false, - "prints the output of this command as JSON") - verify.Flags().SortFlags = false - - return verify -} - -// created the beat, instantiate configuration, and so on -// apm-server.api_key.enabled is implicitly true -func bootstrap(settings instance.Settings) (*config.Config, error) { - beat, err := instance.NewBeat(settings.Name, settings.IndexPrefix, settings.Version) - if err != nil { - return nil, err - } - err = beat.InitWithSettings(settings) - if err != nil { - return nil, err - } - - outCfg := beat.Config.Output - return config.NewConfig(settings.Version, beat.RawConfig, outCfg.Config()) -} - -// if all are false, returns any ("*") -// this is because Elasticsearch requires at least 1 privilege for most queries, -// so "*" acts as default -func booleansToPrivileges(ingest, sourcemap, agentConfig bool) []es.Privilege { - privileges := make([]es.Privilege, 0) - if ingest { - privileges = append(privileges, auth.PrivilegeEventWrite.Action) - } - if sourcemap { - privileges = append(privileges, auth.PrivilegeSourcemapWrite.Action) - } - if agentConfig { - privileges = append(privileges, auth.PrivilegeAgentConfigRead.Action) - } - any := ingest || sourcemap || agentConfig - if !any { - privileges = append(privileges, auth.ActionAny) - } - return privileges -} diff --git a/elasticsearch/client.go b/elasticsearch/client.go index 1c920137272..e7df0cde280 100644 --- a/elasticsearch/client.go +++ b/elasticsearch/client.go @@ -31,9 +31,9 @@ import ( "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/version" - v7 "github.com/elastic/go-elasticsearch/v7" + esv7 "github.com/elastic/go-elasticsearch/v7" - v8 "github.com/elastic/go-elasticsearch/v8" + esv8 "github.com/elastic/go-elasticsearch/v8" ) // Client is an interface designed to abstract away version differences between elasticsearch clients @@ -47,17 +47,17 @@ type Client interface { } type clientV8 struct { - c *v8.Client + v8 *esv8.Client } // Search satisfies the Client interface for version 8 -func (v8 clientV8) Search(index string, body io.Reader) (int, io.ReadCloser, error) { - response, err := v8.c.Search( - v8.c.Search.WithContext(context.Background()), - v8.c.Search.WithIndex(index), - v8.c.Search.WithBody(body), - v8.c.Search.WithTrackTotalHits(true), - v8.c.Search.WithPretty(), +func (c clientV8) Search(index string, body io.Reader) (int, io.ReadCloser, error) { + response, err := c.v8.Search( + c.v8.Search.WithContext(context.Background()), + c.v8.Search.WithIndex(index), + c.v8.Search.WithBody(body), + c.v8.Search.WithTrackTotalHits(true), + c.v8.Search.WithPretty(), ) if err != nil { return 0, nil, err @@ -65,26 +65,26 @@ func (v8 clientV8) Search(index string, body io.Reader) (int, io.ReadCloser, err return response.StatusCode, response.Body, nil } -func (v8 clientV8) JSONRequest(method, path string, body interface{}, headers ...string) JSONResponse { +func (c clientV8) JSONRequest(method, path string, body interface{}, headers ...string) JSONResponse { req, err := makeRequest(method, path, body, headers...) if err != nil { return JSONResponse{nil, err} } - return parseResponse(v8.c.Perform(req)) + return parseResponse(c.v8.Perform(req)) } type clientV7 struct { - c *v7.Client + v7 *esv7.Client } // Search satisfies the Client interface for version 7 -func (v7 clientV7) Search(index string, body io.Reader) (int, io.ReadCloser, error) { - response, err := v7.c.Search( - v7.c.Search.WithContext(context.Background()), - v7.c.Search.WithIndex(index), - v7.c.Search.WithBody(body), - v7.c.Search.WithTrackTotalHits(true), - v7.c.Search.WithPretty(), +func (c clientV7) Search(index string, body io.Reader) (int, io.ReadCloser, error) { + response, err := c.v7.Search( + c.v7.Search.WithContext(context.Background()), + c.v7.Search.WithIndex(index), + c.v7.Search.WithBody(body), + c.v7.Search.WithTrackTotalHits(true), + c.v7.Search.WithPretty(), ) if err != nil { return 0, nil, err @@ -92,12 +92,12 @@ func (v7 clientV7) Search(index string, body io.Reader) (int, io.ReadCloser, err return response.StatusCode, response.Body, nil } -func (v7 clientV7) JSONRequest(method, path string, body interface{}, headers ...string) JSONResponse { +func (c clientV7) JSONRequest(method, path string, body interface{}, headers ...string) JSONResponse { req, err := makeRequest(method, path, body, headers...) if err != nil { return JSONResponse{nil, err} } - return parseResponse(v7.c.Perform(req)) + return parseResponse(c.v7.Perform(req)) } // NewClient parses the given config and returns a version-aware client as an interface @@ -123,8 +123,8 @@ func NewVersionedClient(apikey, user, pwd string, addresses []string, transport return clientV7{c}, err } -func newV7Client(apikey, user, pwd string, addresses []string, transport http.RoundTripper) (*v7.Client, error) { - return v7.NewClient(v7.Config{ +func newV7Client(apikey, user, pwd string, addresses []string, transport http.RoundTripper) (*esv7.Client, error) { + return esv7.NewClient(esv7.Config{ APIKey: apikey, Username: user, Password: pwd, @@ -133,8 +133,8 @@ func newV7Client(apikey, user, pwd string, addresses []string, transport http.Ro }) } -func newV8Client(apikey, user, pwd string, addresses []string, transport http.RoundTripper) (*v8.Client, error) { - return v8.NewClient(v8.Config{ +func newV8Client(apikey, user, pwd string, addresses []string, transport http.RoundTripper) (*esv8.Client, error) { + return esv8.NewClient(esv8.Config{ APIKey: apikey, Username: user, Password: pwd, @@ -152,11 +152,8 @@ func (r JSONResponse) DecodeTo(i interface{}) error { if r.err != nil { return r.err } - bs, err := ioutil.ReadAll(r.content) - if err != nil { - return err - } - err = json.Unmarshal(bs, i) + defer r.content.Close() + err := json.NewDecoder(r.content).Decode(&i) return err } @@ -174,12 +171,9 @@ func makeRequest(method, path string, body interface{}, headers ...string) (*htt } u, _ := url.Parse(path) req := &http.Request{ - Method: method, - URL: u, - Proto: "HTTP/1.1", - ProtoMajor: 1, - ProtoMinor: 1, - Header: header, + Method: method, + URL: u, + Header: header, } bs, err := json.Marshal(body) if err != nil { diff --git a/elasticsearch/security_api.go b/elasticsearch/security_api.go index 2b9d0dbe369..cc15216d4c1 100644 --- a/elasticsearch/security_api.go +++ b/elasticsearch/security_api.go @@ -24,7 +24,7 @@ import ( "strconv" ) -// requires manage_security cluster privilege +// CreateAPIKey requires manage_security cluster privilege func CreateAPIKey(client Client, apikeyReq CreateApiKeyRequest) (CreateApiKeyResponse, error) { response := client.JSONRequest(http.MethodPut, "/_security/api_key", apikeyReq) @@ -33,7 +33,7 @@ func CreateAPIKey(client Client, apikeyReq CreateApiKeyRequest) (CreateApiKeyRes return apikey, err } -// requires manage_security cluster privilege +// GetAPIKeys requires manage_security cluster privilege func GetAPIKeys(client Client, apikeyReq GetApiKeyRequest) (GetApiKeyResponse, error) { u := url.URL{Path: "/_security/api_key"} params := url.Values{} @@ -52,7 +52,7 @@ func GetAPIKeys(client Client, apikeyReq GetApiKeyRequest) (GetApiKeyResponse, e return apikey, err } -// requires manage_security cluster privilege +// CreatePrivileges requires manage_security cluster privilege func CreatePrivileges(client Client, privilegesReq CreatePrivilegesRequest) (CreatePrivilegesResponse, error) { response := client.JSONRequest(http.MethodPut, "/_security/privilege", privilegesReq) @@ -61,7 +61,7 @@ func CreatePrivileges(client Client, privilegesReq CreatePrivilegesRequest) (Cre return privileges, err } -// requires manage_security cluster privilege +// InvalidateAPIKey requires manage_security cluster privilege func InvalidateAPIKey(client Client, apikeyReq InvalidateApiKeyRequest) (InvalidateApiKeyResponse, error) { response := client.JSONRequest(http.MethodDelete, "/_security/api_key", apikeyReq) @@ -70,8 +70,8 @@ func InvalidateAPIKey(client Client, apikeyReq InvalidateApiKeyRequest) (Invalid return confirmation, err } +// DeletePrivileges requires manage_security cluster privilege func DeletePrivileges(client Client, privilegesReq DeletePrivilegeRequest) (DeletePrivilegeResponse, error) { - // requires manage_security cluster privilege path := fmt.Sprintf("/_security/privilege/%v/%v", privilegesReq.Application, privilegesReq.Privilege) response := client.JSONRequest(http.MethodDelete, path, nil) @@ -137,7 +137,6 @@ type DeletePrivilegeRequest struct { Privilege PrivilegeName `json:"privilege"` } -//noinspection GoRedundantParens type DeletePrivilegeResponse map[AppName](map[PrivilegeName]DeleteResponse) type RoleDescriptor map[AppName]Applications @@ -177,9 +176,9 @@ type PrivilegeResponse map[Privilege]PutResponse type PrivilegeGroup map[PrivilegeName]Actions -type Perms map[Privilege]bool +type Permissions map[Privilege]bool -type PrivilegesPerResource map[Resource]Perms +type PrivilegesPerResource map[Resource]Permissions type Actions struct { Actions []Privilege `json:"actions"` diff --git a/x-pack/apm-server/cmd/root.go b/x-pack/apm-server/cmd/root.go index 0ebf14e3e99..c85dc435118 100644 --- a/x-pack/apm-server/cmd/root.go +++ b/x-pack/apm-server/cmd/root.go @@ -14,5 +14,5 @@ import ( var RootCmd = cmd.RootCmd func init() { - xpackcmd.AddXPack(RootCmd.BeatsRootCmd, cmd.Name) + xpackcmd.AddXPack(RootCmd, cmd.Name) } From 9e68751789d5e08beddef18e5516db0a5a39773e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20=C3=81lvarez?= Date: Fri, 20 Dec 2019 14:45:38 +0100 Subject: [PATCH 12/36] require credentials --- cmd/apikey.go | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/cmd/apikey.go b/cmd/apikey.go index 384a9308e7c..90893fedf4c 100644 --- a/cmd/apikey.go +++ b/cmd/apikey.go @@ -198,6 +198,12 @@ If no privilege(s) are specified, the credentials will be queried for all.` printErr(err, "is apm-server configured properly and Elasticsearch reachable?", json) return err } + if credentials == "" { + err := errors.New("credentials argument is required") + // we can't use Cobra to mark the flag as required because it won't print the error as JSON + printErr(err, "", json) + return err + } privileges := booleansToPrivileges(ingest, sourcemap, agentConfig) if len(privileges) == 0 { // can't use "*" for querying @@ -208,7 +214,7 @@ If no privilege(s) are specified, the credentials will be queried for all.` SilenceUsage: true, SilenceErrors: true, } - verify.Flags().StringVar(&credentials, "credentials", "", `credentials for which check privileges`) + verify.Flags().StringVar(&credentials, "credentials", "", `credentials for which check privileges (required)`) verify.Flags().BoolVar(&ingest, "ingest", false, fmt.Sprintf("ask for the %v privilege, required for ingesting events", auth.PrivilegeEventWrite)) verify.Flags().BoolVar(&sourcemap, "sourcemap", false, @@ -358,9 +364,7 @@ func getAPIKey(client es.Client, id, name *string, validOnly, asJSON bool) error } else if isSet(name) { id = nil } else { - return printErr(errors.New("could not query Elasticsearch"), - `either "id" or "name" are required`, - asJSON) + return printErr(errors.New(`either "id" or "name" are required`), "", asJSON) } request := es.GetApiKeyRequest{ ApiKeyQuery: es.ApiKeyQuery{ @@ -405,9 +409,7 @@ func invalidateAPIKey(client es.Client, id, name *string, deletePrivileges, asJS } else if isSet(name) { id = nil } else { - return printErr(errors.New("could not query Elasticsearch"), - `either "id" or "name" are required`, - asJSON) + return printErr(errors.New(`either "id" or "name" are required`), "", asJSON) } invalidateKeysRequest := es.InvalidateApiKeyRequest{ ApiKeyQuery: es.ApiKeyQuery{ @@ -566,7 +568,7 @@ func printErr(err error, help string, asJSON bool) error { // err.Error() is a bare string, likely coming from apm-server data, _ = json.MarshalIndent(struct { Error string `json:"error"` - Help string `json:"help"` + Help string `json:"help,omitempty"` }{ Error: err.Error(), Help: help, From 7843321a826e9289264058eb962d08f247f5972c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20=C3=81lvarez?= Date: Fri, 20 Dec 2019 15:17:35 +0100 Subject: [PATCH 13/36] more code review comments --- beater/authorization/apikey.go | 4 ++-- beater/authorization/apikey_test.go | 4 ++-- beater/authorization/builder.go | 6 +++--- beater/authorization/builder_test.go | 4 ++-- beater/authorization/privilege.go | 6 +++--- cmd/apikey.go | 18 +++++++++--------- elasticsearch/security_api.go | 20 +++++++++----------- 7 files changed, 30 insertions(+), 32 deletions(-) diff --git a/beater/authorization/apikey.go b/beater/authorization/apikey.go index b7234cec4f4..bbcc3a40de7 100644 --- a/beater/authorization/apikey.go +++ b/beater/authorization/apikey.go @@ -41,7 +41,7 @@ var ( type apikeyBuilder struct { esClient es.Client cache *privilegesCache - anyOfPrivileges []es.Privilege + anyOfPrivileges []es.PrivilegeAction } type apikeyAuth struct { @@ -50,7 +50,7 @@ type apikeyAuth struct { key string } -func newApikeyBuilder(client es.Client, cache *privilegesCache, anyOfPrivileges []es.Privilege) *apikeyBuilder { +func newApikeyBuilder(client es.Client, cache *privilegesCache, anyOfPrivileges []es.PrivilegeAction) *apikeyBuilder { return &apikeyBuilder{client, cache, anyOfPrivileges} } diff --git a/beater/authorization/apikey_test.go b/beater/authorization/apikey_test.go index 62520544f6f..4f05635cfba 100644 --- a/beater/authorization/apikey_test.go +++ b/beater/authorization/apikey_test.go @@ -163,7 +163,7 @@ type apikeyTestcase struct { transport *estest.Transport client elasticsearch.Client cache *privilegesCache - anyOfPrivileges []elasticsearch.Privilege + anyOfPrivileges []elasticsearch.PrivilegeAction builder *apikeyBuilder } @@ -186,7 +186,7 @@ func (tc *apikeyTestcase) setup(t *testing.T) { tc.cache = newPrivilegesCache(time.Minute, 5) } if tc.anyOfPrivileges == nil { - tc.anyOfPrivileges = []elasticsearch.Privilege{PrivilegeEventWrite.Action, PrivilegeSourcemapWrite.Action} + tc.anyOfPrivileges = []elasticsearch.PrivilegeAction{PrivilegeEventWrite.Action, PrivilegeSourcemapWrite.Action} } tc.builder = newApikeyBuilder(tc.client, tc.cache, tc.anyOfPrivileges) } diff --git a/beater/authorization/builder.go b/beater/authorization/builder.go index ad637ab762a..ee6ae41f322 100644 --- a/beater/authorization/builder.go +++ b/beater/authorization/builder.go @@ -62,7 +62,7 @@ func NewBuilder(cfg config.Config) (*Builder, error) { size := cfg.APIKeyConfig.LimitMin * cacheTimeoutMinute cache := newPrivilegesCache(cacheTimeoutMinute*time.Minute, size) - b.apikey = newApikeyBuilder(client, cache, []elasticsearch.Privilege{}) + b.apikey = newApikeyBuilder(client, cache, []elasticsearch.PrivilegeAction{}) b.fallback = DenyAuth{} } if cfg.SecretToken != "" { @@ -74,12 +74,12 @@ func NewBuilder(cfg config.Config) (*Builder, error) { } // ForPrivilege creates an authorization Handler checking for this privilege -func (b *Builder) ForPrivilege(privilege elasticsearch.Privilege) *Handler { +func (b *Builder) ForPrivilege(privilege elasticsearch.PrivilegeAction) *Handler { return b.ForAnyOfPrivileges(privilege) } // ForAnyOfPrivileges creates an authorization Handler checking for any of the provided privileges -func (b *Builder) ForAnyOfPrivileges(privileges ...elasticsearch.Privilege) *Handler { +func (b *Builder) ForAnyOfPrivileges(privileges ...elasticsearch.PrivilegeAction) *Handler { handler := Handler{bearer: b.bearer, fallback: b.fallback} if b.apikey != nil { handler.apikey = newApikeyBuilder(b.apikey.esClient, b.apikey.cache, privileges) diff --git a/beater/authorization/builder_test.go b/beater/authorization/builder_test.go index f665d11a585..06ab007d5f2 100644 --- a/beater/authorization/builder_test.go +++ b/beater/authorization/builder_test.go @@ -72,8 +72,8 @@ func TestBuilder(t *testing.T) { assert.Equal(t, builder.bearer, h.bearer) assert.Equal(t, builder.fallback, h.fallback) if tc.withApikey { - assert.Equal(t, []elasticsearch.Privilege{}, builder.apikey.anyOfPrivileges) - assert.Equal(t, []elasticsearch.Privilege{PrivilegeSourcemapWrite.Action}, h.apikey.anyOfPrivileges) + assert.Equal(t, []elasticsearch.PrivilegeAction{}, builder.apikey.anyOfPrivileges) + assert.Equal(t, []elasticsearch.PrivilegeAction{PrivilegeSourcemapWrite.Action}, h.apikey.anyOfPrivileges) assert.Equal(t, builder.apikey.esClient, h.apikey.esClient) assert.Equal(t, builder.apikey.cache, h.apikey.cache) } diff --git a/beater/authorization/privilege.go b/beater/authorization/privilege.go index d51ebeb6cf8..329da04ab1c 100644 --- a/beater/authorization/privilege.go +++ b/beater/authorization/privilege.go @@ -31,9 +31,9 @@ var ( PrivilegeSourcemapWrite = es.NewPrivilege("sourcemap", "sourcemap:write") PrivilegesAll = []es.NamedPrivilege{PrivilegeAgentConfigRead, PrivilegeEventWrite, PrivilegeSourcemapWrite} // ActionAny can't be used for querying, use ActionsAll instead - ActionAny = es.Privilege("*") - ActionsAll = func() []es.Privilege { - actions := make([]es.Privilege, 0) + ActionAny = es.PrivilegeAction("*") + ActionsAll = func() []es.PrivilegeAction { + actions := make([]es.PrivilegeAction, 0) for _, privilege := range PrivilegesAll { actions = append(actions, privilege.Action) } diff --git a/cmd/apikey.go b/cmd/apikey.go index 90893fedf4c..6c58f38942a 100644 --- a/cmd/apikey.go +++ b/cmd/apikey.go @@ -89,7 +89,7 @@ Requires the "manage_security" cluster privilege in Elasticsearch.`, } privileges := booleansToPrivileges(ingest, sourcemap, agentConfig) if len(privileges) == 0 { - privileges = []es.Privilege{auth.ActionAny} + privileges = []es.PrivilegeAction{auth.ActionAny} } return createAPIKeyWithPrivileges(client, keyName, expiration, privileges, json) }, @@ -271,8 +271,8 @@ func bootstrap(settings instance.Settings) (es.Client, *config.Config, error) { return client, apm.Config, err } -func booleansToPrivileges(ingest, sourcemap, agentConfig bool) []es.Privilege { - privileges := make([]es.Privilege, 0) +func booleansToPrivileges(ingest, sourcemap, agentConfig bool) []es.PrivilegeAction { + privileges := make([]es.PrivilegeAction, 0) if ingest { privileges = append(privileges, auth.PrivilegeEventWrite.Action) } @@ -288,15 +288,15 @@ func booleansToPrivileges(ingest, sourcemap, agentConfig bool) []es.Privilege { // creates an API Key with the given privileges, *AND* all the privileges modeled in apm-server // we need to ensure forward-compatibility, for which future privileges must be created here and // during server startup because we don't know if customers will run this command -func createAPIKeyWithPrivileges(client es.Client, apikeyName, expiry string, privileges []es.Privilege, asJSON bool) error { +func createAPIKeyWithPrivileges(client es.Client, apikeyName, expiry string, privileges []es.PrivilegeAction, asJSON bool) error { var privilegesRequest = make(es.CreatePrivilegesRequest) event := auth.PrivilegeEventWrite agentConfig := auth.PrivilegeAgentConfigRead sourcemap := auth.PrivilegeSourcemapWrite privilegesRequest[auth.Application] = map[es.PrivilegeName]es.Actions{ - agentConfig.Name: {Actions: []es.Privilege{agentConfig.Action}}, - event.Name: {Actions: []es.Privilege{event.Action}}, - sourcemap.Name: {Actions: []es.Privilege{sourcemap.Action}}, + agentConfig.Name: {Actions: []es.PrivilegeAction{agentConfig.Action}}, + event.Name: {Actions: []es.PrivilegeAction{event.Action}}, + sourcemap.Name: {Actions: []es.PrivilegeAction{sourcemap.Action}}, } privilegesCreated, err := es.CreatePrivileges(client, privilegesRequest) @@ -459,7 +459,7 @@ func invalidateAPIKey(client es.Client, id, name *string, deletePrivileges, asJS return printJSON(out) } -func verifyAPIKey(config *config.Config, privileges []es.Privilege, credentials string, asJSON bool) error { +func verifyAPIKey(config *config.Config, privileges []es.PrivilegeAction, credentials string, asJSON bool) error { perms := make(es.Permissions) printText, printJSON := printers(asJSON) @@ -498,7 +498,7 @@ func humanBool(b bool) string { return "No" } -func humanPrivilege(privilege es.Privilege) string { +func humanPrivilege(privilege es.PrivilegeAction) string { switch privilege { case auth.ActionAny: return fmt.Sprintf("all privileges (\"%v\")", privilege) diff --git a/elasticsearch/security_api.go b/elasticsearch/security_api.go index cc15216d4c1..cccbda42a2f 100644 --- a/elasticsearch/security_api.go +++ b/elasticsearch/security_api.go @@ -146,9 +146,9 @@ type Applications struct { } type Application struct { - Name AppName `json:"application"` - Privileges []Privilege `json:"privileges"` - Resources []Resource `json:"resources"` + Name AppName `json:"application"` + Privileges []PrivilegeAction `json:"privileges"` + Resources []Resource `json:"resources"` } type ApiKeyResponse struct { @@ -172,16 +172,16 @@ type ApiKey struct { Credentials *string `json:"credentials,omitempty"` } -type PrivilegeResponse map[Privilege]PutResponse +type PrivilegeResponse map[PrivilegeAction]PutResponse type PrivilegeGroup map[PrivilegeName]Actions -type Permissions map[Privilege]bool +type Permissions map[PrivilegeAction]bool type PrivilegesPerResource map[Resource]Permissions type Actions struct { - Actions []Privilege `json:"actions"` + Actions []PrivilegeAction `json:"actions"` } type PutResponse struct { @@ -202,18 +202,16 @@ type Resource string // in apm-server, each name is associated with one action, but that needs not to be the case (see PrivilegeGroup) type NamedPrivilege struct { Name PrivilegeName - Action Privilege + Action PrivilegeAction } -// sometimes referred in Elasticsearch documentation as "action" -// we keep the name "privilege" because is more informative -type Privilege string +type PrivilegeAction string type PrivilegeName string func NewPrivilege(name, action string) NamedPrivilege { return NamedPrivilege{ Name: PrivilegeName(name), - Action: Privilege(action), + Action: PrivilegeAction(action), } } From 653bdd93e4c9b42b8a62065e0dc2457a1ecec083 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20=C3=81lvarez?= Date: Wed, 8 Jan 2020 11:30:55 +0100 Subject: [PATCH 14/36] Add integration tests --- tests/system/test_access.py | 2 + tests/system/test_apikey.py | 161 ++++++++++++++++++++++++++++++++++++ 2 files changed, 163 insertions(+) create mode 100644 tests/system/test_apikey.py diff --git a/tests/system/test_access.py b/tests/system/test_access.py index 9623458e81c..d69933e01b9 100644 --- a/tests/system/test_access.py +++ b/tests/system/test_access.py @@ -359,6 +359,7 @@ def upload(token): resp = upload(token) assert resp.status_code == 202, "token: {}, status_code: {}".format(token, resp.status_code) + @integration_test class TestSelfInstrumentationWithAPIKeys(BaseAPIKeySetup): @@ -388,6 +389,7 @@ def have_apm_server_traces(): return response['count'] != 0 self.wait_until(have_apm_server_traces, max_timeout=20, name="waiting for apm-server traces") + @integration_test class TestSecureServerBaseTest(ServerSetUpBaseTest): @classmethod diff --git a/tests/system/test_apikey.py b/tests/system/test_apikey.py new file mode 100644 index 00000000000..ab27d478b8f --- /dev/null +++ b/tests/system/test_apikey.py @@ -0,0 +1,161 @@ +from apmserver import BaseTest, integration_test +from elasticsearch import Elasticsearch +import inspect +import json +import random + + +class APIKeyBaseTest(BaseTest): + api_key_name = "apm_integration_key" + + def config(self): + return { + "elasticsearch_host": self.get_elasticsearch_url(), + "file_enabled": "false", + "kibana_enabled": "false", + } + + def setUp(self): + super(APIKeyBaseTest, self).setUp() + self.es = Elasticsearch([self.get_elasticsearch_url()]) + self.kibana_url = self.get_kibana_url() + self.render_config_template(**self.config()) + + def subcommand_output(self, *args): + log = self.subcommand(*args) + # command and go test output is combined in log, pull out the command output + command_output = self._trim_golog(log) + return json.loads(command_output) + + def subcommand(self, *args): + caller = inspect.getouterframes(inspect.currentframe())[1][3] + logfile = self.beat_name + "-" + caller + str(random.randint(0, 9999)) + "-" + args[0] + ".log" + subcmd = ["apikey"] + subcmd.extend(args) + subcmd.append("--json") + self.run_beat(logging_args=[], extra_args=subcmd, output=logfile) + return self.get_log(logfile) + + @staticmethod + def _trim_golog(log): + pos = -1 + for _ in range(2): + pos = log[:pos].rfind("\n") + command_output = log[:pos] + for trimmed in log[pos:].strip().splitlines(): + assert trimmed.split(None, 1)[0] in ("PASS", "coverage:"), trimmed + return command_output + + def create(self, *args): + return self.subcommand_output("create", "--name", self.api_key_name, *args) + + +@integration_test +class APIKeyTest(APIKeyBaseTest): + """ + Tests the apikey subcommand. + """ + + def tearDown(self): + super(APIKeyBaseTest, self).tearDown() + invalidated = self.subcommand_output("invalidate", "--name", self.api_key_name) + assert invalidated["error_count"] == 0 + + def test_create(self): + apikey = self.create() + + assert apikey["name"] == self.api_key_name, apikey + + for privilege in ["sourcemap", "agentConfig", "event"]: + apikey["created_privileges"]["apm"][privilege]["created"] = True, apikey + + for attr in ["id", "api_key", "credentials"]: + assert apikey[attr] != "", apikey + + def test_create_with_settings_override(self): + apikey = self.create( + "-E", "output.elasticsearch.enabled=false", + "-E", "apm-server.api_key.elasticsearch.hosts=[{}]".format(self.get_elasticsearch_url()) + ) + assert apikey["credentials"] is not None, apikey + + def test_create_with_expiration(self): + apikey = self.create("--expiration", "1d") + assert apikey["expiration"] is not None, apikey + + def test_invalidate_by_id(self): + apikey = self.create() + invalidated = self.subcommand_output("invalidate", "--id", apikey["id"]) + assert invalidated["invalidated_api_keys"] == [apikey["id"]], invalidated + assert invalidated["error_count"] == 0, invalidated + + def test_invalidate_by_name(self): + self.create() + self.create() + invalidated = self.subcommand_output("invalidate", "--name", self.api_key_name) + assert len(invalidated["invalidated_api_keys"]) == 2, invalidated + assert invalidated["error_count"] == 0, invalidated + + def test_info_by_id(self): + self.create() + apikey = self.create() + info = self.subcommand_output("info", "--id", apikey["id"]) + assert len(info["api_keys"]) == 1, info + assert info["api_keys"][0]["username"] == "admin", info + assert info["api_keys"][0]["id"] == apikey["id"], info + assert info["api_keys"][0]["name"] == apikey["name"], info + assert info["api_keys"][0]["invalidated"] is False, info + + def test_info_by_name(self): + apikey = self.create() + invalidated = self.subcommand_output("invalidate", "--id", apikey["id"]) + assert invalidated["error_count"] == 0 + self.create() + self.create() + + info = self.subcommand_output("info", "--name", self.api_key_name) + # can't test exact number because these tests have side effects + assert len(info["api_keys"]) > 2, info + + info = self.subcommand_output("info", "--name", self.api_key_name, "--valid-only") + assert len(info["api_keys"]) == 2, info + + def test_verify_all(self): + apikey = self.create() + result = self.subcommand_output("verify", "--credentials", apikey["credentials"]) + assert result == {'event:write': True, 'config_agent:read': True, 'sourcemap:write': True}, result + + for privilege in ["ingest", "sourcemap", "agent-config"]: + result = self.subcommand_output("verify", "--credentials", apikey["credentials"], "--" + privilege) + assert len(result) == 1, result + assert result.values()[0] is True + + def test_verify_each(self): + apikey = self.create("--ingest") + result = self.subcommand_output("verify", "--credentials", apikey["credentials"]) + assert result == {'event:write': True, 'config_agent:read': False, 'sourcemap:write': False}, result + + apikey = self.create("--sourcemap") + result = self.subcommand_output("verify", "--credentials", apikey["credentials"]) + assert result == {'event:write': False, 'config_agent:read': False, 'sourcemap:write': True}, result + + apikey = self.create("--agent-config") + result = self.subcommand_output("verify", "--credentials", apikey["credentials"]) + assert result == {'event:write': False, 'config_agent:read': True, 'sourcemap:write': False}, result + + +@integration_test +class APIKeyBadUserTest(APIKeyBaseTest): + + def config(self): + return { + "elasticsearch_host": self.get_elasticsearch_url(user="apm_server_user", password="changeme"), + "file_enabled": "false", + "kibana_enabled": "false", + } + + def test_create_bad_user(self): + out = self.subcommand("create", "--name", self.api_key_name) + result = json.loads(out) + assert result["status"] == 401, result + assert result["error"] is not None From 1c222b0ebb26ea9f6ce772dec68ff9b995230229 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20=C3=81lvarez?= Date: Wed, 8 Jan 2020 13:06:53 +0100 Subject: [PATCH 15/36] return Cobra errors --- cmd/apikey.go | 99 ++++++++++++++++++++++----------------------------- 1 file changed, 43 insertions(+), 56 deletions(-) diff --git a/cmd/apikey.go b/cmd/apikey.go index 6c58f38942a..b78beba5533 100644 --- a/cmd/apikey.go +++ b/cmd/apikey.go @@ -20,6 +20,7 @@ package cmd import ( "encoding/base64" "encoding/json" + "errors" "fmt" "io" "io/ioutil" @@ -37,8 +38,6 @@ import ( "github.com/elastic/beats/libbeat/cmd/instance" "github.com/elastic/beats/libbeat/common" - "github.com/pkg/errors" - "github.com/elastic/apm-server/beater/config" "github.com/elastic/apm-server/beater/headers" @@ -56,7 +55,6 @@ func genApikeyCmd(settings instance.Settings) *cobra.Command { Most operations require the "manage_security" cluster privilege. Ensure to configure "apm-server.api_key.*" or "output.elasticsearch.*" appropriately. APM Server will create security privileges for the "apm" application; you can freely query them. If you modify or delete apm privileges, APM Server might reject all requests. -If an invalid argument is passed, nothing will be printed. Check the Elastic Security API documentation for details.`, } @@ -77,26 +75,21 @@ func createApikeyCmd(settings instance.Settings) *cobra.Command { Use: "create", Short: short, Long: short + `. -If no privilege(s) are specified, the API Key will be valid for all. -Requires the "manage_security" cluster privilege in Elasticsearch.`, +If no privilege(s) are specified, the API Key will be valid for all.`, // always need to return error for possible scripts checking the exit code, // but printing the error must be done inside RunE: func(cmd *cobra.Command, args []string) error { client, _, err := bootstrap(settings) if err != nil { - printErr(err, "is apm-server configured properly and Elasticsearch reachable?", json) return err } privileges := booleansToPrivileges(ingest, sourcemap, agentConfig) if len(privileges) == 0 { privileges = []es.PrivilegeAction{auth.ActionAny} } - return createAPIKeyWithPrivileges(client, keyName, expiration, privileges, json) + createAPIKeyWithPrivileges(client, keyName, expiration, privileges, json) + return nil }, - // these are needed to not break JSON formatting - // this has the caveat that if an invalid argument is passed, the command won't return anything - SilenceUsage: true, - SilenceErrors: true, } create.Flags().StringVar(&keyName, "name", "apm-key", "API Key name") create.Flags().StringVar(&expiration, "expiration", "", @@ -125,18 +118,18 @@ func invalidateApikeyCmd(settings instance.Settings) *cobra.Command { Short: short, Long: short + `. If both "id" and "name" are supplied, only "id" will be used. -If neither of them are, an error will be returned. -Requires the "manage_security" cluster privilege in Elasticsearch.`, +If neither of them are, an error will be returned.`, RunE: func(cmd *cobra.Command, args []string) error { + if id == "" && name == "" { + return errors.New(`either "id" or "name" are required`) + } client, _, err := bootstrap(settings) if err != nil { - printErr(err, "is apm-server configured properly and Elasticsearch reachable?", json) return err } - return invalidateAPIKey(client, &id, &name, purge, json) + invalidateAPIKey(client, &id, &name, purge, json) + return nil }, - SilenceErrors: true, - SilenceUsage: true, } invalidate.Flags().StringVar(&id, "id", "", "id of the API Key to delete") invalidate.Flags().StringVar(&name, "name", "", @@ -158,18 +151,18 @@ func getApikeysCmd(settings instance.Settings) *cobra.Command { Short: short, Long: short + `. If both "id" and "name" are supplied, only "id" will be used. -If neither of them are, an error will be returned. -Requires the "manage_security" cluster privilege in Elasticsearch.`, +If neither of them are, an error will be returned.`, RunE: func(cmd *cobra.Command, args []string) error { + if id == "" && name == "" { + return errors.New(`either "id" or "name" are required`) + } client, _, err := bootstrap(settings) if err != nil { - printErr(err, "is apm-server configured properly and Elasticsearch reachable?", json) return err } - return getAPIKey(client, &id, &name, validOnly, json) + getAPIKey(client, &id, &name, validOnly, json) + return nil }, - SilenceErrors: true, - SilenceUsage: true, } info.Flags().StringVar(&id, "id", "", "id of the API Key to query") info.Flags().StringVar(&name, "name", "", @@ -195,13 +188,6 @@ If no privilege(s) are specified, the credentials will be queried for all.` RunE: func(cmd *cobra.Command, args []string) error { _, config, err := bootstrap(settings) if err != nil { - printErr(err, "is apm-server configured properly and Elasticsearch reachable?", json) - return err - } - if credentials == "" { - err := errors.New("credentials argument is required") - // we can't use Cobra to mark the flag as required because it won't print the error as JSON - printErr(err, "", json) return err } privileges := booleansToPrivileges(ingest, sourcemap, agentConfig) @@ -209,10 +195,9 @@ If no privilege(s) are specified, the credentials will be queried for all.` // can't use "*" for querying privileges = auth.ActionsAll() } - return verifyAPIKey(config, privileges, credentials, json) + verifyAPIKey(config, privileges, credentials, json) + return nil }, - SilenceUsage: true, - SilenceErrors: true, } verify.Flags().StringVar(&credentials, "credentials", "", `credentials for which check privileges (required)`) verify.Flags().BoolVar(&ingest, "ingest", false, @@ -225,6 +210,7 @@ If no privilege(s) are specified, the credentials will be queried for all.` auth.PrivilegeAgentConfigRead)) verify.Flags().BoolVar(&json, "json", false, "prints the output of this command as JSON") + verify.MarkFlagRequired("credentials") verify.Flags().SortFlags = false return verify @@ -288,7 +274,7 @@ func booleansToPrivileges(ingest, sourcemap, agentConfig bool) []es.PrivilegeAct // creates an API Key with the given privileges, *AND* all the privileges modeled in apm-server // we need to ensure forward-compatibility, for which future privileges must be created here and // during server startup because we don't know if customers will run this command -func createAPIKeyWithPrivileges(client es.Client, apikeyName, expiry string, privileges []es.PrivilegeAction, asJSON bool) error { +func createAPIKeyWithPrivileges(client es.Client, apikeyName, expiry string, privileges []es.PrivilegeAction, asJSON bool) { var privilegesRequest = make(es.CreatePrivilegesRequest) event := auth.PrivilegeEventWrite agentConfig := auth.PrivilegeAgentConfigRead @@ -302,9 +288,10 @@ func createAPIKeyWithPrivileges(client es.Client, apikeyName, expiry string, pri privilegesCreated, err := es.CreatePrivileges(client, privilegesRequest) if err != nil { - return printErr(err, + printErr(err, `Error creating privileges for APM Server, do you have the "manage_cluster" security privilege?`, asJSON) + return } printText, printJSON := printers(asJSON) @@ -334,9 +321,10 @@ func createAPIKeyWithPrivileges(client es.Client, apikeyName, expiry string, pri apikey, err := es.CreateAPIKey(client, apikeyRequest) if err != nil { - return printErr(err, fmt.Sprintf( + printErr(err, fmt.Sprintf( `Error creating the API Key %s, do you have the "manage_cluster" security privilege?`, apikeyName), asJSON) + return } credentials := base64.StdEncoding.EncodeToString([]byte(apikey.Id + ":" + apikey.Key)) apikey.Credentials = &credentials @@ -349,7 +337,7 @@ func createAPIKeyWithPrivileges(client es.Client, apikeyName, expiry string, pri printText(`Credentials .... %s (use it as "Authorization: ApiKey " header to communicate with APM Server, won't be shown again)`, credentials) - return printJSON(struct { + printJSON(struct { es.CreateApiKeyResponse Privileges es.CreatePrivilegesResponse `json:"created_privileges,omitempty"` }{ @@ -358,13 +346,11 @@ func createAPIKeyWithPrivileges(client es.Client, apikeyName, expiry string, pri }) } -func getAPIKey(client es.Client, id, name *string, validOnly, asJSON bool) error { +func getAPIKey(client es.Client, id, name *string, validOnly, asJSON bool) { if isSet(id) { name = nil } else if isSet(name) { id = nil - } else { - return printErr(errors.New(`either "id" or "name" are required`), "", asJSON) } request := es.GetApiKeyRequest{ ApiKeyQuery: es.ApiKeyQuery{ @@ -375,9 +361,10 @@ func getAPIKey(client es.Client, id, name *string, validOnly, asJSON bool) error apikeys, err := es.GetAPIKeys(client, request) if err != nil { - return printErr(err, + printErr(err, `Error retrieving API Key(s) for APM Server, do you have the "manage_cluster" security privilege?`, asJSON) + return } transform := es.GetApiKeyResponse{ApiKeys: make([]es.ApiKeyResponse, 0)} @@ -400,16 +387,14 @@ func getAPIKey(client es.Client, id, name *string, validOnly, asJSON bool) error transform.ApiKeys = append(transform.ApiKeys, apikey) } printText("%d API Keys found", len(transform.ApiKeys)) - return printJSON(transform) + printJSON(transform) } -func invalidateAPIKey(client es.Client, id, name *string, deletePrivileges, asJSON bool) error { +func invalidateAPIKey(client es.Client, id, name *string, deletePrivileges, asJSON bool) { if isSet(id) { name = nil } else if isSet(name) { id = nil - } else { - return printErr(errors.New(`either "id" or "name" are required`), "", asJSON) } invalidateKeysRequest := es.InvalidateApiKeyRequest{ ApiKeyQuery: es.ApiKeyQuery{ @@ -420,9 +405,10 @@ func invalidateAPIKey(client es.Client, id, name *string, deletePrivileges, asJS invalidation, err := es.InvalidateAPIKey(client, invalidateKeysRequest) if err != nil { - return printErr(err, + printErr(err, `Error invalidating API Key(s), do you have the "manage_cluster" security privilege?`, asJSON) + return } printText, printJSON := printers(asJSON) out := struct { @@ -456,10 +442,10 @@ func invalidateAPIKey(client es.Client, id, name *string, deletePrivileges, asJS } out.Privileges = append(out.Privileges, deletion) } - return printJSON(out) + printJSON(out) } -func verifyAPIKey(config *config.Config, privileges []es.PrivilegeAction, credentials string, asJSON bool) error { +func verifyAPIKey(config *config.Config, privileges []es.PrivilegeAction, credentials string, asJSON bool) { perms := make(es.Permissions) printText, printJSON := printers(asJSON) @@ -486,9 +472,10 @@ func verifyAPIKey(config *config.Config, privileges []es.PrivilegeAction, creden } if err != nil { - return printErr(err, "could not verify credentials, please check your Elasticsearch connection", asJSON) + printErr(err, "could not verify credentials, please check your Elasticsearch connection", asJSON) + } else { + printJSON(perms) } - return printJSON(perms) } func humanBool(b bool) string { @@ -536,7 +523,7 @@ func humanTime(millis *int64) string { // returns 2 printers, one for text and one for JSON // one of them will be a noop based on the boolean argument -func printers(b bool) (func(string, ...interface{}), func(interface{}) error) { +func printers(b bool) (func(string, ...interface{}), func(interface{})) { var w1 io.Writer = os.Stdout var w2 = ioutil.Discard if b { @@ -546,16 +533,17 @@ func printers(b bool) (func(string, ...interface{}), func(interface{}) error) { return func(f string, i ...interface{}) { fmt.Fprintf(w1, f, i...) fmt.Fprintln(w1) - }, func(i interface{}) error { + }, func(i interface{}) { data, err := json.MarshalIndent(i, "", "\t") + if err != nil { + fmt.Fprintln(w2, err) + } fmt.Fprintln(w2, string(data)) - // conform the interface - return errors.Wrap(err, fmt.Sprintf("%v+", i)) } } // prints an Elasticsearch error to stderr, with some additional contextual information as a hint -func printErr(err error, help string, asJSON bool) error { +func printErr(err error, help string, asJSON bool) { if asJSON { var data []byte var m map[string]interface{} @@ -579,7 +567,6 @@ func printErr(err error, help string, asJSON bool) error { fmt.Fprintln(os.Stderr, help) fmt.Fprintln(os.Stderr, err.Error()) } - return errors.Wrap(err, help) } func isSet(s *string) bool { From 87ab92b2f0b9bd5683c76ab759f05a1f5e24cddf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20=C3=81lvarez?= Date: Thu, 9 Jan 2020 11:21:55 +0100 Subject: [PATCH 16/36] A few more code review comments --- beater/authorization/apikey.go | 2 +- cmd/apikey.go | 27 +++++++++++++-------------- elasticsearch/client.go | 15 +++++---------- 3 files changed, 19 insertions(+), 25 deletions(-) diff --git a/beater/authorization/apikey.go b/beater/authorization/apikey.go index bbcc3a40de7..f17013b35b9 100644 --- a/beater/authorization/apikey.go +++ b/beater/authorization/apikey.go @@ -65,7 +65,7 @@ func (a *apikeyAuth) IsAuthorizationConfigured() bool { // AuthorizedFor checks if the configured api key is authorized. // An api key is considered to be authorized when the api key has the configured privileges for the requested resource. -// PrivilegeGroup are fetched from Elasticsearch and then cached in a global cache. +// Permissions are fetched from Elasticsearch and then cached in a global cache. func (a *apikeyAuth) AuthorizedFor(resource es.Resource) (bool, error) { //fetch from cache if allowed, found := a.fromCache(resource); found { diff --git a/cmd/apikey.go b/cmd/apikey.go index b78beba5533..2fa5c3237c4 100644 --- a/cmd/apikey.go +++ b/cmd/apikey.go @@ -29,9 +29,6 @@ import ( "strings" "time" - logs "github.com/elastic/apm-server/log" - "github.com/elastic/beats/libbeat/logp" - "github.com/spf13/cobra" "github.com/elastic/beats/libbeat/cfgfile" @@ -216,12 +213,6 @@ If no privilege(s) are specified, the credentials will be queried for all.` return verify } -// TODO is there just any other way to do this? -// without the wrapper, YAML settings in "apm-server" are not picked up by ucfg -type ApmConfig struct { - Config *config.Config `config:"apm-server"` -} - // apm-server.api_key.enabled is implicitly true func bootstrap(settings instance.Settings) (es.Client, *config.Config, error) { @@ -243,18 +234,26 @@ func bootstrap(settings instance.Settings) (es.Client, *config.Config, error) { return nil, nil, err } - apm := ApmConfig{config.DefaultConfig(settings.Version)} - err = beat.RawConfig.Unpack(&apm) + cfg, err := beat.BeatConfig() + if err != nil { + return nil, nil, err + } + + var esOutputCfg *common.Config + if beat.Config.Output.Name() == "elasticsearch" { + esOutputCfg = beat.Config.Output.Config() + } + beaterConfig, err := config.NewConfig(beat.Info.Version, cfg, esOutputCfg) if err != nil { return nil, nil, err } var client es.Client - err = apm.Config.APIKeyConfig.Setup(logp.NewLogger(logs.Config), beat.Config.Output.Config()) if err == nil { - client, err = es.NewClient(apm.Config.APIKeyConfig.ESConfig) + client, err = es.NewClient(beaterConfig.APIKeyConfig.ESConfig) } - return client, apm.Config, err + + return client, beaterConfig, err } func booleansToPrivileges(ingest, sourcemap, agentConfig bool) []es.PrivilegeAction { diff --git a/elasticsearch/client.go b/elasticsearch/client.go index e7df0cde280..47925102502 100644 --- a/elasticsearch/client.go +++ b/elasticsearch/client.go @@ -25,7 +25,6 @@ import ( "io" "io/ioutil" "net/http" - "net/url" "strings" "github.com/elastic/beats/libbeat/common" @@ -169,20 +168,15 @@ func makeRequest(method, path string, body interface{}, headers ...string) (*htt header[kv[0]] = strings.Split(kv[1], ",") } } - u, _ := url.Parse(path) - req := &http.Request{ - Method: method, - URL: u, - Header: header, - } bs, err := json.Marshal(body) if err != nil { return nil, err } - if body != nil { - req.Body = ioutil.NopCloser(bytes.NewReader(bs)) - req.ContentLength = int64(len(bs)) + req, err := http.NewRequest(method, path, ioutil.NopCloser(bytes.NewReader(bs))) + if err != nil { + return nil, err } + req.Header = header return req, nil } @@ -194,6 +188,7 @@ func parseResponse(resp *http.Response, err error) JSONResponse { if resp.StatusCode >= http.StatusMultipleChoices { buf := new(bytes.Buffer) buf.ReadFrom(body) + body.Close() return JSONResponse{nil, errors.New(buf.String())} } return JSONResponse{body, nil} From caefc89f0e9fb958b850446da10cb91d4f25a0bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20=C3=81lvarez?= Date: Thu, 9 Jan 2020 12:00:25 +0100 Subject: [PATCH 17/36] Add a few tests --- elasticsearch/client.go | 6 ++-- elasticsearch/client_test.go | 60 ++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 3 deletions(-) diff --git a/elasticsearch/client.go b/elasticsearch/client.go index 47925102502..7aaf2df25d9 100644 --- a/elasticsearch/client.go +++ b/elasticsearch/client.go @@ -65,7 +65,7 @@ func (c clientV8) Search(index string, body io.Reader) (int, io.ReadCloser, erro } func (c clientV8) JSONRequest(method, path string, body interface{}, headers ...string) JSONResponse { - req, err := makeRequest(method, path, body, headers...) + req, err := makeJSONRequest(method, path, body, headers...) if err != nil { return JSONResponse{nil, err} } @@ -92,7 +92,7 @@ func (c clientV7) Search(index string, body io.Reader) (int, io.ReadCloser, erro } func (c clientV7) JSONRequest(method, path string, body interface{}, headers ...string) JSONResponse { - req, err := makeRequest(method, path, body, headers...) + req, err := makeJSONRequest(method, path, body, headers...) if err != nil { return JSONResponse{nil, err} } @@ -157,7 +157,7 @@ func (r JSONResponse) DecodeTo(i interface{}) error { } // each header has the format "key: value" -func makeRequest(method, path string, body interface{}, headers ...string) (*http.Request, error) { +func makeJSONRequest(method, path string, body interface{}, headers ...string) (*http.Request, error) { header := http.Header{ "Content-Type": []string{"application/json"}, "Accept": []string{"application/json"}, diff --git a/elasticsearch/client_test.go b/elasticsearch/client_test.go index fa2917412e2..19f493d1621 100644 --- a/elasticsearch/client_test.go +++ b/elasticsearch/client_test.go @@ -18,6 +18,11 @@ package elasticsearch import ( + "bytes" + "errors" + "io" + "io/ioutil" + "net/http" "strings" "testing" @@ -57,3 +62,58 @@ func TestClient(t *testing.T) { }) } + +func TestMakeJSONRequest(t *testing.T) { + var body interface{} + req, err := makeJSONRequest(http.MethodGet, "/path", body, "Authorization:foo", "Header-X:bar") + assert.Nil(t, err) + assert.Equal(t, http.MethodGet, req.Method) + assert.Equal(t, "/path", req.URL.Path) + assert.NotNil(t, req.Body) + assert.NotNil(t, req.Header) + assert.Equal(t, "application/json", req.Header.Get("Content-Type")) + assert.Equal(t, "application/json", req.Header.Get("Accept")) + assert.Equal(t, "foo", req.Header.Get("Authorization")) + assert.Equal(t, "bar", req.Header.Get("Header-X")) +} + +func TestParseResponse(t *testing.T) { + body := "body" + for _, testCase := range []struct { + code int + expectedBody io.ReadCloser + expectedErr error + }{ + {404, nil, errors.New(body)}, + {200, ioutil.NopCloser(strings.NewReader(body)), nil}, + } { + jsonResponse := parseResponse(&http.Response{ + StatusCode: testCase.code, + Body: ioutil.NopCloser(strings.NewReader(body)), + }, nil) + assert.Equal(t, testCase.expectedBody, jsonResponse.content) + assert.Equal(t, testCase.expectedErr, jsonResponse.err) + } +} + +func TestDecodeTo(t *testing.T) { + type target map[string]string + err := errors.New("error") + + for _, testCase := range []struct { + content []byte + err error + expectedError error + expectedEffect target + }{ + {nil, err, err, target(nil)}, + {[]byte(`{"foo":"bar"}`), nil, nil, target{"foo": "bar"}}, + } { + var to target + assert.Equal(t, testCase.expectedError, JSONResponse{ + content: ioutil.NopCloser(bytes.NewReader(testCase.content)), + err: testCase.err, + }.DecodeTo(&to)) + assert.Equal(t, testCase.expectedEffect, to) + } +} From b2bfde0057d6cd2e3757c18a5b53e4464444ae59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20=C3=81lvarez?= Date: Thu, 9 Jan 2020 12:11:31 +0100 Subject: [PATCH 18/36] No help --- cmd/apikey.go | 26 +++++++------------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/cmd/apikey.go b/cmd/apikey.go index 2fa5c3237c4..65ce2139edd 100644 --- a/cmd/apikey.go +++ b/cmd/apikey.go @@ -287,9 +287,7 @@ func createAPIKeyWithPrivileges(client es.Client, apikeyName, expiry string, pri privilegesCreated, err := es.CreatePrivileges(client, privilegesRequest) if err != nil { - printErr(err, - `Error creating privileges for APM Server, do you have the "manage_cluster" security privilege?`, - asJSON) + printErr(err, asJSON) return } @@ -320,9 +318,7 @@ func createAPIKeyWithPrivileges(client es.Client, apikeyName, expiry string, pri apikey, err := es.CreateAPIKey(client, apikeyRequest) if err != nil { - printErr(err, fmt.Sprintf( - `Error creating the API Key %s, do you have the "manage_cluster" security privilege?`, apikeyName), - asJSON) + printErr(err, asJSON) return } credentials := base64.StdEncoding.EncodeToString([]byte(apikey.Id + ":" + apikey.Key)) @@ -360,9 +356,7 @@ func getAPIKey(client es.Client, id, name *string, validOnly, asJSON bool) { apikeys, err := es.GetAPIKeys(client, request) if err != nil { - printErr(err, - `Error retrieving API Key(s) for APM Server, do you have the "manage_cluster" security privilege?`, - asJSON) + printErr(err, asJSON) return } @@ -404,9 +398,7 @@ func invalidateAPIKey(client es.Client, id, name *string, deletePrivileges, asJS invalidation, err := es.InvalidateAPIKey(client, invalidateKeysRequest) if err != nil { - printErr(err, - `Error invalidating API Key(s), do you have the "manage_cluster" security privilege?`, - asJSON) + printErr(err, asJSON) return } printText, printJSON := printers(asJSON) @@ -471,7 +463,7 @@ func verifyAPIKey(config *config.Config, privileges []es.PrivilegeAction, creden } if err != nil { - printErr(err, "could not verify credentials, please check your Elasticsearch connection", asJSON) + printErr(err, asJSON) } else { printJSON(perms) } @@ -541,29 +533,25 @@ func printers(b bool) (func(string, ...interface{}), func(interface{})) { } } -// prints an Elasticsearch error to stderr, with some additional contextual information as a hint -func printErr(err error, help string, asJSON bool) { +// prints an Elasticsearch error to stderr +func printErr(err error, asJSON bool) { if asJSON { var data []byte var m map[string]interface{} e := json.Unmarshal([]byte(err.Error()), &m) if e == nil { // err.Error() has JSON shape, likely coming from Elasticsearch - m["help"] = help data, _ = json.MarshalIndent(m, "", "\t") } else { // err.Error() is a bare string, likely coming from apm-server data, _ = json.MarshalIndent(struct { Error string `json:"error"` - Help string `json:"help,omitempty"` }{ Error: err.Error(), - Help: help, }, "", "\t") } fmt.Fprintln(os.Stderr, string(data)) } else { - fmt.Fprintln(os.Stderr, help) fmt.Fprintln(os.Stderr, err.Error()) } } From e73a09bed2f15b4d92481b16b6f5d79641a047bf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20=C3=81lvarez?= Date: Thu, 9 Jan 2020 12:18:12 +0100 Subject: [PATCH 19/36] More renames --- beater/authorization/apikey.go | 4 ++-- elasticsearch/security_api.go | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/beater/authorization/apikey.go b/beater/authorization/apikey.go index f17013b35b9..55b20a49d3e 100644 --- a/beater/authorization/apikey.go +++ b/beater/authorization/apikey.go @@ -123,8 +123,8 @@ func (a *apikeyAuth) queryES(resource es.Resource) (es.Permissions, error) { return nil, err } if resources, ok := info.Application[Application]; ok { - if privileges, ok := resources[resource]; ok { - return privileges, nil + if permissions, ok := resources[resource]; ok { + return permissions, nil } } return es.Permissions{}, nil diff --git a/elasticsearch/security_api.go b/elasticsearch/security_api.go index cccbda42a2f..05460781ae0 100644 --- a/elasticsearch/security_api.go +++ b/elasticsearch/security_api.go @@ -118,9 +118,9 @@ type HasPrivilegesRequest struct { Applications []Application `json:"application"` } type HasPrivilegesResponse struct { - Username string `json:"username"` - HasAll bool `json:"has_all_requested"` - Application map[AppName]PrivilegesPerResource `json:"application"` + Username string `json:"username"` + HasAll bool `json:"has_all_requested"` + Application map[AppName]PermissionsPerResource `json:"application"` } type InvalidateApiKeyRequest struct { @@ -178,7 +178,7 @@ type PrivilegeGroup map[PrivilegeName]Actions type Permissions map[PrivilegeAction]bool -type PrivilegesPerResource map[Resource]Permissions +type PermissionsPerResource map[Resource]Permissions type Actions struct { Actions []PrivilegeAction `json:"actions"` From d1123e02522c41e44c8256b12bf14cfd807a4d86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20=C3=81lvarez?= Date: Thu, 9 Jan 2020 16:21:44 +0100 Subject: [PATCH 20/36] query privileges before creating them, and fix issue intruducen in last commit --- cmd/apikey.go | 36 +++++++++++++- elasticsearch/client.go | 13 +++-- elasticsearch/client_test.go | 2 +- elasticsearch/security_api.go | 5 +- testing/docker/elasticsearch/roles.yml | 2 +- tests/system/test_apikey.py | 67 ++++++++++++++++---------- 6 files changed, 90 insertions(+), 35 deletions(-) diff --git a/cmd/apikey.go b/cmd/apikey.go index 65ce2139edd..3e20fda54d0 100644 --- a/cmd/apikey.go +++ b/cmd/apikey.go @@ -273,7 +273,7 @@ func booleansToPrivileges(ingest, sourcemap, agentConfig bool) []es.PrivilegeAct // creates an API Key with the given privileges, *AND* all the privileges modeled in apm-server // we need to ensure forward-compatibility, for which future privileges must be created here and // during server startup because we don't know if customers will run this command -func createAPIKeyWithPrivileges(client es.Client, apikeyName, expiry string, privileges []es.PrivilegeAction, asJSON bool) { +func createAPIKeyWithPrivileges(client es.Client, keyName, expiry string, privileges []es.PrivilegeAction, asJSON bool) { var privilegesRequest = make(es.CreatePrivilegesRequest) event := auth.PrivilegeEventWrite agentConfig := auth.PrivilegeAgentConfigRead @@ -291,6 +291,38 @@ func createAPIKeyWithPrivileges(client es.Client, apikeyName, expiry string, pri return } + // Elasticsearch will allow a user without the right apm privileges to create API keys, but the keys won't validate + // check first whether the user has the right privileges, and bail out early if not + // is not possible to always do it automatically, because file-based users and roles are not queryable + hasPrivileges, err := es.HasPrivileges(client, es.HasPrivilegesRequest{ + Applications: []es.Application{ + { + Name: auth.Application, + Privileges: auth.ActionsAll(), + Resources: []es.Resource{auth.ResourceInternal}, + }, + }, + }, "") + if err != nil { + printErr(err, asJSON) + return + } + if !hasPrivileges.HasAll { + printErr(errors.New(fmt.Sprintf(`%s does not have privileges to create API keys. +You might try with the superuser, or add the APM application privileges to the role of the authenticated user, eg.: +PUT /_security/role/my_role { + ... + "applications": [{ + "application": "apm", + "privileges": ["sourcemap:write", "event:write", "config_agent:read"], + "resources": ["*"] + }], + ... +} + `, hasPrivileges.Username)), asJSON) + return + } + printText, printJSON := printers(asJSON) for privilege, result := range privilegesCreated[auth.Application] { if result.Created { @@ -299,7 +331,7 @@ func createAPIKeyWithPrivileges(client es.Client, apikeyName, expiry string, pri } apikeyRequest := es.CreateApiKeyRequest{ - Name: apikeyName, + Name: keyName, RoleDescriptors: es.RoleDescriptor{ auth.Application: es.Applications{ Applications: []es.Application{ diff --git a/elasticsearch/client.go b/elasticsearch/client.go index 7aaf2df25d9..235883c797d 100644 --- a/elasticsearch/client.go +++ b/elasticsearch/client.go @@ -23,7 +23,6 @@ import ( "encoding/json" "errors" "io" - "io/ioutil" "net/http" "strings" @@ -168,11 +167,15 @@ func makeJSONRequest(method, path string, body interface{}, headers ...string) ( header[kv[0]] = strings.Split(kv[1], ",") } } - bs, err := json.Marshal(body) - if err != nil { - return nil, err + var reader io.Reader + if body != nil { + bs, err := json.Marshal(body) + if err != nil { + return nil, err + } + reader = bytes.NewReader(bs) } - req, err := http.NewRequest(method, path, ioutil.NopCloser(bytes.NewReader(bs))) + req, err := http.NewRequest(method, path, reader) if err != nil { return nil, err } diff --git a/elasticsearch/client_test.go b/elasticsearch/client_test.go index 19f493d1621..44f901e8a05 100644 --- a/elasticsearch/client_test.go +++ b/elasticsearch/client_test.go @@ -69,7 +69,7 @@ func TestMakeJSONRequest(t *testing.T) { assert.Nil(t, err) assert.Equal(t, http.MethodGet, req.Method) assert.Equal(t, "/path", req.URL.Path) - assert.NotNil(t, req.Body) + assert.Nil(t, req.Body) assert.NotNil(t, req.Header) assert.Equal(t, "application/json", req.Header.Get("Content-Type")) assert.Equal(t, "application/json", req.Header.Get("Accept")) diff --git a/elasticsearch/security_api.go b/elasticsearch/security_api.go index 05460781ae0..12ca364df49 100644 --- a/elasticsearch/security_api.go +++ b/elasticsearch/security_api.go @@ -81,7 +81,10 @@ func DeletePrivileges(client Client, privilegesReq DeletePrivilegeRequest) (Dele } func HasPrivileges(client Client, privileges HasPrivilegesRequest, credentials string) (HasPrivilegesResponse, error) { - h := fmt.Sprintf("Authorization: ApiKey %s", credentials) + var h string + if credentials != "" { + h = fmt.Sprintf("Authorization: ApiKey %s", credentials) + } response := client.JSONRequest(http.MethodGet, "/_security/user/_has_privileges", privileges, h) var info HasPrivilegesResponse diff --git a/testing/docker/elasticsearch/roles.yml b/testing/docker/elasticsearch/roles.yml index b67c0a0aec2..c48bb4a13b6 100644 --- a/testing/docker/elasticsearch/roles.yml +++ b/testing/docker/elasticsearch/roles.yml @@ -4,7 +4,7 @@ apm_server: - names: ['apm-*'] privileges: ['write','create_index','manage','manage_ilm'] beats: - cluster: ['manage_index_templates','monitor','manage_ingest_pipelines','manage_ilm'] + cluster: ['manage_index_templates','monitor','manage_ingest_pipelines','manage_ilm', 'manage_security','manage_api_key'] indices: - names: ['filebeat-*','shrink-filebeat-*'] privileges: ['all'] diff --git a/tests/system/test_apikey.py b/tests/system/test_apikey.py index ab27d478b8f..a09b23cacad 100644 --- a/tests/system/test_apikey.py +++ b/tests/system/test_apikey.py @@ -1,6 +1,5 @@ from apmserver import BaseTest, integration_test from elasticsearch import Elasticsearch -import inspect import json import random @@ -28,8 +27,7 @@ def subcommand_output(self, *args): return json.loads(command_output) def subcommand(self, *args): - caller = inspect.getouterframes(inspect.currentframe())[1][3] - logfile = self.beat_name + "-" + caller + str(random.randint(0, 9999)) + "-" + args[0] + ".log" + logfile = self.beat_name + "-" + str(random.randint(0, 99999)) + "-" + args[0] + ".log" subcmd = ["apikey"] subcmd.extend(args) subcmd.append("--json") @@ -59,66 +57,66 @@ class APIKeyTest(APIKeyBaseTest): def tearDown(self): super(APIKeyBaseTest, self).tearDown() invalidated = self.subcommand_output("invalidate", "--name", self.api_key_name) - assert invalidated["error_count"] == 0 + assert invalidated.get("error_count") == 0 def test_create(self): apikey = self.create() - assert apikey["name"] == self.api_key_name, apikey + assert apikey.get("name") == self.api_key_name, apikey for privilege in ["sourcemap", "agentConfig", "event"]: apikey["created_privileges"]["apm"][privilege]["created"] = True, apikey for attr in ["id", "api_key", "credentials"]: - assert apikey[attr] != "", apikey + assert apikey.get(attr) != "", apikey def test_create_with_settings_override(self): apikey = self.create( "-E", "output.elasticsearch.enabled=false", "-E", "apm-server.api_key.elasticsearch.hosts=[{}]".format(self.get_elasticsearch_url()) ) - assert apikey["credentials"] is not None, apikey + assert apikey.get("credentials") is not None, apikey def test_create_with_expiration(self): apikey = self.create("--expiration", "1d") - assert apikey["expiration"] is not None, apikey + assert apikey.get("expiration") is not None, apikey def test_invalidate_by_id(self): apikey = self.create() invalidated = self.subcommand_output("invalidate", "--id", apikey["id"]) - assert invalidated["invalidated_api_keys"] == [apikey["id"]], invalidated - assert invalidated["error_count"] == 0, invalidated + assert invalidated.get("invalidated_api_keys") == [apikey["id"]], invalidated + assert invalidated.get("error_count") == 0, invalidated def test_invalidate_by_name(self): self.create() self.create() invalidated = self.subcommand_output("invalidate", "--name", self.api_key_name) - assert len(invalidated["invalidated_api_keys"]) == 2, invalidated - assert invalidated["error_count"] == 0, invalidated + assert len(invalidated.get("invalidated_api_keys")) == 2, invalidated + assert invalidated.get("error_count") == 0, invalidated def test_info_by_id(self): self.create() apikey = self.create() info = self.subcommand_output("info", "--id", apikey["id"]) - assert len(info["api_keys"]) == 1, info - assert info["api_keys"][0]["username"] == "admin", info - assert info["api_keys"][0]["id"] == apikey["id"], info - assert info["api_keys"][0]["name"] == apikey["name"], info - assert info["api_keys"][0]["invalidated"] is False, info + assert len(info.get("api_keys")) == 1, info + assert info["api_keys"][0].get("username") == "admin", info + assert info["api_keys"][0].get("id") == apikey["id"], info + assert info["api_keys"][0].get("name") == apikey["name"], info + assert info["api_keys"][0].get("invalidated") is False, info def test_info_by_name(self): apikey = self.create() invalidated = self.subcommand_output("invalidate", "--id", apikey["id"]) - assert invalidated["error_count"] == 0 + assert invalidated.get("error_count") == 0 self.create() self.create() info = self.subcommand_output("info", "--name", self.api_key_name) # can't test exact number because these tests have side effects - assert len(info["api_keys"]) > 2, info + assert len(info.get("api_keys")) > 2, info info = self.subcommand_output("info", "--name", self.api_key_name, "--valid-only") - assert len(info["api_keys"]) == 2, info + assert len(info.get("api_keys")) == 2, info def test_verify_all(self): apikey = self.create() @@ -149,13 +147,32 @@ class APIKeyBadUserTest(APIKeyBaseTest): def config(self): return { - "elasticsearch_host": self.get_elasticsearch_url(user="apm_server_user", password="changeme"), + "elasticsearch_host": self.get_elasticsearch_url(user="heartbeat_user", password="changeme"), "file_enabled": "false", "kibana_enabled": "false", } def test_create_bad_user(self): - out = self.subcommand("create", "--name", self.api_key_name) - result = json.loads(out) - assert result["status"] == 401, result - assert result["error"] is not None + """heartbeat_user doesn't have required cluster privileges, so it can't create keys""" + result = self.subcommand_output("create", "--name", self.api_key_name) + assert result.get("status") == 403, result + assert result.get("error") is not None + + +@integration_test +class APIKeyBadUser2Test(APIKeyBaseTest): + + def config(self): + return { + "elasticsearch_host": self.get_elasticsearch_url(user="beats_user", password="changeme"), + "file_enabled": "false", + "kibana_enabled": "false", + } + + def test_create_bad_user(self): + """beats_user does have required cluster privileges, but not APM application privileges, + so it can't create keys + """ + result = self.subcommand_output("create", "--name", self.api_key_name) + assert result.get("error") is not None, result + assert "beats_user does not have privileges to create API keys" in result.get("error"), result From 939b9aec4f8d7831ad37449d6fca9e6494fdcaa0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20=C3=81lvarez?= Date: Fri, 10 Jan 2020 09:23:55 +0100 Subject: [PATCH 21/36] please the linter --- beater/authorization/apikey.go | 4 +- beater/authorization/builder_test.go | 2 +- beater/headers/keys.go | 2 +- cmd/apikey.go | 42 ++++++++++---------- elasticsearch/security_api.go | 58 ++++++++++++++-------------- tests/system/apmserver.py | 4 +- tests/system/test_jaeger.py | 8 ++-- 7 files changed, 61 insertions(+), 59 deletions(-) diff --git a/beater/authorization/apikey.go b/beater/authorization/apikey.go index 55b20a49d3e..2d45dc1faee 100644 --- a/beater/authorization/apikey.go +++ b/beater/authorization/apikey.go @@ -29,10 +29,10 @@ import ( const cleanupInterval = 60 * time.Second var ( - // Constant mapped to the "application" field for the Elasticsearch security API + // Application is a constant mapped to the "application" field for the Elasticsearch security API // This identifies privileges and keys created for APM Application = es.AppName("apm") - // Only valid for first authorization of a request. + // ResourceInternal is only valid for first authorization of a request. // The API Key needs to grant privileges to additional resources for successful processing of requests. ResourceInternal = es.Resource("-") ResourceAny = es.Resource("*") diff --git a/beater/authorization/builder_test.go b/beater/authorization/builder_test.go index 06ab007d5f2..60f20408ad7 100644 --- a/beater/authorization/builder_test.go +++ b/beater/authorization/builder_test.go @@ -82,7 +82,7 @@ func TestBuilder(t *testing.T) { t.Run("AuthorizationFor"+name, func(t *testing.T) { builder := setup() h := builder.ForPrivilege(PrivilegeSourcemapWrite.Action) - auth := h.AuthorizationFor("ApiKey", "") + auth := h.AuthorizationFor("APIKey", "") if tc.withApikey { assert.IsType(t, &apikeyAuth{}, auth) } else { diff --git a/beater/headers/keys.go b/beater/headers/keys.go index 82f944ea09b..895474e4856 100644 --- a/beater/headers/keys.go +++ b/beater/headers/keys.go @@ -26,7 +26,7 @@ const ( AccessControlExposeHeaders = "Access-Control-Expose-Headers" AccessControlMaxAge = "Access-Control-Max-Age" Authorization = "Authorization" - APIKey = "ApiKey" + APIKey = "APIKey" Bearer = "Bearer" CacheControl = "Cache-Control" Connection = "Connection" diff --git a/cmd/apikey.go b/cmd/apikey.go index 3e20fda54d0..cbd5675f30f 100644 --- a/cmd/apikey.go +++ b/cmd/apikey.go @@ -308,7 +308,7 @@ func createAPIKeyWithPrivileges(client es.Client, keyName, expiry string, privil return } if !hasPrivileges.HasAll { - printErr(errors.New(fmt.Sprintf(`%s does not have privileges to create API keys. + printErr(fmt.Errorf(`%s does not have privileges to create API keys. You might try with the superuser, or add the APM application privileges to the role of the authenticated user, eg.: PUT /_security/role/my_role { ... @@ -319,7 +319,7 @@ PUT /_security/role/my_role { }], ... } - `, hasPrivileges.Username)), asJSON) + `, hasPrivileges.Username), asJSON) return } @@ -330,7 +330,7 @@ PUT /_security/role/my_role { } } - apikeyRequest := es.CreateApiKeyRequest{ + apikeyRequest := es.CreateAPIKeyRequest{ Name: keyName, RoleDescriptors: es.RoleDescriptor{ auth.Application: es.Applications{ @@ -353,22 +353,22 @@ PUT /_security/role/my_role { printErr(err, asJSON) return } - credentials := base64.StdEncoding.EncodeToString([]byte(apikey.Id + ":" + apikey.Key)) + credentials := base64.StdEncoding.EncodeToString([]byte(apikey.ID + ":" + apikey.Key)) apikey.Credentials = &credentials printText("API Key created:") printText("") printText("Name ........... %s", apikey.Name) printText("Expiration ..... %s", humanTime(apikey.ExpirationMs)) - printText("Id ............. %s", apikey.Id) + printText("Id ............. %s", apikey.ID) printText("API Key ........ %s (won't be shown again)", apikey.Key) - printText(`Credentials .... %s (use it as "Authorization: ApiKey " header to communicate with APM Server, won't be shown again)`, + printText(`Credentials .... %s (use it as "Authorization: APIKey " header to communicate with APM Server, won't be shown again)`, credentials) printJSON(struct { - es.CreateApiKeyResponse + es.CreateAPIKeyResponse Privileges es.CreatePrivilegesResponse `json:"created_privileges,omitempty"` }{ - CreateApiKeyResponse: apikey, + CreateAPIKeyResponse: apikey, Privileges: privilegesCreated, }) } @@ -379,9 +379,9 @@ func getAPIKey(client es.Client, id, name *string, validOnly, asJSON bool) { } else if isSet(name) { id = nil } - request := es.GetApiKeyRequest{ - ApiKeyQuery: es.ApiKeyQuery{ - Id: id, + request := es.GetAPIKeyRequest{ + APIKeyQuery: es.APIKeyQuery{ + ID: id, Name: name, }, } @@ -392,9 +392,9 @@ func getAPIKey(client es.Client, id, name *string, validOnly, asJSON bool) { return } - transform := es.GetApiKeyResponse{ApiKeys: make([]es.ApiKeyResponse, 0)} + transform := es.GetAPIKeyResponse{APIKeys: make([]es.APIKeyResponse, 0)} printText, printJSON := printers(asJSON) - for _, apikey := range apikeys.ApiKeys { + for _, apikey := range apikeys.APIKeys { expiry := humanTime(apikey.ExpirationMs) if validOnly && (apikey.Invalidated || expiry == "expired") { continue @@ -402,16 +402,16 @@ func getAPIKey(client es.Client, id, name *string, validOnly, asJSON bool) { creation := time.Unix(apikey.Creation/1000, 0).Format("2006-02-01 15:04") printText("Username ....... %s", apikey.Username) printText("Api Key Name ... %s", apikey.Name) - printText("Id ............. %s", apikey.Id) + printText("Id ............. %s", apikey.ID) printText("Creation ....... %s", creation) printText("Invalidated .... %t", apikey.Invalidated) if !apikey.Invalidated { printText("Expiration ..... %s", expiry) } printText("") - transform.ApiKeys = append(transform.ApiKeys, apikey) + transform.APIKeys = append(transform.APIKeys, apikey) } - printText("%d API Keys found", len(transform.ApiKeys)) + printText("%d API Keys found", len(transform.APIKeys)) printJSON(transform) } @@ -421,9 +421,9 @@ func invalidateAPIKey(client es.Client, id, name *string, deletePrivileges, asJS } else if isSet(name) { id = nil } - invalidateKeysRequest := es.InvalidateApiKeyRequest{ - ApiKeyQuery: es.ApiKeyQuery{ - Id: id, + invalidateKeysRequest := es.InvalidateAPIKeyRequest{ + APIKeyQuery: es.APIKeyQuery{ + ID: id, Name: name, }, } @@ -435,10 +435,10 @@ func invalidateAPIKey(client es.Client, id, name *string, deletePrivileges, asJS } printText, printJSON := printers(asJSON) out := struct { - es.InvalidateApiKeyResponse + es.InvalidateAPIKeyResponse Privileges []es.DeletePrivilegeResponse `json:"deleted_privileges,omitempty"` }{ - InvalidateApiKeyResponse: invalidation, + InvalidateAPIKeyResponse: invalidation, Privileges: make([]es.DeletePrivilegeResponse, 0), } printText("Invalidated keys ... %s", strings.Join(invalidation.Invalidated, ", ")) diff --git a/elasticsearch/security_api.go b/elasticsearch/security_api.go index 12ca364df49..0c3ad1d1df9 100644 --- a/elasticsearch/security_api.go +++ b/elasticsearch/security_api.go @@ -25,21 +25,21 @@ import ( ) // CreateAPIKey requires manage_security cluster privilege -func CreateAPIKey(client Client, apikeyReq CreateApiKeyRequest) (CreateApiKeyResponse, error) { +func CreateAPIKey(client Client, apikeyReq CreateAPIKeyRequest) (CreateAPIKeyResponse, error) { response := client.JSONRequest(http.MethodPut, "/_security/api_key", apikeyReq) - var apikey CreateApiKeyResponse + var apikey CreateAPIKeyResponse err := response.DecodeTo(&apikey) return apikey, err } // GetAPIKeys requires manage_security cluster privilege -func GetAPIKeys(client Client, apikeyReq GetApiKeyRequest) (GetApiKeyResponse, error) { +func GetAPIKeys(client Client, apikeyReq GetAPIKeyRequest) (GetAPIKeyResponse, error) { u := url.URL{Path: "/_security/api_key"} params := url.Values{} params.Set("owner", strconv.FormatBool(apikeyReq.Owner)) - if apikeyReq.Id != nil { - params.Set("id", *apikeyReq.Id) + if apikeyReq.ID != nil { + params.Set("id", *apikeyReq.ID) } else if apikeyReq.Name != nil { params.Set("name", *apikeyReq.Name) } @@ -47,7 +47,7 @@ func GetAPIKeys(client Client, apikeyReq GetApiKeyRequest) (GetApiKeyResponse, e response := client.JSONRequest(http.MethodGet, u.String(), nil) - var apikey GetApiKeyResponse + var apikey GetAPIKeyResponse err := response.DecodeTo(&apikey) return apikey, err } @@ -62,10 +62,10 @@ func CreatePrivileges(client Client, privilegesReq CreatePrivilegesRequest) (Cre } // InvalidateAPIKey requires manage_security cluster privilege -func InvalidateAPIKey(client Client, apikeyReq InvalidateApiKeyRequest) (InvalidateApiKeyResponse, error) { +func InvalidateAPIKey(client Client, apikeyReq InvalidateAPIKeyRequest) (InvalidateAPIKeyResponse, error) { response := client.JSONRequest(http.MethodDelete, "/_security/api_key", apikeyReq) - var confirmation InvalidateApiKeyResponse + var confirmation InvalidateAPIKeyResponse err := response.DecodeTo(&confirmation) return confirmation, err } @@ -83,7 +83,7 @@ func DeletePrivileges(client Client, privilegesReq DeletePrivilegeRequest) (Dele func HasPrivileges(client Client, privileges HasPrivilegesRequest, credentials string) (HasPrivilegesResponse, error) { var h string if credentials != "" { - h = fmt.Sprintf("Authorization: ApiKey %s", credentials) + h = fmt.Sprintf("Authorization: APIKey %s", credentials) } response := client.JSONRequest(http.MethodGet, "/_security/user/_has_privileges", privileges, h) @@ -92,24 +92,24 @@ func HasPrivileges(client Client, privileges HasPrivilegesRequest, credentials s return info, err } -type CreateApiKeyRequest struct { +type CreateAPIKeyRequest struct { Name string `json:"name"` Expiration *string `json:"expiration,omitempty"` RoleDescriptors RoleDescriptor `json:"role_descriptors"` } -type CreateApiKeyResponse struct { - ApiKey +type CreateAPIKeyResponse struct { + APIKey Key string `json:"api_key"` } -type GetApiKeyRequest struct { - ApiKeyQuery +type GetAPIKeyRequest struct { + APIKeyQuery Owner bool `json:"owner"` } -type GetApiKeyResponse struct { - ApiKeys []ApiKeyResponse `json:"api_keys"` +type GetAPIKeyResponse struct { + APIKeys []APIKeyResponse `json:"api_keys"` } type CreatePrivilegesRequest map[AppName]PrivilegeGroup @@ -126,11 +126,11 @@ type HasPrivilegesResponse struct { Application map[AppName]PermissionsPerResource `json:"application"` } -type InvalidateApiKeyRequest struct { - ApiKeyQuery +type InvalidateAPIKeyRequest struct { + APIKeyQuery } -type InvalidateApiKeyResponse struct { +type InvalidateAPIKeyResponse struct { Invalidated []string `json:"invalidated_api_keys"` ErrorCount int `json:"error_count"` } @@ -154,21 +154,21 @@ type Application struct { Resources []Resource `json:"resources"` } -type ApiKeyResponse struct { - ApiKey +type APIKeyResponse struct { + APIKey Creation int64 `json:"creation"` Invalidated bool `json:"invalidated"` Username string `json:"username"` } -type ApiKeyQuery struct { +type APIKeyQuery struct { // normally the Elasticsearch API will require either Id or Name, but not both - Id *string `json:"id,omitempty"` + ID *string `json:"id,omitempty"` Name *string `json:"name,omitempty"` } -type ApiKey struct { - Id string `json:"id"` +type APIKey struct { + ID string `json:"id"` Name string `json:"name"` ExpirationMs *int64 `json:"expiration,omitempty"` // This attribute does not come from Elasticsearch, but is filled in by APM Server @@ -199,10 +199,10 @@ type AppName string type Resource string -// in Elasticsearch a "privilege" represents both an "action" that a user might/might not have authorization to -// perform; and a tuple consisting of a name and an action -// for differentiation, we call the tuple NamedPrivilege -// in apm-server, each name is associated with one action, but that needs not to be the case (see PrivilegeGroup) +// NamedPrivilege is a tuple consisting of a name and an action. +// In Elasticsearch a "privilege" represents both an "action" that a user might/might not have authorization to +// perform, and such a tuple. +// In apm-server, each name is associated with one action, but that needs not to be the case (see PrivilegeGroup) type NamedPrivilege struct { Name PrivilegeName Action PrivilegeAction diff --git a/tests/system/apmserver.py b/tests/system/apmserver.py index 605cfc228c2..6532f8b256d 100644 --- a/tests/system/apmserver.py +++ b/tests/system/apmserver.py @@ -346,7 +346,8 @@ def wait_for_events(self, processor_name, expected_count, index=None, max_timeou self.es.indices.refresh(index=index) query = {"term": {"processor.name": processor_name}} - result = {} # TODO(axw) use "nonlocal" when we migrate to Python 3 + result = {} # TODO(axw) use "nonlocal" when we migrate to Python 3 + def get_docs(): hits = self.es.search(index=index, body={"query": query})['hits'] result['docs'] = hits['hits'] @@ -465,6 +466,7 @@ def __str__(self): ApprovalException.__name__ = type(exc).__name__ raise ApprovalException, exc, sys.exc_info()[2] + class ClientSideBaseTest(ServerBaseTest): sourcemap_url = 'http://localhost:8200/assets/v1/sourcemaps' intake_url = 'http://localhost:8200/intake/v2/rum/events' diff --git a/tests/system/test_jaeger.py b/tests/system/test_jaeger.py index 978eb21ebf3..c25d412dc07 100644 --- a/tests/system/test_jaeger.py +++ b/tests/system/test_jaeger.py @@ -51,10 +51,10 @@ def test_jaeger_grpc(self): client = os.path.join(os.path.dirname(__file__), 'jaegergrpc') subprocess.check_call(['go', 'run', client, - '-addr', self.jaeger_grpc_addr, - '-insecure', - jaeger_request_data, - ]) + '-addr', self.jaeger_grpc_addr, + '-insecure', + jaeger_request_data, + ]) self.assert_no_logged_warnings() transaction_docs = self.wait_for_events('transaction', 1) From fa827edc46a0bf578545939a3bd93abc9eee26a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20=C3=81lvarez?= Date: Fri, 10 Jan 2020 11:36:25 +0100 Subject: [PATCH 22/36] fix unit test --- beater/config/api_key.go | 2 +- beater/config/api_key_test.go | 2 +- beater/config/config.go | 2 +- beater/server_test.go | 4 ++-- cmd/root.go | 8 ++++---- idxmgmt/ilm/config.go | 2 +- idxmgmt/manager_test.go | 8 ++++---- vendor/github.com/elastic/go-ucfg/error.go | 3 +++ 8 files changed, 17 insertions(+), 14 deletions(-) diff --git a/beater/config/api_key.go b/beater/config/api_key.go index 2a523ad6245..3e4ed840589 100644 --- a/beater/config/api_key.go +++ b/beater/config/api_key.go @@ -41,7 +41,7 @@ func (c *APIKeyConfig) IsEnabled() bool { return c != nil && c.Enabled } -func (c *APIKeyConfig) Setup(log *logp.Logger, outputESCfg *common.Config) error { +func (c *APIKeyConfig) setup(log *logp.Logger, outputESCfg *common.Config) error { if c == nil || !c.Enabled || c.ESConfig != nil { return nil } diff --git a/beater/config/api_key_test.go b/beater/config/api_key_test.go index ad995e55a2b..112904732fa 100644 --- a/beater/config/api_key_test.go +++ b/beater/config/api_key_test.go @@ -84,7 +84,7 @@ func TestAPIKeyConfig_ESConfig(t *testing.T) { }, } { t.Run(name, func(t *testing.T) { - err := tc.cfg.Setup(logp.NewLogger("api_key"), tc.esCfg) + err := tc.cfg.setup(logp.NewLogger("api_key"), tc.esCfg) if tc.expectedErr == nil { assert.NoError(t, err) } else { diff --git a/beater/config/config.go b/beater/config/config.go index d2f099a1665..2b1de44e45c 100644 --- a/beater/config/config.go +++ b/beater/config/config.go @@ -117,7 +117,7 @@ func NewConfig(version string, ucfg *common.Config, outputESCfg *common.Config) return nil, err } - if err := c.APIKeyConfig.Setup(logger, outputESCfg); err != nil { + if err := c.APIKeyConfig.setup(logger, outputESCfg); err != nil { return nil, err } diff --git a/beater/server_test.go b/beater/server_test.go index 9cd617c1e70..55f7a895ccb 100644 --- a/beater/server_test.go +++ b/beater/server_test.go @@ -241,11 +241,11 @@ func TestServerSourcemapBadConfig(t *testing.T) { ucfg, err := common.NewConfigFrom(m{"rum": m{"enabled": true, "source_mapping": m{"elasticsearch": m{"hosts": []string{}}}}}) require.NoError(t, err) s, teardown, err := setupServer(t, ucfg, nil, nil) - require.Nil(t, s) + require.NotNil(t, s) if err == nil { defer teardown() } - require.Error(t, err) + require.NoError(t, err) } func TestServerCORS(t *testing.T) { diff --git a/cmd/root.go b/cmd/root.go index 71d343a494a..a2794e3d939 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -111,7 +111,7 @@ func init() { } // only add defined flags to setup command setup := RootCmd.SetupCmd - setup.Short = "Setup Elasticsearch index management components and pipelines" + setup.Short = "setup Elasticsearch index management components and pipelines" setup.Long = `This command does initial setup of the environment: * Index management including loading Elasticsearch templates, ILM policies and write aliases. @@ -120,9 +120,9 @@ func init() { setup.ResetFlags() //lint:ignore SA1019 Setting up template must still be supported until next major version upgrade. tmplKey := cmd.TemplateKey - setup.Flags().Bool(tmplKey, false, "Setup index template") + setup.Flags().Bool(tmplKey, false, "setup index template") setup.Flags().MarkDeprecated(tmplKey, fmt.Sprintf("please use --%s instead", cmd.IndexManagementKey)) - setup.Flags().Bool(cmd.IndexManagementKey, false, "Setup Elasticsearch index management") - setup.Flags().Bool(cmd.PipelineKey, false, "Setup ingest pipelines") + setup.Flags().Bool(cmd.IndexManagementKey, false, "setup Elasticsearch index management") + setup.Flags().Bool(cmd.PipelineKey, false, "setup ingest pipelines") } diff --git a/idxmgmt/ilm/config.go b/idxmgmt/ilm/config.go index d73141eed08..1b0a3efdf6e 100644 --- a/idxmgmt/ilm/config.go +++ b/idxmgmt/ilm/config.go @@ -35,7 +35,7 @@ type Config struct { Setup Setup `config:"setup"` } -//Setup holds information about how to setup ILM +//setup holds information about how to setup ILM type Setup struct { Enabled bool `config:"enabled"` Overwrite bool `config:"overwrite"` diff --git a/idxmgmt/manager_test.go b/idxmgmt/manager_test.go index 9165a28e8fd..88e4660e297 100644 --- a/idxmgmt/manager_test.go +++ b/idxmgmt/manager_test.go @@ -324,12 +324,12 @@ func TestManager_SetupILM(t *testing.T) { version: "6.2.0", templatesILMDisabled: 4, }, - "Default ES Unsupported ILM Setup disabled": { + "Default ES Unsupported ILM setup disabled": { cfg: common.MapStr{"apm-server.ilm.setup.enabled": false}, loadMode: libidxmgmt.LoadModeEnabled, version: "6.2.0", }, - "ILM True ES Unsupported ILM Setup disabled": { + "ILM True ES Unsupported ILM setup disabled": { cfg: common.MapStr{"apm-server.ilm.setup.enabled": false, "apm-server.ilm.enabled": true}, loadMode: libidxmgmt.LoadModeEnabled, version: "6.2.0", @@ -359,7 +359,7 @@ func TestManager_SetupILM(t *testing.T) { loadMode: libidxmgmt.LoadModeEnabled, templatesILMDisabled: 4, }, - "ESIndexConfigured Setup disabled": { + "ESIndexConfigured setup disabled": { cfg: common.MapStr{ "apm-server.ilm.enabled": "auto", "apm-server.ilm.setup.enabled": false, @@ -368,7 +368,7 @@ func TestManager_SetupILM(t *testing.T) { "output.elasticsearch.index": "custom"}, loadMode: libidxmgmt.LoadModeEnabled, }, - "ESIndicesConfigured Setup disabled": { + "ESIndicesConfigured setup disabled": { cfg: common.MapStr{ "apm-server.ilm.enabled": "auto", "apm-server.ilm.setup.enabled": false, diff --git a/vendor/github.com/elastic/go-ucfg/error.go b/vendor/github.com/elastic/go-ucfg/error.go index a48690e30a8..9bd994e4526 100644 --- a/vendor/github.com/elastic/go-ucfg/error.go +++ b/vendor/github.com/elastic/go-ucfg/error.go @@ -320,6 +320,9 @@ func raiseInvalidDuration(v value, err error) Error { func raiseValidation(ctx context, meta *Meta, field string, err error) Error { path := "" + if path == "rum.source_mapping.elasticsearch.hosts" { + fmt.Sprintf("FUUUCK") + } if field == "" { path = ctx.path(".") } else { From 6f28f3cfee1689aa465751db6b306c5d77e8b133 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20=C3=81lvarez?= Date: Fri, 10 Jan 2020 11:47:22 +0100 Subject: [PATCH 23/36] Reset libbeat api key --- beater/authorization/builder.go | 1 + idxmgmt/ilm/config.go | 2 +- tests/system/test_instrumentation.py | 2 ++ 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/beater/authorization/builder.go b/beater/authorization/builder.go index ee6ae41f322..f673291c154 100644 --- a/beater/authorization/builder.go +++ b/beater/authorization/builder.go @@ -55,6 +55,7 @@ func NewBuilder(cfg config.Config) (*Builder, error) { // do not use username+password for API Key requests cfg.APIKeyConfig.ESConfig.Username = "" cfg.APIKeyConfig.ESConfig.Password = "" + cfg.APIKeyConfig.ESConfig.APIKey = "" client, err := elasticsearch.NewClient(cfg.APIKeyConfig.ESConfig) if err != nil { return nil, err diff --git a/idxmgmt/ilm/config.go b/idxmgmt/ilm/config.go index 1b0a3efdf6e..d73141eed08 100644 --- a/idxmgmt/ilm/config.go +++ b/idxmgmt/ilm/config.go @@ -35,7 +35,7 @@ type Config struct { Setup Setup `config:"setup"` } -//setup holds information about how to setup ILM +//Setup holds information about how to setup ILM type Setup struct { Enabled bool `config:"enabled"` Overwrite bool `config:"overwrite"` diff --git a/tests/system/test_instrumentation.py b/tests/system/test_instrumentation.py index 86c4b9c4d03..44d507e8ea8 100644 --- a/tests/system/test_instrumentation.py +++ b/tests/system/test_instrumentation.py @@ -42,6 +42,7 @@ def test_api_key_auth(self): query = {"term": {"processor.name": "transaction"}} index = self.index_transaction + def get_transactions(): self.es.indices.refresh(index=index) return self.es.count(index=index, body={"query": query})['count'] > 0 @@ -75,6 +76,7 @@ def test_secret_token_auth(self): query = {"term": {"processor.name": "transaction"}} index = self.index_transaction + def get_transactions(): self.es.indices.refresh(index=index) return self.es.count(index=index, body={"query": query})['count'] > 0 From 0eff8ddd17d30bdfdc2ca44f1d1a574e366e8a69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20=C3=81lvarez?= Date: Fri, 10 Jan 2020 12:03:41 +0100 Subject: [PATCH 24/36] make update --- apm-server.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apm-server.yml b/apm-server.yml index 759025bddf5..cd9ebe676df 100644 --- a/apm-server.yml +++ b/apm-server.yml @@ -157,6 +157,8 @@ apm-server: #protocol: "http" + # Username and password are only needed for the apm-server apikey sub-command, and they are ignored otherwise + # See `apm-server apikey --help` for details. #username: "elastic" #password: "changeme" From 790a4fe26ce3c9e7b03f8828d43ac78441cdbdb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20=C3=81lvarez?= Date: Fri, 10 Jan 2020 13:56:00 +0100 Subject: [PATCH 25/36] undo golint refactor and update user in test assertion --- elasticsearch/security_api.go | 2 +- tests/system/test_apikey.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/elasticsearch/security_api.go b/elasticsearch/security_api.go index 0c3ad1d1df9..ae4d34c6a45 100644 --- a/elasticsearch/security_api.go +++ b/elasticsearch/security_api.go @@ -83,7 +83,7 @@ func DeletePrivileges(client Client, privilegesReq DeletePrivilegeRequest) (Dele func HasPrivileges(client Client, privileges HasPrivilegesRequest, credentials string) (HasPrivilegesResponse, error) { var h string if credentials != "" { - h = fmt.Sprintf("Authorization: APIKey %s", credentials) + h = fmt.Sprintf("Authorization: ApiKey %s", credentials) } response := client.JSONRequest(http.MethodGet, "/_security/user/_has_privileges", privileges, h) diff --git a/tests/system/test_apikey.py b/tests/system/test_apikey.py index a09b23cacad..0bf6b67af97 100644 --- a/tests/system/test_apikey.py +++ b/tests/system/test_apikey.py @@ -99,7 +99,7 @@ def test_info_by_id(self): apikey = self.create() info = self.subcommand_output("info", "--id", apikey["id"]) assert len(info.get("api_keys")) == 1, info - assert info["api_keys"][0].get("username") == "admin", info + assert info["api_keys"][0].get("username") == "apm_server_user", info assert info["api_keys"][0].get("id") == apikey["id"], info assert info["api_keys"][0].get("name") == apikey["name"], info assert info["api_keys"][0].get("invalidated") is False, info From f135da3f5177da66b4e63d3c614779026720337f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20=C3=81lvarez?= Date: Fri, 10 Jan 2020 14:49:46 +0100 Subject: [PATCH 26/36] fix another golint refactor screwup --- beater/headers/keys.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beater/headers/keys.go b/beater/headers/keys.go index 895474e4856..82f944ea09b 100644 --- a/beater/headers/keys.go +++ b/beater/headers/keys.go @@ -26,7 +26,7 @@ const ( AccessControlExposeHeaders = "Access-Control-Expose-Headers" AccessControlMaxAge = "Access-Control-Max-Age" Authorization = "Authorization" - APIKey = "APIKey" + APIKey = "ApiKey" Bearer = "Bearer" CacheControl = "Cache-Control" Connection = "Connection" From baa566fdefeddf9e2d61a8abdcaa1af6d31d0af7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20=C3=81lvarez?= Date: Fri, 10 Jan 2020 15:18:24 +0100 Subject: [PATCH 27/36] another one --- beater/authorization/builder_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beater/authorization/builder_test.go b/beater/authorization/builder_test.go index 60f20408ad7..06ab007d5f2 100644 --- a/beater/authorization/builder_test.go +++ b/beater/authorization/builder_test.go @@ -82,7 +82,7 @@ func TestBuilder(t *testing.T) { t.Run("AuthorizationFor"+name, func(t *testing.T) { builder := setup() h := builder.ForPrivilege(PrivilegeSourcemapWrite.Action) - auth := h.AuthorizationFor("APIKey", "") + auth := h.AuthorizationFor("ApiKey", "") if tc.withApikey { assert.IsType(t, &apikeyAuth{}, auth) } else { From 256b434432725523d775eaa20c20e0bf7a3dafa4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20=C3=81lvarez?= Date: Mon, 13 Jan 2020 10:16:58 +0100 Subject: [PATCH 28/36] logic for checking any privileges action (*) --- beater/api/root/handler.go | 4 +++- beater/authorization/apikey.go | 32 +++++++++++++------------------- beater/authorization/builder.go | 2 +- 3 files changed, 17 insertions(+), 21 deletions(-) diff --git a/beater/api/root/handler.go b/beater/api/root/handler.go index f088210128c..a187afbaed2 100644 --- a/beater/api/root/handler.go +++ b/beater/api/root/handler.go @@ -20,6 +20,8 @@ package root import ( "time" + "github.com/elastic/apm-server/beater/authorization" + "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/monitoring" "github.com/elastic/beats/libbeat/version" @@ -51,7 +53,7 @@ func Handler() request.Handler { } c.Result.SetDefault(request.IDResponseValidOK) - authorized, err := c.Authorization.AuthorizedFor("") + authorized, err := c.Authorization.AuthorizedFor(authorization.ResourceInternal) if err != nil { c.Result.Err = err } diff --git a/beater/authorization/apikey.go b/beater/authorization/apikey.go index 2d45dc1faee..3c337af5ee3 100644 --- a/beater/authorization/apikey.go +++ b/beater/authorization/apikey.go @@ -67,9 +67,9 @@ func (a *apikeyAuth) IsAuthorizationConfigured() bool { // An api key is considered to be authorized when the api key has the configured privileges for the requested resource. // Permissions are fetched from Elasticsearch and then cached in a global cache. func (a *apikeyAuth) AuthorizedFor(resource es.Resource) (bool, error) { - //fetch from cache - if allowed, found := a.fromCache(resource); found { - return allowed, nil + privileges := a.cache.get(id(a.key, resource)) + if privileges != nil { + return a.allowed(privileges), nil } if a.cache.isFull() { @@ -78,32 +78,26 @@ func (a *apikeyAuth) AuthorizedFor(resource es.Resource) (bool, error) { "or consider increasing config option `apm-server.api_key.limit`") } - //fetch from ES privileges, err := a.queryES(resource) if err != nil { return false, err } - //add to cache a.cache.add(id(a.key, resource), privileges) - - allowed, _ := a.fromCache(resource) - return allowed, nil + return a.allowed(privileges), nil } -func (a *apikeyAuth) fromCache(resource es.Resource) (allowed bool, found bool) { - privileges := a.cache.get(id(a.key, resource)) - if privileges == nil { - return - } - found = true - allowed = false +func (a *apikeyAuth) allowed(permissions es.Permissions) bool { + var allowed bool for _, privilege := range a.anyOfPrivileges { - if privilegeAllowed, ok := privileges[privilege]; ok && privilegeAllowed { - allowed = true - return + if privilege == ActionAny { + for _, value := range permissions { + allowed = allowed || value + } } + value, _ := permissions[privilege] + allowed = allowed || value } - return + return allowed } func (a *apikeyAuth) queryES(resource es.Resource) (es.Permissions, error) { diff --git a/beater/authorization/builder.go b/beater/authorization/builder.go index f673291c154..188705e01a0 100644 --- a/beater/authorization/builder.go +++ b/beater/authorization/builder.go @@ -37,7 +37,7 @@ type Handler Builder // Authorization interface to be implemented by different auth types type Authorization interface { - AuthorizedFor(_ elasticsearch.Resource) (bool, error) + AuthorizedFor(elasticsearch.Resource) (bool, error) IsAuthorizationConfigured() bool } From 022ae2ea9fcf3d15320f1d2eb55deb203611a1d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20=C3=81lvarez?= Date: Mon, 13 Jan 2020 10:18:24 +0100 Subject: [PATCH 29/36] Add changelog --- changelogs/head.asciidoc | 1 + 1 file changed, 1 insertion(+) diff --git a/changelogs/head.asciidoc b/changelogs/head.asciidoc index 8b6c2166684..04a1b9281aa 100644 --- a/changelogs/head.asciidoc +++ b/changelogs/head.asciidoc @@ -26,4 +26,5 @@ https://github.com/elastic/apm-server/compare/7.5\...master[View commits] - Upgrade Go to 1.13.5 {pull}3069[3069]. - Add experimental support for receiving Jaeger trace data {pull}3129[3129] - Upgrade APM Go agent to 1.7.0, and add support for API Key auth for self-instrumentation {pull}3134[3134] +- Add subcommand to create API Keys {pull}3063[3063] From 2d6001d7fb7160e0b06410e1085a4b374193f34a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20=C3=81lvarez?= Date: Mon, 13 Jan 2020 10:57:34 +0100 Subject: [PATCH 30/36] more golint && fmt --- beater/authorization/apikey.go | 3 +-- beater/middleware/log_middleware.go | 3 ++- beater/middleware/log_middleware_test.go | 3 ++- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/beater/authorization/apikey.go b/beater/authorization/apikey.go index 3c337af5ee3..59c6d9c4f70 100644 --- a/beater/authorization/apikey.go +++ b/beater/authorization/apikey.go @@ -94,8 +94,7 @@ func (a *apikeyAuth) allowed(permissions es.Permissions) bool { allowed = allowed || value } } - value, _ := permissions[privilege] - allowed = allowed || value + allowed = allowed || permissions[privilege] } return allowed } diff --git a/beater/middleware/log_middleware.go b/beater/middleware/log_middleware.go index 1e01d833f6a..e2a18474096 100644 --- a/beater/middleware/log_middleware.go +++ b/beater/middleware/log_middleware.go @@ -20,9 +20,10 @@ package middleware import ( "github.com/gofrs/uuid" - "github.com/elastic/beats/libbeat/logp" "go.elastic.co/apm" + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/apm-server/beater/headers" "github.com/elastic/apm-server/beater/request" logs "github.com/elastic/apm-server/log" diff --git a/beater/middleware/log_middleware_test.go b/beater/middleware/log_middleware_test.go index 9f30dabf03c..2b132f7d204 100644 --- a/beater/middleware/log_middleware_test.go +++ b/beater/middleware/log_middleware_test.go @@ -26,10 +26,11 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/zap/zapcore" - "github.com/elastic/beats/libbeat/logp" "go.elastic.co/apm" "go.elastic.co/apm/apmtest" + "github.com/elastic/beats/libbeat/logp" + "github.com/elastic/apm-server/beater/beatertest" "github.com/elastic/apm-server/beater/headers" "github.com/elastic/apm-server/beater/request" From 82285cab33defa42247882e1b7c66159de2538de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20=C3=81lvarez?= Date: Mon, 13 Jan 2020 13:20:31 +0100 Subject: [PATCH 31/36] a bit more code review --- elasticsearch/client.go | 123 +++++++++------------------------- elasticsearch/client_test.go | 60 ----------------- elasticsearch/security_api.go | 56 +++++++--------- sourcemap/es_store.go | 2 +- 4 files changed, 58 insertions(+), 183 deletions(-) diff --git a/elasticsearch/client.go b/elasticsearch/client.go index 235883c797d..aadadfdfdf9 100644 --- a/elasticsearch/client.go +++ b/elasticsearch/client.go @@ -18,16 +18,16 @@ package elasticsearch import ( - "bytes" "context" "encoding/json" "errors" "io" + "io/ioutil" "net/http" - "strings" "github.com/elastic/beats/libbeat/common" "github.com/elastic/beats/libbeat/version" + "github.com/elastic/go-elasticsearch/v7/esapi" esv7 "github.com/elastic/go-elasticsearch/v7" @@ -36,26 +36,23 @@ import ( // Client is an interface designed to abstract away version differences between elasticsearch clients type Client interface { + // Perform satisfies esapi.Transport + Perform(*http.Request) (*http.Response, error) // TODO: deprecate - // Search performs a query against the given index with the given body - Search(index string, body io.Reader) (int, io.ReadCloser, error) - // Makes a request with application/json Content-Type and Accept headers by default - // pass/overwrite headers with "key: value" format - JSONRequest(method, path string, body interface{}, headers ...string) JSONResponse + SearchQuery(index string, body io.Reader) (int, io.ReadCloser, error) } type clientV8 struct { - v8 *esv8.Client + *esv8.Client } -// Search satisfies the Client interface for version 8 -func (c clientV8) Search(index string, body io.Reader) (int, io.ReadCloser, error) { - response, err := c.v8.Search( - c.v8.Search.WithContext(context.Background()), - c.v8.Search.WithIndex(index), - c.v8.Search.WithBody(body), - c.v8.Search.WithTrackTotalHits(true), - c.v8.Search.WithPretty(), +func (c clientV8) SearchQuery(index string, body io.Reader) (int, io.ReadCloser, error) { + response, err := c.Search( + c.Search.WithContext(context.Background()), + c.Search.WithIndex(index), + c.Search.WithBody(body), + c.Search.WithTrackTotalHits(true), + c.Search.WithPretty(), ) if err != nil { return 0, nil, err @@ -63,26 +60,17 @@ func (c clientV8) Search(index string, body io.Reader) (int, io.ReadCloser, erro return response.StatusCode, response.Body, nil } -func (c clientV8) JSONRequest(method, path string, body interface{}, headers ...string) JSONResponse { - req, err := makeJSONRequest(method, path, body, headers...) - if err != nil { - return JSONResponse{nil, err} - } - return parseResponse(c.v8.Perform(req)) -} - type clientV7 struct { - v7 *esv7.Client + *esv7.Client } -// Search satisfies the Client interface for version 7 -func (c clientV7) Search(index string, body io.Reader) (int, io.ReadCloser, error) { - response, err := c.v7.Search( - c.v7.Search.WithContext(context.Background()), - c.v7.Search.WithIndex(index), - c.v7.Search.WithBody(body), - c.v7.Search.WithTrackTotalHits(true), - c.v7.Search.WithPretty(), +func (c clientV7) SearchQuery(index string, body io.Reader) (int, io.ReadCloser, error) { + response, err := c.Search( + c.Search.WithContext(context.Background()), + c.Search.WithIndex(index), + c.Search.WithBody(body), + c.Search.WithTrackTotalHits(true), + c.Search.WithPretty(), ) if err != nil { return 0, nil, err @@ -90,14 +78,6 @@ func (c clientV7) Search(index string, body io.Reader) (int, io.ReadCloser, erro return response.StatusCode, response.Body, nil } -func (c clientV7) JSONRequest(method, path string, body interface{}, headers ...string) JSONResponse { - req, err := makeJSONRequest(method, path, body, headers...) - if err != nil { - return JSONResponse{nil, err} - } - return parseResponse(c.v7.Perform(req)) -} - // NewClient parses the given config and returns a version-aware client as an interface func NewClient(config *Config) (Client, error) { if config == nil { @@ -141,58 +121,21 @@ func newV8Client(apikey, user, pwd string, addresses []string, transport http.Ro }) } -type JSONResponse struct { - content io.ReadCloser - err error -} - -func (r JSONResponse) DecodeTo(i interface{}) error { - if r.err != nil { - return r.err - } - defer r.content.Close() - err := json.NewDecoder(r.content).Decode(&i) - return err -} - -// each header has the format "key: value" -func makeJSONRequest(method, path string, body interface{}, headers ...string) (*http.Request, error) { - header := http.Header{ - "Content-Type": []string{"application/json"}, - "Accept": []string{"application/json"}, - } - for _, h := range headers { - kv := strings.Split(h, ":") - if len(kv) == 2 { - header[kv[0]] = strings.Split(kv[1], ",") - } +func doRequest(transport esapi.Transport, req esapi.Request, out interface{}) error { + resp, err := req.Do(context.TODO(), transport) + if err != nil { + return err } - var reader io.Reader - if body != nil { - bs, err := json.Marshal(body) + defer resp.Body.Close() + if resp.IsError() { + bytes, err := ioutil.ReadAll(resp.Body) if err != nil { - return nil, err + return err } - reader = bytes.NewReader(bs) - } - req, err := http.NewRequest(method, path, reader) - if err != nil { - return nil, err - } - req.Header = header - return req, nil -} - -func parseResponse(resp *http.Response, err error) JSONResponse { - if err != nil { - return JSONResponse{nil, err} + return errors.New(string(bytes)) } - body := resp.Body - if resp.StatusCode >= http.StatusMultipleChoices { - buf := new(bytes.Buffer) - buf.ReadFrom(body) - body.Close() - return JSONResponse{nil, errors.New(buf.String())} + if out != nil { + err = json.NewDecoder(resp.Body).Decode(out) } - return JSONResponse{body, nil} + return err } diff --git a/elasticsearch/client_test.go b/elasticsearch/client_test.go index 44f901e8a05..fa2917412e2 100644 --- a/elasticsearch/client_test.go +++ b/elasticsearch/client_test.go @@ -18,11 +18,6 @@ package elasticsearch import ( - "bytes" - "errors" - "io" - "io/ioutil" - "net/http" "strings" "testing" @@ -62,58 +57,3 @@ func TestClient(t *testing.T) { }) } - -func TestMakeJSONRequest(t *testing.T) { - var body interface{} - req, err := makeJSONRequest(http.MethodGet, "/path", body, "Authorization:foo", "Header-X:bar") - assert.Nil(t, err) - assert.Equal(t, http.MethodGet, req.Method) - assert.Equal(t, "/path", req.URL.Path) - assert.Nil(t, req.Body) - assert.NotNil(t, req.Header) - assert.Equal(t, "application/json", req.Header.Get("Content-Type")) - assert.Equal(t, "application/json", req.Header.Get("Accept")) - assert.Equal(t, "foo", req.Header.Get("Authorization")) - assert.Equal(t, "bar", req.Header.Get("Header-X")) -} - -func TestParseResponse(t *testing.T) { - body := "body" - for _, testCase := range []struct { - code int - expectedBody io.ReadCloser - expectedErr error - }{ - {404, nil, errors.New(body)}, - {200, ioutil.NopCloser(strings.NewReader(body)), nil}, - } { - jsonResponse := parseResponse(&http.Response{ - StatusCode: testCase.code, - Body: ioutil.NopCloser(strings.NewReader(body)), - }, nil) - assert.Equal(t, testCase.expectedBody, jsonResponse.content) - assert.Equal(t, testCase.expectedErr, jsonResponse.err) - } -} - -func TestDecodeTo(t *testing.T) { - type target map[string]string - err := errors.New("error") - - for _, testCase := range []struct { - content []byte - err error - expectedError error - expectedEffect target - }{ - {nil, err, err, target(nil)}, - {[]byte(`{"foo":"bar"}`), nil, nil, target{"foo": "bar"}}, - } { - var to target - assert.Equal(t, testCase.expectedError, JSONResponse{ - content: ioutil.NopCloser(bytes.NewReader(testCase.content)), - err: testCase.err, - }.DecodeTo(&to)) - assert.Equal(t, testCase.expectedEffect, to) - } -} diff --git a/elasticsearch/security_api.go b/elasticsearch/security_api.go index ae4d34c6a45..8ef344c7711 100644 --- a/elasticsearch/security_api.go +++ b/elasticsearch/security_api.go @@ -18,77 +18,69 @@ package elasticsearch import ( - "fmt" "net/http" - "net/url" - "strconv" + + "github.com/elastic/go-elasticsearch/v7/esapi" + "github.com/elastic/go-elasticsearch/v7/esutil" ) // CreateAPIKey requires manage_security cluster privilege func CreateAPIKey(client Client, apikeyReq CreateAPIKeyRequest) (CreateAPIKeyResponse, error) { - response := client.JSONRequest(http.MethodPut, "/_security/api_key", apikeyReq) - var apikey CreateAPIKeyResponse - err := response.DecodeTo(&apikey) + req := esapi.SecurityCreateAPIKeyRequest{Body: esutil.NewJSONReader(apikeyReq)} + err := doRequest(client, req, &apikey) return apikey, err } // GetAPIKeys requires manage_security cluster privilege func GetAPIKeys(client Client, apikeyReq GetAPIKeyRequest) (GetAPIKeyResponse, error) { - u := url.URL{Path: "/_security/api_key"} - params := url.Values{} - params.Set("owner", strconv.FormatBool(apikeyReq.Owner)) + req := esapi.SecurityGetAPIKeyRequest{} if apikeyReq.ID != nil { - params.Set("id", *apikeyReq.ID) + req.ID = *apikeyReq.ID } else if apikeyReq.Name != nil { - params.Set("name", *apikeyReq.Name) + req.Name = *apikeyReq.Name } - u.RawQuery = params.Encode() - - response := client.JSONRequest(http.MethodGet, u.String(), nil) - var apikey GetAPIKeyResponse - err := response.DecodeTo(&apikey) + err := doRequest(client, req, &apikey) return apikey, err } // CreatePrivileges requires manage_security cluster privilege func CreatePrivileges(client Client, privilegesReq CreatePrivilegesRequest) (CreatePrivilegesResponse, error) { - response := client.JSONRequest(http.MethodPut, "/_security/privilege", privilegesReq) - var privileges CreatePrivilegesResponse - err := response.DecodeTo(&privileges) + req := esapi.SecurityPutPrivilegesRequest{Body: esutil.NewJSONReader(privilegesReq)} + err := doRequest(client, req, &privileges) return privileges, err } // InvalidateAPIKey requires manage_security cluster privilege func InvalidateAPIKey(client Client, apikeyReq InvalidateAPIKeyRequest) (InvalidateAPIKeyResponse, error) { - response := client.JSONRequest(http.MethodDelete, "/_security/api_key", apikeyReq) - var confirmation InvalidateAPIKeyResponse - err := response.DecodeTo(&confirmation) + req := esapi.SecurityInvalidateAPIKeyRequest{Body: esutil.NewJSONReader(apikeyReq)} + err := doRequest(client, req, &confirmation) return confirmation, err } // DeletePrivileges requires manage_security cluster privilege func DeletePrivileges(client Client, privilegesReq DeletePrivilegeRequest) (DeletePrivilegeResponse, error) { - path := fmt.Sprintf("/_security/privilege/%v/%v", privilegesReq.Application, privilegesReq.Privilege) - response := client.JSONRequest(http.MethodDelete, path, nil) - var confirmation DeletePrivilegeResponse - err := response.DecodeTo(&confirmation) + req := esapi.SecurityDeletePrivilegesRequest{ + Application: string(privilegesReq.Application), + Name: string(privilegesReq.Privilege), + } + err := doRequest(client, req, &confirmation) return confirmation, err } func HasPrivileges(client Client, privileges HasPrivilegesRequest, credentials string) (HasPrivilegesResponse, error) { - var h string + var info HasPrivilegesResponse + req := esapi.SecurityHasPrivilegesRequest{Body: esutil.NewJSONReader(privileges)} if credentials != "" { - h = fmt.Sprintf("Authorization: ApiKey %s", credentials) + header := make(http.Header) + header.Set("Authorization", "ApiKey "+credentials) + req.Header = header } - response := client.JSONRequest(http.MethodGet, "/_security/user/_has_privileges", privileges, h) - - var info HasPrivilegesResponse - err := response.DecodeTo(&info) + err := doRequest(client, req, &info) return info, err } diff --git a/sourcemap/es_store.go b/sourcemap/es_store.go index eca8f5cf41c..ef44dc14220 100644 --- a/sourcemap/es_store.go +++ b/sourcemap/es_store.go @@ -93,7 +93,7 @@ func (s *esStore) runSearchQuery(name, version, path string) (int, io.ReadCloser return 0, nil, err } // Perform the runSearchQuery request. - return s.client.Search(s.index, &buf) + return s.client.SearchQuery(s.index, &buf) } func parse(body io.ReadCloser, name, version, path string, logger *logp.Logger) (string, error) { From 5c852da7150ee26bf6502fdb8fa250abf14e3178 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20=C3=81lvarez?= Date: Tue, 14 Jan 2020 09:04:27 +0100 Subject: [PATCH 32/36] Update beater/authorization/apikey.go Co-Authored-By: Andrew Wilkins --- beater/authorization/apikey.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beater/authorization/apikey.go b/beater/authorization/apikey.go index 59c6d9c4f70..7cb1a2c4c29 100644 --- a/beater/authorization/apikey.go +++ b/beater/authorization/apikey.go @@ -28,7 +28,7 @@ import ( const cleanupInterval = 60 * time.Second -var ( +const ( // Application is a constant mapped to the "application" field for the Elasticsearch security API // This identifies privileges and keys created for APM Application = es.AppName("apm") From 3da13486583bf08ee46494c71676e8f0c1bb8f1c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20=C3=81lvarez?= Date: Tue, 14 Jan 2020 09:06:03 +0100 Subject: [PATCH 33/36] Update cmd/apikey.go Co-Authored-By: Andrew Wilkins --- cmd/apikey.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/apikey.go b/cmd/apikey.go index cbd5675f30f..48ea34c86ba 100644 --- a/cmd/apikey.go +++ b/cmd/apikey.go @@ -326,7 +326,7 @@ PUT /_security/role/my_role { printText, printJSON := printers(asJSON) for privilege, result := range privilegesCreated[auth.Application] { if result.Created { - printText("Security privilege \"%v\" created", privilege) + printText("Security privilege %q created", privilege) } } From a708f599d5fb6b2d26049c53ea32dc9b6dcdbef4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20=C3=81lvarez?= Date: Tue, 14 Jan 2020 09:06:19 +0100 Subject: [PATCH 34/36] Update beater/authorization/apikey.go Co-Authored-By: Andrew Wilkins --- beater/authorization/apikey.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beater/authorization/apikey.go b/beater/authorization/apikey.go index 7cb1a2c4c29..becd9b2dc23 100644 --- a/beater/authorization/apikey.go +++ b/beater/authorization/apikey.go @@ -31,7 +31,7 @@ const cleanupInterval = 60 * time.Second const ( // Application is a constant mapped to the "application" field for the Elasticsearch security API // This identifies privileges and keys created for APM - Application = es.AppName("apm") + Application es.AppName = "apm" // ResourceInternal is only valid for first authorization of a request. // The API Key needs to grant privileges to additional resources for successful processing of requests. ResourceInternal = es.Resource("-") From cef8ef45d4cbb0af57d42f586c970be0768312a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20=C3=81lvarez?= Date: Tue, 14 Jan 2020 09:06:51 +0100 Subject: [PATCH 35/36] Update beater/authorization/apikey.go Co-Authored-By: Andrew Wilkins --- beater/authorization/apikey.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beater/authorization/apikey.go b/beater/authorization/apikey.go index becd9b2dc23..a777bed8d29 100644 --- a/beater/authorization/apikey.go +++ b/beater/authorization/apikey.go @@ -124,5 +124,5 @@ func (a *apikeyAuth) queryES(resource es.Resource) (es.Permissions, error) { } func id(apiKey string, resource es.Resource) string { - return apiKey + "_" + fmt.Sprintf("%v", resource) + return apiKey + "_" + string(resource) } From 983714f9926e8a9b23b109e30f3a66d7ac96d8b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Juan=20=C3=81lvarez?= Date: Tue, 14 Jan 2020 09:15:09 +0100 Subject: [PATCH 36/36] fix more errors introduced lately --- cmd/apikey.go | 43 ++++++++++------------ vendor/github.com/elastic/go-ucfg/error.go | 3 -- 2 files changed, 20 insertions(+), 26 deletions(-) diff --git a/cmd/apikey.go b/cmd/apikey.go index cbd5675f30f..e6a985a0351 100644 --- a/cmd/apikey.go +++ b/cmd/apikey.go @@ -84,8 +84,7 @@ If no privilege(s) are specified, the API Key will be valid for all.`, if len(privileges) == 0 { privileges = []es.PrivilegeAction{auth.ActionAny} } - createAPIKeyWithPrivileges(client, keyName, expiration, privileges, json) - return nil + return createAPIKeyWithPrivileges(client, keyName, expiration, privileges, json) }, } create.Flags().StringVar(&keyName, "name", "apm-key", "API Key name") @@ -124,8 +123,7 @@ If neither of them are, an error will be returned.`, if err != nil { return err } - invalidateAPIKey(client, &id, &name, purge, json) - return nil + return invalidateAPIKey(client, &id, &name, purge, json) }, } invalidate.Flags().StringVar(&id, "id", "", "id of the API Key to delete") @@ -157,8 +155,7 @@ If neither of them are, an error will be returned.`, if err != nil { return err } - getAPIKey(client, &id, &name, validOnly, json) - return nil + return getAPIKey(client, &id, &name, validOnly, json) }, } info.Flags().StringVar(&id, "id", "", "id of the API Key to query") @@ -192,8 +189,7 @@ If no privilege(s) are specified, the credentials will be queried for all.` // can't use "*" for querying privileges = auth.ActionsAll() } - verifyAPIKey(config, privileges, credentials, json) - return nil + return verifyAPIKey(config, privileges, credentials, json) }, } verify.Flags().StringVar(&credentials, "credentials", "", `credentials for which check privileges (required)`) @@ -273,7 +269,7 @@ func booleansToPrivileges(ingest, sourcemap, agentConfig bool) []es.PrivilegeAct // creates an API Key with the given privileges, *AND* all the privileges modeled in apm-server // we need to ensure forward-compatibility, for which future privileges must be created here and // during server startup because we don't know if customers will run this command -func createAPIKeyWithPrivileges(client es.Client, keyName, expiry string, privileges []es.PrivilegeAction, asJSON bool) { +func createAPIKeyWithPrivileges(client es.Client, keyName, expiry string, privileges []es.PrivilegeAction, asJSON bool) error { var privilegesRequest = make(es.CreatePrivilegesRequest) event := auth.PrivilegeEventWrite agentConfig := auth.PrivilegeAgentConfigRead @@ -288,7 +284,7 @@ func createAPIKeyWithPrivileges(client es.Client, keyName, expiry string, privil if err != nil { printErr(err, asJSON) - return + return err } // Elasticsearch will allow a user without the right apm privileges to create API keys, but the keys won't validate @@ -305,7 +301,7 @@ func createAPIKeyWithPrivileges(client es.Client, keyName, expiry string, privil }, "") if err != nil { printErr(err, asJSON) - return + return err } if !hasPrivileges.HasAll { printErr(fmt.Errorf(`%s does not have privileges to create API keys. @@ -320,7 +316,7 @@ PUT /_security/role/my_role { ... } `, hasPrivileges.Username), asJSON) - return + return err } printText, printJSON := printers(asJSON) @@ -351,7 +347,7 @@ PUT /_security/role/my_role { apikey, err := es.CreateAPIKey(client, apikeyRequest) if err != nil { printErr(err, asJSON) - return + return err } credentials := base64.StdEncoding.EncodeToString([]byte(apikey.ID + ":" + apikey.Key)) apikey.Credentials = &credentials @@ -371,9 +367,10 @@ PUT /_security/role/my_role { CreateAPIKeyResponse: apikey, Privileges: privilegesCreated, }) + return nil } -func getAPIKey(client es.Client, id, name *string, validOnly, asJSON bool) { +func getAPIKey(client es.Client, id, name *string, validOnly, asJSON bool) error { if isSet(id) { name = nil } else if isSet(name) { @@ -389,7 +386,7 @@ func getAPIKey(client es.Client, id, name *string, validOnly, asJSON bool) { apikeys, err := es.GetAPIKeys(client, request) if err != nil { printErr(err, asJSON) - return + return err } transform := es.GetAPIKeyResponse{APIKeys: make([]es.APIKeyResponse, 0)} @@ -413,9 +410,10 @@ func getAPIKey(client es.Client, id, name *string, validOnly, asJSON bool) { } printText("%d API Keys found", len(transform.APIKeys)) printJSON(transform) + return nil } -func invalidateAPIKey(client es.Client, id, name *string, deletePrivileges, asJSON bool) { +func invalidateAPIKey(client es.Client, id, name *string, deletePrivileges, asJSON bool) error { if isSet(id) { name = nil } else if isSet(name) { @@ -431,7 +429,7 @@ func invalidateAPIKey(client es.Client, id, name *string, deletePrivileges, asJS invalidation, err := es.InvalidateAPIKey(client, invalidateKeysRequest) if err != nil { printErr(err, asJSON) - return + return err } printText, printJSON := printers(asJSON) out := struct { @@ -457,18 +455,16 @@ func invalidateAPIKey(client es.Client, id, name *string, deletePrivileges, asJS if err != nil { continue } - if _, ok := deletion[auth.Application]; !ok { - continue - } if result, ok := deletion[auth.Application][privilege.Name]; ok && result.Found { printText("Deleted privilege \"%v\"", privilege) } out.Privileges = append(out.Privileges, deletion) } printJSON(out) + return nil } -func verifyAPIKey(config *config.Config, privileges []es.PrivilegeAction, credentials string, asJSON bool) { +func verifyAPIKey(config *config.Config, privileges []es.PrivilegeAction, credentials string, asJSON bool) error { perms := make(es.Permissions) printText, printJSON := printers(asJSON) @@ -499,6 +495,7 @@ func verifyAPIKey(config *config.Config, privileges []es.PrivilegeAction, creden } else { printJSON(perms) } + return err } func humanBool(b bool) string { @@ -519,11 +516,11 @@ func humanPrivilege(privilege es.PrivilegeAction) string { func humanTime(millis *int64) string { if millis == nil { - return fmt.Sprint("never") + return "never" } seconds := time.Until(time.Unix(*millis/1000, 0)).Seconds() if seconds < 0 { - return fmt.Sprintf("expired") + return "expired" } minutes := math.Round(seconds / 60) if minutes < 2 { diff --git a/vendor/github.com/elastic/go-ucfg/error.go b/vendor/github.com/elastic/go-ucfg/error.go index 9bd994e4526..a48690e30a8 100644 --- a/vendor/github.com/elastic/go-ucfg/error.go +++ b/vendor/github.com/elastic/go-ucfg/error.go @@ -320,9 +320,6 @@ func raiseInvalidDuration(v value, err error) Error { func raiseValidation(ctx context, meta *Meta, field string, err error) Error { path := "" - if path == "rum.source_mapping.elasticsearch.hosts" { - fmt.Sprintf("FUUUCK") - } if field == "" { path = ctx.path(".") } else {