diff --git a/NOTICE.txt b/NOTICE.txt index 233e5a353bc..94d3efe4c4e 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -1264,11 +1264,11 @@ SOFTWARE -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-libs -Version: v0.12.1-0.20241009140206-6634efed32ab +Version: v0.12.1 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.12.1-0.20241009140206-6634efed32ab/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.12.1/LICENSE: Apache License Version 2.0, January 2004 diff --git a/changelog/fragments/1726735182-Elastic-Defend-accepts-passphrase-protected-client-certificate-key-for-mTLS.yaml b/changelog/fragments/1726735182-Elastic-Defend-accepts-passphrase-protected-client-certificate-key-for-mTLS.yaml new file mode 100644 index 00000000000..36b7579892c --- /dev/null +++ b/changelog/fragments/1726735182-Elastic-Defend-accepts-passphrase-protected-client-certificate-key-for-mTLS.yaml @@ -0,0 +1,32 @@ +# Kind can be one of: +# - breaking-change: a change to previously-documented behavior +# - deprecation: functionality that is being removed in a later release +# - bug-fix: fixes a problem in a previous version +# - enhancement: extends functionality but does not break or fix existing behavior +# - feature: new functionality +# - known-issue: problems that we are aware of in a given version +# - security: impacts on the security of a product or a user’s deployment. +# - upgrade: important information for someone upgrading from a prior version +# - other: does not fit into any of the other categories +kind: feature + +# Change summary; a 80ish characters long description of the change. +summary: Elastic Defend accepts passphrase protected client certificate key for mTLS + +# Long description; in case the summary is not enough to describe the change +# this field accommodate a description without length limits. +# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment. +#description: + +# Affected component; a word indicating the component this changeset affects. +#component: + +# PR URL; optional; the PR number that added the changeset. +# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added. +# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number. +# Please provide it if you are adding a fragment for a different PR. +#pr: https://github.com/owner/repo/1234 + +# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of). +# If not present is automatically filled by the tooling with the issue linked to the PR number. +issue: https://github.com/elastic/elastic-agent/issues/5490 diff --git a/go.mod b/go.mod index d1c664dc62c..e24f69b15a6 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/dolmen-go/contextio v0.0.0-20200217195037-68fc5150bcd5 github.com/elastic/elastic-agent-autodiscover v0.9.0 github.com/elastic/elastic-agent-client/v7 v7.16.0 - github.com/elastic/elastic-agent-libs v0.12.1-0.20241009140206-6634efed32ab + github.com/elastic/elastic-agent-libs v0.12.1 github.com/elastic/elastic-agent-system-metrics v0.11.3 github.com/elastic/elastic-transport-go/v8 v8.6.0 github.com/elastic/go-elasticsearch/v8 v8.15.0 diff --git a/go.sum b/go.sum index 90b594116bf..2ebcfc9dc71 100644 --- a/go.sum +++ b/go.sum @@ -264,8 +264,8 @@ github.com/elastic/elastic-agent-autodiscover v0.9.0 h1:+iWIKh0u3e8I+CJa3FfWe9h0 github.com/elastic/elastic-agent-autodiscover v0.9.0/go.mod h1:5iUxLHhVdaGSWYTveSwfJEY4RqPXTG13LPiFoxcpFd4= github.com/elastic/elastic-agent-client/v7 v7.16.0 h1:yKGq2+CxAuW8Kh0EoNl202tqAyQKfBcPRawVKs2Jve0= github.com/elastic/elastic-agent-client/v7 v7.16.0/go.mod h1:6h+f9QdIr3GO2ODC0Y8+aEXRwzbA5W4eV4dd/67z7nI= -github.com/elastic/elastic-agent-libs v0.12.1-0.20241009140206-6634efed32ab h1:7XHtHd5w4BV48ToCKGE4NYA0kpI+hv+7S08kdiTjpac= -github.com/elastic/elastic-agent-libs v0.12.1-0.20241009140206-6634efed32ab/go.mod h1:5CR02awPrBr+tfmjBBK+JI+dMmHNQjpVY24J0wjbC7M= +github.com/elastic/elastic-agent-libs v0.12.1 h1:5jkxMx15Bna8cq7/Sz/XUIVUXfNWiJ80iSk4ICQ7KJ0= +github.com/elastic/elastic-agent-libs v0.12.1/go.mod h1:5CR02awPrBr+tfmjBBK+JI+dMmHNQjpVY24J0wjbC7M= github.com/elastic/elastic-agent-system-metrics v0.11.3 h1:LDzRwP8kxvsYEtMDgMSKZs1TgPcSEukit+/EAP5Y28A= github.com/elastic/elastic-agent-system-metrics v0.11.3/go.mod h1:saqLKe9fuyuAo6IADAnnuy1kaBI7VNlxfwMo8KzSRyQ= github.com/elastic/elastic-transport-go/v8 v8.6.0 h1:Y2S/FBjx1LlCv5m6pWAF2kDJAHoSjSRSJCApolgfthA= diff --git a/internal/pkg/agent/application/application.go b/internal/pkg/agent/application/application.go index c9d50e4dd3a..eb7fbe1ab3f 100644 --- a/internal/pkg/agent/application/application.go +++ b/internal/pkg/agent/application/application.go @@ -172,9 +172,11 @@ func New( log.Info("Parsed configuration and determined agent is managed by Fleet") composableManaged = true - compModifiers = append(compModifiers, FleetServerComponentModifier(cfg.Fleet.Server), + compModifiers = append(compModifiers, + FleetServerComponentModifier(cfg.Fleet.Server), InjectFleetConfigComponentModifier(cfg.Fleet, agentInfo), EndpointSignedComponentModifier(), + EndpointTLSComponentModifier(log), InjectProxyEndpointModifier(), ) diff --git a/internal/pkg/agent/application/endpoint_component_modifier.go b/internal/pkg/agent/application/endpoint_component_modifier.go new file mode 100644 index 00000000000..bc9b4b1db07 --- /dev/null +++ b/internal/pkg/agent/application/endpoint_component_modifier.go @@ -0,0 +1,265 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package application + +import ( + "encoding/pem" + "errors" + "fmt" + "os" + "sync" + + "github.com/elastic/elastic-agent-client/v7/pkg/client" + "github.com/elastic/elastic-agent-libs/transport/tlscommon" + "github.com/elastic/elastic-agent/internal/pkg/agent/application/coordinator" + "github.com/elastic/elastic-agent/pkg/component" + "github.com/elastic/elastic-agent/pkg/core/logger" +) + +// tlsCache is used to cache the decrypted client certificate and key. +// In environments with a high rate of config changes, such as in Kubernetes +// using autodiscover, loading files and decrypting them might have a +// non-negligible overhead. +type tlsCache struct { + mu *sync.Mutex + + CacheKey string + + Certificate string + Key string +} + +func (tlsCache) MakeKey(keyPassPath, certPath, keyPath string) string { + return keyPassPath + certPath + keyPath +} + +// EndpointSignedComponentModifier copies "signed" properties to the top level "signed" for the endpoint input. +// Enpoint team want to be able to validate the signature and parse the signed configuration (not trust the agent). +// Endpoint uses uninstall_token_hash in order to verify uninstall command token +// and signing_key in order validate the action signature. +// Example: +// +// { +// .... +// "signed": { +// "data": "eyJpZCI6ImFhZWM4OTYwLWJiYjAtMTFlZC1hYzBkLTVmNjI0YTQxZjM4OCIsImFnZW50Ijp7InByb3RlY3Rpb24iOnsiZW5hYmxlZCI6dHJ1ZSwidW5pbnN0YWxsX3Rva2VuX2hhc2giOiIiLCJzaWduaW5nX2tleSI6Ik1Ga3dFd1lIS29aSXpqMENBUVlJS29aSXpqMERBUWNEUWdBRW1tckhDSTdtZ2tuZUJlYVJkc2VkQXZBU2l0UHRLbnpPdUlzeHZJRWdGTkFLVlg3MWpRTTVmalo1eUdsSDB0TmJuR2JrU2pVM0VEVUZsOWllQ1J0ME5nPT0ifX19", +// "signature": "MEUCIQCWoScyJW0dejHFxXBTEcSCOZiBHRVMjuJRPwFCwOdA1QIgKrtKUBzkvVeljRtJyMXfD8zIvWjrMzqhSkgjNESPW5E=" +// }, +// "revision": 1, +// "type": "endpoint" +// } +func EndpointSignedComponentModifier() coordinator.ComponentsModifier { + return func(comps []component.Component, cfg map[string]interface{}) ([]component.Component, error) { + const signedKey = "signed" + + compIdx, unitIdx, ok := findEndpointUnit(comps, client.UnitTypeInput) + if !ok { + return comps, nil + } + + unit := comps[compIdx].Units[unitIdx] + unitCfgMap := unit.Config.Source.AsMap() + if signed, ok := cfg[signedKey]; ok { + unitCfgMap[signedKey] = signed + } + + unitCfg, err := component.ExpectedConfig(unitCfgMap) + if err != nil { + return nil, err + } + + unit.Config = unitCfg + comps[compIdx].Units[unitIdx] = unit + + return comps, nil + } +} + +// EndpointTLSComponentModifier decrypts the client TLS certificate key if it's +// passphrase-protected. It replaces the content of 'fleet.ssl.key' +// and 'certificate' with theirs decrypted version and removes +// 'key_passphrase_path'. +// It does so, ONLY for the client TLS configuration for mTLS used with +// fleet-server. +func EndpointTLSComponentModifier(log *logger.Logger) coordinator.ComponentsModifier { + return newEndpointTLSComponentModifier(log, &tlsCache{mu: &sync.Mutex{}}) +} + +func newEndpointTLSComponentModifier(log *logger.Logger, cache *tlsCache) func(comps []component.Component, cfg map[string]interface{}) ([]component.Component, error) { + return func(comps []component.Component, cfg map[string]interface{}) ([]component.Component, error) { + compIdx, unitIdx, ok := findEndpointUnit(comps, client.UnitTypeInput) + if !ok { + // endpoint not present, nothing to do + return comps, nil + } + + unit := comps[compIdx].Units[unitIdx] + unitCfgMap := unit.Config.Source.AsMap() + + // ensure the following config exists: + // fleet.ssl: + // key_passphrase_path + // certificate + // key + fleetNode, ok := unitCfgMap["fleet"] + if !ok { + // if 'fleet' isn't, present nothing to do + return comps, nil + } + fleetMap, ok := fleetNode.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("EndpointTLSComponentModifier: 'fleet' node isn't a map, it is: %T", fleetNode) + } + + sslNode, ok := fleetMap["ssl"] + if !ok { + // 'ssl' node not present isn't an issue + return comps, nil + } + sslMap, ok := sslNode.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("EndpointTLSComponentModifier: 'ssl' node isn't a map, it is: %T", sslNode) + } + + keyPassPathI, ok := sslMap["key_passphrase_path"] + if !ok { + // if no key_passphrase_path, nothing to decrypt + return comps, nil + } + keyPassPath, ok := keyPassPathI.(string) + if !ok { + return nil, errors.New("EndpointTLSComponentModifier: 'key_passphrase_path' isn't a string") + } + if keyPassPath == "" { + // key_passphrase_path shouldn't be empty, but if it's, nothing to decrypt + return comps, nil + } + + keyI, ok := sslMap["key"] + if !ok { + // if there is a key_passphrase_path, the key must be present + return nil, errors.New("EndpointTLSComponentModifier: 'key_passphrase_path' present, but 'key' isn't present") + } + keyPath, ok := keyI.(string) + if !ok { + return nil, fmt.Errorf("EndpointTLSComponentModifier: 'key' isn't a string, it's %T", keyI) + } + + certI, ok := sslMap["certificate"] + if !ok { + // if there is a key_passphrase_path, the certificate must be present + return nil, errors.New("EndpointTLSComponentModifier: 'key_passphrase_path' present, but 'certificate' isn't present") + } + certPath, ok := certI.(string) + if !ok { + return nil, errors.New("EndpointTLSComponentModifier: 'certificate' isn't a string") + } + + // all TLS config exists and the certificate key is passphrase protected, + // now load and decrypt the key. + cert, key, err := loadCertificatesWithCache(log, cache, keyPassPath, certPath, keyPath) + if err != nil { + return nil, err + } + + // remove 'key_passphrase_path' as the certificate key isn't encrypted + // anymore. + delete(sslMap, "key_passphrase_path") + + // update the certificate and its key with their decrypted version. + sslMap["certificate"] = cert + sslMap["key"] = key + + unitCfg, err := component.ExpectedConfig(unitCfgMap) + if err != nil { + return nil, fmt.Errorf("EndpointTLSComponentModifier: could not covert modified config to expected config: %w", err) + } + + unit.Config = unitCfg + comps[compIdx].Units[unitIdx] = unit + + return comps, nil + } +} + +func loadCertificatesWithCache(log *logger.Logger, cache *tlsCache, keyPassPath string, certPath string, keyPath string) (string, string, error) { + cache.mu.Lock() + defer cache.mu.Unlock() + + cacheKey := cache.MakeKey(keyPassPath, certPath, keyPath) + + // cache hit + if cache.CacheKey == cacheKey { + return cache.Certificate, cache.Key, nil + } + + cert, key, err := loadCertificates(log, keyPassPath, certPath, keyPath) + if err != nil { + return "", "", err + } + + cache.CacheKey = cacheKey + cache.Certificate = cert + cache.Key = key + + return cert, key, nil +} + +func loadCertificates(log *logger.Logger, keyPassPathStr string, certStr string, keyStr string) (string, string, error) { + pass, err := os.ReadFile(keyPassPathStr) + if err != nil { + return "", "", fmt.Errorf("EndpointTLSComponentModifier: failed to read client certificate passphrase file: %w", err) + } + + // we don't really support encrypted certificates, but it's how + // tlscommon.LoadCertificate does. Thus, let's keep the same behaviour. + // Also, tlscommon.LoadCertificate 'loses' the type of the private key. + // It stores they private key as an interface and there is no way to + // retrieve the type os the private key without a type assertion. + // Therefore, instead of manually checking the type, which would mean + // to check for all supported private key types and keep it up to date, + // better to load the certificate and its key directly from the PEM file. + cert, err := tlscommon.ReadPEMFile(log, + certStr, string(pass)) + if err != nil { + return "", "", fmt.Errorf("EndpointTLSComponentModifier: unable to load TLS certificate: %w", err) + } + key, err := tlscommon.ReadPEMFile(log, + keyStr, + string(pass)) + if err != nil { + return "", "", fmt.Errorf("EndpointTLSComponentModifier: unable to load TLS certificate key: %w", err) + } + + // tlscommon.ReadPEMFile only removes the 'DEK-Info' header, not the + // 'Proc-Type', so remove it now. Create a pem.Block to avoid editing + // the PEM data manually: + keyBlock, _ := pem.Decode(key) + delete(keyBlock.Headers, "Proc-Type") + key = pem.EncodeToMemory(keyBlock) + + return string(cert), string(key), nil +} + +// findEndpointUnit finds the endpoint component and its unit of type 'unitType'. +// It returns the component and unit index and true if found, if not, it returns +// 0, 0, false. +func findEndpointUnit(comps []component.Component, unitType client.UnitType) (int, int, bool) { + // find the endpoint component + for compIdx, comp := range comps { + if comp.InputSpec != nil && comp.InputSpec.InputType != endpoint { + continue + } + + for unitIdx, unit := range comp.Units { + if unit.Type != unitType { + continue + } + + return compIdx, unitIdx, true + } + } + return 0, 0, false +} diff --git a/internal/pkg/agent/application/endpoint_component_modifier_test.go b/internal/pkg/agent/application/endpoint_component_modifier_test.go new file mode 100644 index 00000000000..424b2709f52 --- /dev/null +++ b/internal/pkg/agent/application/endpoint_component_modifier_test.go @@ -0,0 +1,612 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package application + +import ( + "crypto/rand" + "crypto/x509" + "encoding/pem" + "fmt" + "os" + "path/filepath" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/elastic-agent-client/v7/pkg/client" + "github.com/elastic/elastic-agent-client/v7/pkg/proto" + "github.com/elastic/elastic-agent-libs/testing/certutil" + "github.com/elastic/elastic-agent/internal/pkg/agent/application/coordinator" + "github.com/elastic/elastic-agent/pkg/core/logger/loggertest" + + "github.com/elastic/elastic-agent/pkg/component" + + "google.golang.org/protobuf/types/known/structpb" +) + +func TestEndpointComponentModifier(t *testing.T) { + log, obs := loggertest.New("TestEndpointSignedComponentModifier") + defer func() { + if !t.Failed() { + return + } + + loggertest.PrintObservedLogs(obs.TakeAll(), t.Log) + }() + + pair, certPath, certKeyPath, certKeyPassPath := prepareEncTLSCertificates(t) + + tests := map[string][]struct { + name string + compModifier coordinator.ComponentsModifier + comps []component.Component + cfg map[string]interface{} + wantComps []component.Component + wantErr func(*testing.T, error) + }{ + "EndpointSignedComponentModifier": { + { + name: "nil", + compModifier: EndpointSignedComponentModifier(), + }, + { + name: "non endpoint", + compModifier: EndpointSignedComponentModifier(), + comps: []component.Component{ + { + ID: "asdfasd", + InputSpec: &component.InputRuntimeSpec{ + InputType: "osquery", + }, + Units: []component.Unit{ + { + ID: "34534", + Type: client.UnitTypeInput, + }, + }, + }, + }, + wantComps: []component.Component{ + { + ID: "asdfasd", + InputSpec: &component.InputRuntimeSpec{ + InputType: "osquery", + }, + Units: []component.Unit{ + { + ID: "34534", + Type: client.UnitTypeInput, + }, + }, + }, + }, + }, + { + name: "endpoint", + compModifier: EndpointSignedComponentModifier(), + comps: []component.Component{ + { + ID: "asdfasd", + InputSpec: &component.InputRuntimeSpec{ + InputType: "endpoint", + }, + Units: []component.Unit{ + { + ID: "34534", + Type: client.UnitTypeInput, + Config: &proto.UnitExpectedConfig{ + Type: "endpoint", + Source: &structpb.Struct{}, + }, + }, + }, + }, + }, + cfg: map[string]interface{}{ + "signed": map[string]interface{}{ + "data": "eyJpZCI6ImFhZWM4OTYwLWJiYjAtMTFlZC1hYzBkLTVmNjI0YTQxZjM4OCIsImFnZW50Ijp7InByb3RlY3Rpb24iOnsiZW5hYmxlZCI6dHJ1ZSwidW5pbnN0YWxsX3Rva2VuX2hhc2giOiIiLCJzaWduaW5nX2tleSI6Ik1Ga3dFd1lIS29aSXpqMENBUVlJS29aSXpqMERBUWNEUWdBRW1tckhDSTdtZ2tuZUJlYVJkc2VkQXZBU2l0UHRLbnpPdUlzeHZJRWdGTkFLVlg3MWpRTTVmalo1eUdsSDB0TmJuR2JrU2pVM0VEVUZsOWllQ1J0ME5nPT0ifX19", + "signature": "MEUCIQCWoScyJW0dejHFxXBTEcSCOZiBHRVMjuJRPwFCwOdA1QIgKrtKUBzkvVeljRtJyMXfD8zIvWjrMzqhSkgjNESPW5E=", + }, + }, + wantComps: []component.Component{ + { + ID: "asdfasd", + InputSpec: &component.InputRuntimeSpec{ + InputType: "endpoint", + }, + Units: []component.Unit{ + { + ID: "34534", + Type: client.UnitTypeInput, + Config: &proto.UnitExpectedConfig{ + Source: func() *structpb.Struct { + var source structpb.Struct + err := source.UnmarshalJSON([]byte(`{"signed":{"data":"eyJpZCI6ImFhZWM4OTYwLWJiYjAtMTFlZC1hYzBkLTVmNjI0YTQxZjM4OCIsImFnZW50Ijp7InByb3RlY3Rpb24iOnsiZW5hYmxlZCI6dHJ1ZSwidW5pbnN0YWxsX3Rva2VuX2hhc2giOiIiLCJzaWduaW5nX2tleSI6Ik1Ga3dFd1lIS29aSXpqMENBUVlJS29aSXpqMERBUWNEUWdBRW1tckhDSTdtZ2tuZUJlYVJkc2VkQXZBU2l0UHRLbnpPdUlzeHZJRWdGTkFLVlg3MWpRTTVmalo1eUdsSDB0TmJuR2JrU2pVM0VEVUZsOWllQ1J0ME5nPT0ifX19", "signature":"MEUCIQCWoScyJW0dejHFxXBTEcSCOZiBHRVMjuJRPwFCwOdA1QIgKrtKUBzkvVeljRtJyMXfD8zIvWjrMzqhSkgjNESPW5E="}}`)) + require.NoError(t, err, "could not create want component source config") + return &source + }(), + }, + }, + }, + }, + }, + }, + }, + "EndpointTLSComponentModifier": { + { + name: "nil", + compModifier: EndpointSignedComponentModifier(), + }, + { + name: "non endpoint", + compModifier: EndpointSignedComponentModifier(), + comps: makeComponent(t, "{}"), + wantComps: makeComponent(t, "{}"), + }, + + { + name: "endpoint-no-fleet", + compModifier: EndpointTLSComponentModifier(log), + comps: makeComponent(t, `{}`), + cfg: map[string]interface{}{ + "fleet": map[string]interface{}{}, + }, + wantComps: makeComponent(t, `{}`), + }, + { + name: "endpoint-no-fleet-wrong-type", + compModifier: EndpointTLSComponentModifier(log), + comps: makeComponent(t, `{"fleet": 42}`), + cfg: map[string]interface{}{ + "fleet": 1, + }, + wantComps: nil, + wantErr: func(t *testing.T, err error) { + assert.ErrorContains(t, err, "'fleet' node isn't a map") + }, + }, + { + name: "endpoint-no-fleet.ssl", + compModifier: EndpointTLSComponentModifier(log), + comps: makeComponent(t, `{"fleet": {}}`), + cfg: map[string]interface{}{ + "fleet": map[string]interface{}{}, + }, + wantComps: makeComponent(t, `{"fleet": {}}`), + }, + { + name: "endpoint-wrong-fleet.ssl", + compModifier: EndpointTLSComponentModifier(log), + comps: makeComponent(t, `{"fleet": {"ssl": 42}}`), + cfg: map[string]interface{}{ + "fleet": map[string]interface{}{}, + }, + wantComps: nil, + wantErr: func(t *testing.T, err error) { + assert.ErrorContains(t, err, "'ssl' node isn't a map") + }, + }, + { + name: "endpoint-wrong-fleet.ssl.key_passphrase_path", + compModifier: EndpointTLSComponentModifier(log), + comps: makeComponent(t, ` + {"fleet": {"ssl": + {"key_passphrase_path": 42}}}`), + cfg: map[string]interface{}{ + "fleet": map[string]interface{}{}, + }, + wantComps: nil, + wantErr: func(t *testing.T, err error) { + assert.ErrorContains(t, err, "'key_passphrase_path' isn't a string") + }, + }, + { + name: "endpoint-wrong-fleet.ssl.key", + compModifier: EndpointTLSComponentModifier(log), + comps: makeComponent(t, ` +{"fleet": {"ssl": { + "key_passphrase_path": "/path/to/passphrase", + "key": 42}}}`), + cfg: map[string]interface{}{ + "fleet": map[string]interface{}{}, + }, + wantComps: nil, + wantErr: func(t *testing.T, err error) { + assert.ErrorContains(t, err, "'key' isn't a string") + }, + }, + { + name: "endpoint-wrong-fleet.ssl.certificate", + compModifier: EndpointTLSComponentModifier(log), + comps: makeComponent(t, ` + {"fleet": {"ssl": { + "key_passphrase_path": "/path/to/passphrase", + "key": "", + "certificate": 42}}}`), + cfg: map[string]interface{}{ + "fleet": map[string]interface{}{}, + }, + wantComps: nil, + wantErr: func(t *testing.T, err error) { + assert.ErrorContains(t, err, "'certificate' isn't a string") + }, + }, + + { + name: "endpoint-mTLS-passphrase", + compModifier: EndpointTLSComponentModifier(log), + comps: makeComponent(t, fmt.Sprintf(`{ + "fleet": { + "ssl": { + "certificate": %q, + "key": %q, + "key_passphrase_path": %q + } + } + }`, certPath, certKeyPath, certKeyPassPath)), + cfg: map[string]interface{}{ + "fleet": map[string]interface{}{ + "ssl": map[string]interface{}{ + "certificate": certPath, + "key": certKeyPath, + "key_passphrase_path": certKeyPassPath, + }, + }, + }, + wantComps: makeComponent(t, fmt.Sprintf(`{ + "fleet": { + "ssl": { + "certificate": %q, + "key": %q + } + } + }`, pair.Cert, pair.Key)), + }, + { + name: "endpoint-mTLS-passphrase-no-key", + compModifier: EndpointTLSComponentModifier(log), + comps: makeComponent(t, fmt.Sprintf(`{ + "fleet": { + "ssl": { + "certificate": %q, + "key_passphrase_path": %q + } + } + }`, certPath, certKeyPassPath)), + cfg: map[string]interface{}{ + "fleet": map[string]interface{}{ + "ssl": map[string]interface{}{ + "certificate": certPath, + "key_passphrase_path": certKeyPassPath, + }, + }, + }, + wantComps: nil, + wantErr: func(t *testing.T, err error) { + assert.ErrorContains(t, err, "'key' isn't present") + }, + }, + { + name: "endpoint-mTLS-passphrase-no-certificate", + compModifier: EndpointTLSComponentModifier(log), + comps: makeComponent(t, `{ + "fleet": { + "ssl": { + "key": "/path/to/key", + "key_passphrase_path": "/path/to/key_passphrase_path" + } + } + }`), + cfg: map[string]interface{}{ + "fleet": map[string]interface{}{ + "ssl": map[string]interface{}{ + "key": "/path/to/cert", + "key_passphrase_path": "/path/to/key_passphrase_path", + }, + }, + }, + wantComps: nil, + wantErr: func(t *testing.T, err error) { + assert.ErrorContains(t, err, "'certificate' isn't present") + }, + }, + { + name: "endpoint-mTLS-no-passphrase", + compModifier: EndpointTLSComponentModifier(log), + comps: makeComponent(t, `{ + "fleet": { + "ssl": { + "certificate": "/path/to/cert", + "key": "/path/to/key" + } + } + }`), + cfg: map[string]interface{}{ + "fleet": map[string]interface{}{ + "ssl": map[string]interface{}{ + "certificate": "/path/to/cert", + "key": "/path/to/key", + }, + }, + }, + wantComps: makeComponent(t, `{ + "fleet": { + "ssl": { + "certificate": "/path/to/cert", + "key": "/path/to/key" + } + } + }`), + }, + { + name: "endpoint-mTLS-empty-passphrase", + compModifier: EndpointTLSComponentModifier(log), + comps: makeComponent(t, `{ + "fleet": { + "ssl": { + "key_passphrase_path": "", + "certificate": "/path/to/cert", + "key": "/path/to/key" + } + } + }`), + cfg: map[string]interface{}{ + "fleet": map[string]interface{}{ + "ssl": map[string]interface{}{ + "key_passphrase_path": "", + "certificate": "/path/to/cert", + "key": "/path/to/key", + }, + }, + }, + wantComps: makeComponent(t, `{ + "fleet": { + "ssl": { + "key_passphrase_path": "", + "certificate": "/path/to/cert", + "key": "/path/to/key" + } + } + }`), + }, + { + name: "endpoint-TLS", + compModifier: EndpointTLSComponentModifier(log), + comps: makeComponent(t, `{ + "fleet": { + "ssl": { + "certificate_authorities": ["/path/to/ca1", "/path/to/ca2"] + } + } + }`), + cfg: map[string]interface{}{ + "fleet": map[string]interface{}{ + "ssl": map[string]interface{}{ + "certificate_authorities": []string{"/path/to/ca1", "/path/to/ca2"}, + }, + }, + }, + wantComps: makeComponent(t, `{ + "fleet": { + "ssl": { + "certificate_authorities": ["/path/to/ca1", "/path/to/ca2"] + } + } + }`), + }, + }, + } + + for name, tcs := range tests { + t.Run(name, func(t *testing.T) { + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + comps, err := tc.compModifier(tc.comps, tc.cfg) + + if tc.wantErr != nil { + tc.wantErr(t, err) + } else { + assert.NoError(t, err) + } + + // Cumbersome comparison of the source config encoded in protobuf, + // cmp panics protobufs comparison otherwise. + compareComponents(t, comps, tc.wantComps) + }) + } + }) + } +} + +func compareComponents(t *testing.T, got, want []component.Component) { + if len(want) > 0 && + len(want[0].Units) > 0 && + got[0].Units[0].Config != nil && + got[0].Units[0].Config.Source != nil { + + unitCgf := got[0].Units[0].Config.Source.AsMap() + wantUnitCfg := want[0].Units[0].Config.Source.AsMap() + + assert.Equal(t, wantUnitCfg, unitCgf, "unit config do not match") + } +} + +func TestEndpointTLSComponentModifier_cache_miss(t *testing.T) { + log, obs := loggertest.New("TestEndpointSignedComponentModifier") + defer func() { + if !t.Failed() { + return + } + + loggertest.PrintObservedLogs(obs.TakeAll(), t.Log) + }() + + cache := tlsCache{ + mu: &sync.Mutex{}, + + CacheKey: "/old-cache-key", + Certificate: "cached certificate", + Key: "cached key", + } + pair, certPath, certKeyPath, certKeyPassPath := prepareEncTLSCertificates(t) + cackeKey := cache.MakeKey(certKeyPassPath, certPath, certKeyPath) + + comps := makeComponent(t, fmt.Sprintf(`{ + "fleet": { + "ssl": { + "certificate": %q, + "key": %q, + "key_passphrase_path": %q + } + } + }`, certPath, certKeyPath, certKeyPassPath)) + cfg := map[string]interface{}{ + "fleet": map[string]interface{}{ + "ssl": map[string]interface{}{ + "certificate": certPath, + "key": certKeyPath, + "key_passphrase_path": certKeyPassPath, + }, + }, + } + wantComps := makeComponent(t, fmt.Sprintf(`{ + "fleet": { + "ssl": { + "certificate": %q, + "key": %q + } + } + }`, pair.Cert, pair.Key)) + + modifier := newEndpointTLSComponentModifier(log, &cache) + got, err := modifier(comps, cfg) + require.NoError(t, err, "unexpected error") + + assert.Equal(t, cackeKey, cache.CacheKey, "passphrase path did not match") + assert.Equal(t, string(pair.Cert), cache.Certificate, "certificate did not match") + assert.Equal(t, string(pair.Key), cache.Key, "key did not match") + + compareComponents(t, got, wantComps) +} + +func TestEndpointTLSComponentModifier_cache_hit(t *testing.T) { + log, obs := loggertest.New("TestEndpointSignedComponentModifier") + defer func() { + if !t.Failed() { + return + } + + loggertest.PrintObservedLogs(obs.TakeAll(), t.Log) + }() + + certPath := "/path/to/cert" + certKeyPath := "/path/to/key" + certKeyPassPath := "/path/to/key_passphrase_path" //nolint:gosec // not a real key + + cache := tlsCache{ + mu: &sync.Mutex{}, + + Certificate: "cached certificate", + Key: "cached key", + } + cacheKey := cache.MakeKey(certKeyPassPath, certPath, certKeyPath) + cache.CacheKey = cacheKey + + comps := makeComponent(t, fmt.Sprintf(`{ + "fleet": { + "ssl": { + "certificate": %q, + "key": %q, + "key_passphrase_path": %q + } + } + }`, certPath, certKeyPath, certKeyPassPath)) + cfg := map[string]interface{}{ + "fleet": map[string]interface{}{ + "ssl": map[string]interface{}{ + "certificate": cache.Certificate, + "key": cache.Key, + "key_passphrase_path": cache.CacheKey, + }, + }, + } + + wantComps := makeComponent(t, fmt.Sprintf(`{ + "fleet": { + "ssl": { + "certificate": %q, + "key": %q + } + } + }`, cache.Certificate, cache.Key)) + + modifier := newEndpointTLSComponentModifier(log, &cache) + got, err := modifier(comps, cfg) + require.NoError(t, err, "unexpected error") + + assert.Equal(t, cacheKey, cache.CacheKey, "passphrase should not have changed") + compareComponents(t, got, wantComps) +} + +func prepareEncTLSCertificates(t *testing.T) (certutil.Pair, string, string, string) { + passphrase := "secure_passphrase" + _, _, pair, err := certutil.NewRootCA() + require.NoError(t, err, "could not create TLS certificate") + agentChildDERKey, _ := pem.Decode(pair.Key) + require.NoError(t, err, "could not create tls.Certificates from child certificate") + + encPem, err := x509.EncryptPEMBlock( //nolint:staticcheck // we need to drop support for this, but while we don't, it needs to be tested. + rand.Reader, + "EC PRIVATE KEY", + agentChildDERKey.Bytes, + []byte(passphrase), + x509.PEMCipherAES128) + require.NoError(t, err, "failed encrypting agent child certificate key block") + + certKeyEnc := pem.EncodeToMemory(encPem) + + // save to disk + tmpDir := t.TempDir() + certPath := filepath.Join(tmpDir, "cert.pem") + certKeyPath := filepath.Join(tmpDir, "key.pem") + certKeyPassPath := filepath.Join(tmpDir, "key_pass.pem") + + err = os.WriteFile(certPath, pair.Cert, 0400) + require.NoError(t, err, "could write certificate key") + err = os.WriteFile(certKeyPath, certKeyEnc, 0400) + require.NoError(t, err, "could write certificate key") + err = os.WriteFile(certKeyPassPath, []byte(passphrase), 0400) + require.NoError(t, err, "could write certificate key passphrase") + + return pair, certPath, certKeyPath, certKeyPassPath +} + +func makeComponent(t *testing.T, sourceCfg string) []component.Component { + return []component.Component{ + { + ID: "ClientCertKey", + InputSpec: &component.InputRuntimeSpec{ + InputType: "endpoint", + }, + Units: []component.Unit{ + { + ID: "34534", + Type: client.UnitTypeInput, + Config: &proto.UnitExpectedConfig{ + Type: "endpoint", + Source: func() *structpb.Struct { + var source structpb.Struct + err := source.UnmarshalJSON([]byte(sourceCfg)) + require.NoError(t, err, "could not create component source config") + return &source + }(), + }, + }, + }, + }, + } +} diff --git a/internal/pkg/agent/application/endpoint_signed_component_modifier.go b/internal/pkg/agent/application/endpoint_signed_component_modifier.go deleted file mode 100644 index 39aac35604a..00000000000 --- a/internal/pkg/agent/application/endpoint_signed_component_modifier.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License 2.0; -// you may not use this file except in compliance with the Elastic License 2.0. - -package application - -import ( - "github.com/elastic/elastic-agent-client/v7/pkg/client" - "github.com/elastic/elastic-agent/internal/pkg/agent/application/coordinator" - "github.com/elastic/elastic-agent/pkg/component" -) - -// EndpointSignedComponentModifier copies "signed" properties to the top level "signed" for the endpoint input. -// Enpoint team want to be able to validate the signature and parse the signed configuration (not trust the agent). -// Endpoint uses uninstall_token_hash in order to verify uninstall command token -// and signing_key in order validate the action signature. -// Example: -// -// { -// .... -// "signed": { -// "data": "eyJpZCI6ImFhZWM4OTYwLWJiYjAtMTFlZC1hYzBkLTVmNjI0YTQxZjM4OCIsImFnZW50Ijp7InByb3RlY3Rpb24iOnsiZW5hYmxlZCI6dHJ1ZSwidW5pbnN0YWxsX3Rva2VuX2hhc2giOiIiLCJzaWduaW5nX2tleSI6Ik1Ga3dFd1lIS29aSXpqMENBUVlJS29aSXpqMERBUWNEUWdBRW1tckhDSTdtZ2tuZUJlYVJkc2VkQXZBU2l0UHRLbnpPdUlzeHZJRWdGTkFLVlg3MWpRTTVmalo1eUdsSDB0TmJuR2JrU2pVM0VEVUZsOWllQ1J0ME5nPT0ifX19", -// "signature": "MEUCIQCWoScyJW0dejHFxXBTEcSCOZiBHRVMjuJRPwFCwOdA1QIgKrtKUBzkvVeljRtJyMXfD8zIvWjrMzqhSkgjNESPW5E=" -// }, -// "revision": 1, -// "type": "endpoint" -// } -func EndpointSignedComponentModifier() coordinator.ComponentsModifier { - return func(comps []component.Component, cfg map[string]interface{}) ([]component.Component, error) { - const signedKey = "signed" - for i, comp := range comps { - if comp.InputSpec != nil && (comp.InputSpec.InputType == endpoint) { - for j, unit := range comp.Units { - if unit.Type == client.UnitTypeInput && (unit.Config.Type == endpoint) { - unitCfgMap := unit.Config.Source.AsMap() - if signed, ok := cfg[signedKey]; ok { - unitCfgMap[signedKey] = signed - } - - unitCfg, err := component.ExpectedConfig(unitCfgMap) - if err != nil { - return nil, err - } - - unit.Config = unitCfg - comp.Units[j] = unit - } - } - } - comps[i] = comp - } - return comps, nil - } -} diff --git a/internal/pkg/agent/application/endpoint_signed_component_modifier_test.go b/internal/pkg/agent/application/endpoint_signed_component_modifier_test.go deleted file mode 100644 index fcc16321d5c..00000000000 --- a/internal/pkg/agent/application/endpoint_signed_component_modifier_test.go +++ /dev/null @@ -1,137 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License 2.0; -// you may not use this file except in compliance with the Elastic License 2.0. - -package application - -import ( - "testing" - - "github.com/elastic/elastic-agent-client/v7/pkg/client" - "github.com/elastic/elastic-agent-client/v7/pkg/proto" - - "github.com/elastic/elastic-agent/pkg/component" - - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - "google.golang.org/protobuf/types/known/structpb" -) - -func TestEndpointSignedComponentModifier(t *testing.T) { - compModifier := EndpointSignedComponentModifier() - - tests := []struct { - name string - comps []component.Component - cfg map[string]interface{} - wantComps []component.Component - wantErr error - }{ - { - name: "nil", - }, - { - name: "non endpoint", - comps: []component.Component{ - { - ID: "asdfasd", - InputSpec: &component.InputRuntimeSpec{ - InputType: "osquery", - }, - Units: []component.Unit{ - { - ID: "34534", - Type: client.UnitTypeInput, - }, - }, - }, - }, - wantComps: []component.Component{ - { - ID: "asdfasd", - InputSpec: &component.InputRuntimeSpec{ - InputType: "osquery", - }, - Units: []component.Unit{ - { - ID: "34534", - Type: client.UnitTypeInput, - }, - }, - }, - }, - }, - { - name: "endpoint", - comps: []component.Component{ - { - ID: "asdfasd", - InputSpec: &component.InputRuntimeSpec{ - InputType: "endpoint", - }, - Units: []component.Unit{ - { - ID: "34534", - Type: client.UnitTypeInput, - Config: &proto.UnitExpectedConfig{ - Type: "endpoint", - Source: &structpb.Struct{}, - }, - }, - }, - }, - }, - cfg: map[string]interface{}{ - "signed": map[string]interface{}{ - "data": "eyJpZCI6ImFhZWM4OTYwLWJiYjAtMTFlZC1hYzBkLTVmNjI0YTQxZjM4OCIsImFnZW50Ijp7InByb3RlY3Rpb24iOnsiZW5hYmxlZCI6dHJ1ZSwidW5pbnN0YWxsX3Rva2VuX2hhc2giOiIiLCJzaWduaW5nX2tleSI6Ik1Ga3dFd1lIS29aSXpqMENBUVlJS29aSXpqMERBUWNEUWdBRW1tckhDSTdtZ2tuZUJlYVJkc2VkQXZBU2l0UHRLbnpPdUlzeHZJRWdGTkFLVlg3MWpRTTVmalo1eUdsSDB0TmJuR2JrU2pVM0VEVUZsOWllQ1J0ME5nPT0ifX19", - "signature": "MEUCIQCWoScyJW0dejHFxXBTEcSCOZiBHRVMjuJRPwFCwOdA1QIgKrtKUBzkvVeljRtJyMXfD8zIvWjrMzqhSkgjNESPW5E=", - }, - }, - wantComps: []component.Component{ - { - ID: "asdfasd", - InputSpec: &component.InputRuntimeSpec{ - InputType: "endpoint", - }, - Units: []component.Unit{ - { - ID: "34534", - Type: client.UnitTypeInput, - Config: &proto.UnitExpectedConfig{ - Source: func() *structpb.Struct { - var source structpb.Struct - err := source.UnmarshalJSON([]byte(`{"signed":{"data":"eyJpZCI6ImFhZWM4OTYwLWJiYjAtMTFlZC1hYzBkLTVmNjI0YTQxZjM4OCIsImFnZW50Ijp7InByb3RlY3Rpb24iOnsiZW5hYmxlZCI6dHJ1ZSwidW5pbnN0YWxsX3Rva2VuX2hhc2giOiIiLCJzaWduaW5nX2tleSI6Ik1Ga3dFd1lIS29aSXpqMENBUVlJS29aSXpqMERBUWNEUWdBRW1tckhDSTdtZ2tuZUJlYVJkc2VkQXZBU2l0UHRLbnpPdUlzeHZJRWdGTkFLVlg3MWpRTTVmalo1eUdsSDB0TmJuR2JrU2pVM0VEVUZsOWllQ1J0ME5nPT0ifX19", "signature":"MEUCIQCWoScyJW0dejHFxXBTEcSCOZiBHRVMjuJRPwFCwOdA1QIgKrtKUBzkvVeljRtJyMXfD8zIvWjrMzqhSkgjNESPW5E="}}`)) - if err != nil { - t.Fatal(err) - } - return &source - }(), - }, - }, - }, - }, - }, - }, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - comps, err := compModifier(tc.comps, tc.cfg) - - diff := cmp.Diff(tc.wantErr, err, cmpopts.EquateErrors()) - if diff != "" { - t.Fatal(diff) - } - - // Cumbersome comparison of the source config encoded in protobuf, cmp panics protobufs comparison otherwise - if len(tc.wantComps) > 0 && len(tc.wantComps[0].Units) > 0 && comps[0].Units[0].Config != nil && comps[0].Units[0].Config.Source != nil { - m := comps[0].Units[0].Config.Source.AsMap() - wantM := tc.wantComps[0].Units[0].Config.Source.AsMap() - - diff = cmp.Diff(wantM, m) - if diff != "" { - t.Fatal(diff) - } - } - }) - } -} diff --git a/testing/integration/metrics_monitoring_test.go b/testing/integration/metrics_monitoring_test.go index 08eda658ce6..b4d885a19cb 100644 --- a/testing/integration/metrics_monitoring_test.go +++ b/testing/integration/metrics_monitoring_test.go @@ -8,6 +8,7 @@ package integration import ( "context" + "encoding/json" "fmt" "testing" "time" @@ -80,11 +81,14 @@ func (runner *MetricsRunner) SetupSuite() { } func (runner *MetricsRunner) TestBeatsMetrics() { + t := runner.T() + UnitOutputName := "default" ctx, cancel := context.WithTimeout(context.Background(), time.Minute*20) defer cancel() + agentStatus, err := runner.agentFixture.ExecStatus(ctx) - require.NoError(runner.T(), err) + require.NoError(t, err, "could not to get agent status") componentIds := []string{ fmt.Sprintf("system/metrics-%s", UnitOutputName), @@ -95,19 +99,36 @@ func (runner *MetricsRunner) TestBeatsMetrics() { "filestream-monitoring", } - require.Eventually(runner.T(), func() bool { + now := time.Now() + var query map[string]any + defer func() { + if t.Failed() { + bs, err := json.Marshal(query) + if err != nil { + // nothing we can do, just log the map + t.Errorf("executed at %s: %v", + now.Format(time.RFC3339Nano), query) + return + } + t.Errorf("executed at %s: query: %s", + now.Format(time.RFC3339Nano), string(bs)) + } + }() + + t.Logf("starting to ES for metrics at %s", now.Format(time.RFC3339Nano)) + require.Eventually(t, func() bool { for _, cid := range componentIds { - query := genESQuery(agentStatus.Info.ID, cid) + query = genESQuery(agentStatus.Info.ID, cid) + now = time.Now() res, err := estools.PerformQueryForRawQuery(ctx, query, "metrics-elastic_agent*", runner.info.ESClient) - require.NoError(runner.T(), err) - runner.T().Logf("Fetched metrics for %s, got %d hits", cid, res.Hits.Total.Value) + require.NoError(t, err) + t.Logf("Fetched metrics for %s, got %d hits", cid, res.Hits.Total.Value) if res.Hits.Total.Value < 1 { return false } - } return true - }, time.Minute*10, time.Second*10, "could not fetch metrics for all known beats in default install: %v", componentIds) + }, time.Minute*10, time.Second*10, "could not fetch metrics for all known components in default install: %v", componentIds) } func genESQuery(agentID string, componentID string) map[string]interface{} {