From 2b6e38f01701c18239d5ef489b819450baae6203 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Thu, 19 Sep 2024 12:18:18 +0200 Subject: [PATCH 01/13] decrypt client mTLS certificate key for Elastic Defend It adds EndpointTLSComponentModifier which will check if the client certificate key is encrypted, if so, it'll decrypt the key and pass it decrypted to endpoint (Elastic Defend) --- ...ected-client-certificate-key-for-mTLS.yaml | 32 ++ internal/pkg/agent/application/application.go | 4 +- .../endpoint_component_modifier.go | 212 ++++++++ .../endpoint_component_modifier_test.go | 483 ++++++++++++++++++ .../endpoint_signed_component_modifier.go | 54 -- ...endpoint_signed_component_modifier_test.go | 137 ----- 6 files changed, 730 insertions(+), 192 deletions(-) create mode 100644 changelog/fragments/1726735182-Elastic-Defend-accepts-passphrase-protected-client-certificate-key-for-mTLS.yaml create mode 100644 internal/pkg/agent/application/endpoint_component_modifier.go create mode 100644 internal/pkg/agent/application/endpoint_component_modifier_test.go delete mode 100644 internal/pkg/agent/application/endpoint_signed_component_modifier.go delete mode 100644 internal/pkg/agent/application/endpoint_signed_component_modifier_test.go diff --git a/changelog/fragments/1726735182-Elastic-Defend-accepts-passphrase-protected-client-certificate-key-for-mTLS.yaml b/changelog/fragments/1726735182-Elastic-Defend-accepts-passphrase-protected-client-certificate-key-for-mTLS.yaml new file mode 100644 index 00000000000..36b7579892c --- /dev/null +++ b/changelog/fragments/1726735182-Elastic-Defend-accepts-passphrase-protected-client-certificate-key-for-mTLS.yaml @@ -0,0 +1,32 @@ +# Kind can be one of: +# - breaking-change: a change to previously-documented behavior +# - deprecation: functionality that is being removed in a later release +# - bug-fix: fixes a problem in a previous version +# - enhancement: extends functionality but does not break or fix existing behavior +# - feature: new functionality +# - known-issue: problems that we are aware of in a given version +# - security: impacts on the security of a product or a user’s deployment. +# - upgrade: important information for someone upgrading from a prior version +# - other: does not fit into any of the other categories +kind: feature + +# Change summary; a 80ish characters long description of the change. +summary: Elastic Defend accepts passphrase protected client certificate key for mTLS + +# Long description; in case the summary is not enough to describe the change +# this field accommodate a description without length limits. +# NOTE: This field will be rendered only for breaking-change and known-issue kinds at the moment. +#description: + +# Affected component; a word indicating the component this changeset affects. +#component: + +# PR URL; optional; the PR number that added the changeset. +# If not present is automatically filled by the tooling finding the PR where this changelog fragment has been added. +# NOTE: the tooling supports backports, so it's able to fill the original PR number instead of the backport PR number. +# Please provide it if you are adding a fragment for a different PR. +#pr: https://github.com/owner/repo/1234 + +# Issue URL; optional; the GitHub issue related to this changeset (either closes or is part of). +# If not present is automatically filled by the tooling with the issue linked to the PR number. +issue: https://github.com/elastic/elastic-agent/issues/5490 diff --git a/internal/pkg/agent/application/application.go b/internal/pkg/agent/application/application.go index c9d50e4dd3a..eb7fbe1ab3f 100644 --- a/internal/pkg/agent/application/application.go +++ b/internal/pkg/agent/application/application.go @@ -172,9 +172,11 @@ func New( log.Info("Parsed configuration and determined agent is managed by Fleet") composableManaged = true - compModifiers = append(compModifiers, FleetServerComponentModifier(cfg.Fleet.Server), + compModifiers = append(compModifiers, + FleetServerComponentModifier(cfg.Fleet.Server), InjectFleetConfigComponentModifier(cfg.Fleet, agentInfo), EndpointSignedComponentModifier(), + EndpointTLSComponentModifier(log), InjectProxyEndpointModifier(), ) diff --git a/internal/pkg/agent/application/endpoint_component_modifier.go b/internal/pkg/agent/application/endpoint_component_modifier.go new file mode 100644 index 00000000000..f1f3a4b6414 --- /dev/null +++ b/internal/pkg/agent/application/endpoint_component_modifier.go @@ -0,0 +1,212 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package application + +import ( + "encoding/pem" + "errors" + "fmt" + "os" + + "github.com/elastic/elastic-agent-client/v7/pkg/client" + "github.com/elastic/elastic-agent-libs/transport/tlscommon" + "github.com/elastic/elastic-agent/internal/pkg/agent/application/coordinator" + "github.com/elastic/elastic-agent/pkg/component" + "github.com/elastic/elastic-agent/pkg/core/logger" +) + +// EndpointSignedComponentModifier copies "signed" properties to the top level "signed" for the endpoint input. +// Enpoint team want to be able to validate the signature and parse the signed configuration (not trust the agent). +// Endpoint uses uninstall_token_hash in order to verify uninstall command token +// and signing_key in order validate the action signature. +// Example: +// +// { +// .... +// "signed": { +// "data": "eyJpZCI6ImFhZWM4OTYwLWJiYjAtMTFlZC1hYzBkLTVmNjI0YTQxZjM4OCIsImFnZW50Ijp7InByb3RlY3Rpb24iOnsiZW5hYmxlZCI6dHJ1ZSwidW5pbnN0YWxsX3Rva2VuX2hhc2giOiIiLCJzaWduaW5nX2tleSI6Ik1Ga3dFd1lIS29aSXpqMENBUVlJS29aSXpqMERBUWNEUWdBRW1tckhDSTdtZ2tuZUJlYVJkc2VkQXZBU2l0UHRLbnpPdUlzeHZJRWdGTkFLVlg3MWpRTTVmalo1eUdsSDB0TmJuR2JrU2pVM0VEVUZsOWllQ1J0ME5nPT0ifX19", +// "signature": "MEUCIQCWoScyJW0dejHFxXBTEcSCOZiBHRVMjuJRPwFCwOdA1QIgKrtKUBzkvVeljRtJyMXfD8zIvWjrMzqhSkgjNESPW5E=" +// }, +// "revision": 1, +// "type": "endpoint" +// } +func EndpointSignedComponentModifier() coordinator.ComponentsModifier { + return func(comps []component.Component, cfg map[string]interface{}) ([]component.Component, error) { + const signedKey = "signed" + + compIdx, unitIdx, ok := findEndpointUnit(comps, client.UnitTypeInput) + if !ok { + return comps, nil + } + + unit := comps[compIdx].Units[unitIdx] + unitCfgMap := unit.Config.Source.AsMap() + if signed, ok := cfg[signedKey]; ok { + unitCfgMap[signedKey] = signed + } + + unitCfg, err := component.ExpectedConfig(unitCfgMap) + if err != nil { + return nil, err + } + + unit.Config = unitCfg + comps[compIdx].Units[unitIdx] = unit + + return comps, nil + } +} + +// EndpointTLSComponentModifier decrypts the client TLS certificate key if it's +// passphrase-protected. It replaces the content of 'fleet.ssl.key' +// and 'certificate' with theirs decrypted version and removes +// 'key_passphrase_path'. +// It does so, ONLY for the client TLS configuration for mTLS used with +// fleet-server. +func EndpointTLSComponentModifier(log *logger.Logger) coordinator.ComponentsModifier { + return func(comps []component.Component, cfg map[string]interface{}) ([]component.Component, error) { + compIdx, unitIdx, ok := findEndpointUnit(comps, client.UnitTypeInput) + if !ok { + // endpoint not present, nothing to do + return comps, nil + } + + unit := comps[compIdx].Units[unitIdx] + unitCfgMap := unit.Config.Source.AsMap() + + // ensure the following config exists: + // fleet.ssl: + // key_passphrase_path + // certificate + // key + fleetNode, ok := unitCfgMap["fleet"] + if !ok { + // if 'fleet' isn't, present nothing to do + return comps, nil + } + fleetMap, ok := fleetNode.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("EndpointTLSComponentModifier: 'fleet' node isn't a map, it is: %T", fleetNode) + } + + sslNode, ok := fleetMap["ssl"] + if !ok { + // 'ssl' node not present isn't an issue + return comps, nil + } + sslMap, ok := sslNode.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("EndpointTLSComponentModifier: 'ssl' node isn't a map, it is: %T", sslNode) + } + + keyPassPathI, ok := sslMap["key_passphrase_path"] + if !ok { + // if no key_passphrase_path, nothing to decrypt + return comps, nil + } + keyPassPathStr, ok := keyPassPathI.(string) + if !ok { + return nil, errors.New("EndpointTLSComponentModifier: 'key_passphrase_path' isn't a string") + } + if keyPassPathStr == "" { + // the key shouldn't be empty, but if it's, nothing to decrypt + return comps, nil + } + + keyI, ok := sslMap["key"] + if !ok { + // if there is a key_passphrase_path, the key must be present + return nil, errors.New("EndpointTLSComponentModifier: 'key_passphrase_path' present, but 'key' isn't present") + } + keyStr, ok := keyI.(string) + if !ok { + return nil, fmt.Errorf("EndpointTLSComponentModifier: 'key' isn't a string, it's %T", keyI) + } + + certI, ok := sslMap["certificate"] + if !ok { + // if there is a key_passphrase_path, the certificate must be present + return nil, errors.New("EndpointTLSComponentModifier: 'key_passphrase_path' present, but 'certificate' isn't present") + } + certStr, ok := certI.(string) + if !ok { + return nil, errors.New("EndpointTLSComponentModifier: 'certificate' isn't a string") + } + + // all SSL config exists and the certificate key is passphrase protected, + // now decrypt the key + + pass, err := os.ReadFile(keyPassPathStr) + if err != nil { + return nil, fmt.Errorf("EndpointTLSComponentModifier: failed to read client certificate passphrase file: %w", err) + } + + // we don't really support encrypted certificates, but it's how + // tlscommon.LoadCertificate does. Thus, let's keep the same behaviour. + // Also, tlscommon.LoadCertificate 'loses' the type of the private key. + // It stores they private key as an interface and there is no way to + // retrieve the type os the private key without a type assertion. + // Therefore, instead of manually checking the type, which would mean + // to check for all supported private key types and keep it up to date, + // better to load the certificate and its key directly from the PEM file. + cert, err := tlscommon.ReadPEMFile(log, + certStr, string(pass)) + if err != nil { + return nil, fmt.Errorf("EndpointTLSComponentModifier: unable to load TLS certificate: %w", err) + } + key, err := tlscommon.ReadPEMFile(log, + keyStr, + string(pass)) + if err != nil { + return nil, fmt.Errorf("EndpointTLSComponentModifier: unable to load TLS certificate key: %w", err) + } + + // tlscommon.ReadPEMFile only removes the 'DEK-Info' header, not the + // 'Proc-Type', so remove it now. Create a pem.Block to avoid editing + // the PEM data manually: + keyBlock, _ := pem.Decode(key) + delete(keyBlock.Headers, "Proc-Type") + key = pem.EncodeToMemory(keyBlock) + + // remove 'key_passphrase_path' as the certificate key isn't encrypted + // anymore. + delete(sslMap, "key_passphrase_path") + + // update the certificate and its key with their decrypted version. + sslMap["certificate"] = string(cert) + sslMap["key"] = string(key) + + unitCfg, err := component.ExpectedConfig(unitCfgMap) + if err != nil { + return nil, fmt.Errorf("EndpointTLSComponentModifier: could not covert modified config to expected config: %w", err) + } + + unit.Config = unitCfg + comps[compIdx].Units[unitIdx] = unit + + return comps, nil + } +} + +// findEndpointUnit finds the endpoint component and its unit of type 'unitType'. +// It returns the component and unit index and true if found, if not, it returns +// 0, 0, false. +func findEndpointUnit(comps []component.Component, unitType client.UnitType) (int, int, bool) { + // find the endpoint component + for compIdx, comp := range comps { + if comp.InputSpec != nil && comp.InputSpec.InputType != endpoint { + continue + } + + for unitIdx, unit := range comp.Units { + if unit.Type != unitType { + continue + } + + return compIdx, unitIdx, true + } + } + return 0, 0, false +} diff --git a/internal/pkg/agent/application/endpoint_component_modifier_test.go b/internal/pkg/agent/application/endpoint_component_modifier_test.go new file mode 100644 index 00000000000..6ce66e97602 --- /dev/null +++ b/internal/pkg/agent/application/endpoint_component_modifier_test.go @@ -0,0 +1,483 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package application + +import ( + "crypto/rand" + "crypto/x509" + "encoding/pem" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/elastic/elastic-agent-client/v7/pkg/client" + "github.com/elastic/elastic-agent-client/v7/pkg/proto" + "github.com/elastic/elastic-agent-libs/testing/certutil" + "github.com/elastic/elastic-agent/internal/pkg/agent/application/coordinator" + "github.com/elastic/elastic-agent/pkg/core/logger/loggertest" + + "github.com/elastic/elastic-agent/pkg/component" + + "google.golang.org/protobuf/types/known/structpb" +) + +func TestEndpointComponentModifier(t *testing.T) { + log, obs := loggertest.New("TestEndpointSignedComponentModifier") + defer func() { + if !t.Failed() { + return + } + + loggertest.PrintObservedLogs(obs.TakeAll(), t.Log) + }() + passphrase := "secure_passphrase" + _, _, pair, err := certutil.NewRootCA() + require.NoError(t, err, "could not create TLS certificate") + agentChildDERKey, _ := pem.Decode(pair.Key) + require.NoError(t, err, "could not create tls.Certificates from child certificate") + + encPem, err := x509.EncryptPEMBlock( //nolint:staticcheck // we need to drop support for this, but while we don't, it needs to be tested. + rand.Reader, + "EC PRIVATE KEY", + agentChildDERKey.Bytes, + []byte(passphrase), + x509.PEMCipherAES128) + require.NoError(t, err, "failed encrypting agent child certificate key block") + + certKeyEnc := pem.EncodeToMemory(encPem) + + // save to disk + tmpDir := t.TempDir() + certPath := filepath.Join(tmpDir, "cert.pem") + certKeyPath := filepath.Join(tmpDir, "key.pem") + certKeyPassPath := filepath.Join(tmpDir, "key_pass.pem") + + err = os.WriteFile(certPath, pair.Cert, 0400) + require.NoError(t, err, "could write certificate key") + err = os.WriteFile(certKeyPath, certKeyEnc, 0400) + require.NoError(t, err, "could write certificate key") + err = os.WriteFile(certKeyPassPath, []byte(passphrase), 0400) + require.NoError(t, err, "could write certificate key passphrase") + + tests := map[string][]struct { + name string + compModifier coordinator.ComponentsModifier + comps []component.Component + cfg map[string]interface{} + wantComps []component.Component + wantErr func(*testing.T, error) + }{ + "EndpointSignedComponentModifier": { + { + name: "nil", + compModifier: EndpointSignedComponentModifier(), + }, + { + name: "non endpoint", + compModifier: EndpointSignedComponentModifier(), + comps: []component.Component{ + { + ID: "asdfasd", + InputSpec: &component.InputRuntimeSpec{ + InputType: "osquery", + }, + Units: []component.Unit{ + { + ID: "34534", + Type: client.UnitTypeInput, + }, + }, + }, + }, + wantComps: []component.Component{ + { + ID: "asdfasd", + InputSpec: &component.InputRuntimeSpec{ + InputType: "osquery", + }, + Units: []component.Unit{ + { + ID: "34534", + Type: client.UnitTypeInput, + }, + }, + }, + }, + }, + { + name: "endpoint", + compModifier: EndpointSignedComponentModifier(), + comps: []component.Component{ + { + ID: "asdfasd", + InputSpec: &component.InputRuntimeSpec{ + InputType: "endpoint", + }, + Units: []component.Unit{ + { + ID: "34534", + Type: client.UnitTypeInput, + Config: &proto.UnitExpectedConfig{ + Type: "endpoint", + Source: &structpb.Struct{}, + }, + }, + }, + }, + }, + cfg: map[string]interface{}{ + "signed": map[string]interface{}{ + "data": "eyJpZCI6ImFhZWM4OTYwLWJiYjAtMTFlZC1hYzBkLTVmNjI0YTQxZjM4OCIsImFnZW50Ijp7InByb3RlY3Rpb24iOnsiZW5hYmxlZCI6dHJ1ZSwidW5pbnN0YWxsX3Rva2VuX2hhc2giOiIiLCJzaWduaW5nX2tleSI6Ik1Ga3dFd1lIS29aSXpqMENBUVlJS29aSXpqMERBUWNEUWdBRW1tckhDSTdtZ2tuZUJlYVJkc2VkQXZBU2l0UHRLbnpPdUlzeHZJRWdGTkFLVlg3MWpRTTVmalo1eUdsSDB0TmJuR2JrU2pVM0VEVUZsOWllQ1J0ME5nPT0ifX19", + "signature": "MEUCIQCWoScyJW0dejHFxXBTEcSCOZiBHRVMjuJRPwFCwOdA1QIgKrtKUBzkvVeljRtJyMXfD8zIvWjrMzqhSkgjNESPW5E=", + }, + }, + wantComps: []component.Component{ + { + ID: "asdfasd", + InputSpec: &component.InputRuntimeSpec{ + InputType: "endpoint", + }, + Units: []component.Unit{ + { + ID: "34534", + Type: client.UnitTypeInput, + Config: &proto.UnitExpectedConfig{ + Source: func() *structpb.Struct { + var source structpb.Struct + err = source.UnmarshalJSON([]byte(`{"signed":{"data":"eyJpZCI6ImFhZWM4OTYwLWJiYjAtMTFlZC1hYzBkLTVmNjI0YTQxZjM4OCIsImFnZW50Ijp7InByb3RlY3Rpb24iOnsiZW5hYmxlZCI6dHJ1ZSwidW5pbnN0YWxsX3Rva2VuX2hhc2giOiIiLCJzaWduaW5nX2tleSI6Ik1Ga3dFd1lIS29aSXpqMENBUVlJS29aSXpqMERBUWNEUWdBRW1tckhDSTdtZ2tuZUJlYVJkc2VkQXZBU2l0UHRLbnpPdUlzeHZJRWdGTkFLVlg3MWpRTTVmalo1eUdsSDB0TmJuR2JrU2pVM0VEVUZsOWllQ1J0ME5nPT0ifX19", "signature":"MEUCIQCWoScyJW0dejHFxXBTEcSCOZiBHRVMjuJRPwFCwOdA1QIgKrtKUBzkvVeljRtJyMXfD8zIvWjrMzqhSkgjNESPW5E="}}`)) + require.NoError(t, err, "could not create want component source config") + return &source + }(), + }, + }, + }, + }, + }, + }, + }, + "EndpointTLSComponentModifier": { + { + name: "nil", + compModifier: EndpointSignedComponentModifier(), + }, + { + name: "non endpoint", + compModifier: EndpointSignedComponentModifier(), + comps: makeComponent(t, "{}"), + wantComps: makeComponent(t, "{}"), + }, + + { + name: "endpoint-no-fleet", + compModifier: EndpointTLSComponentModifier(log), + comps: makeComponent(t, `{}`), + cfg: map[string]interface{}{ + "fleet": map[string]interface{}{}, + }, + wantComps: makeComponent(t, `{}`), + }, + { + name: "endpoint-no-fleet-wrong-type", + compModifier: EndpointTLSComponentModifier(log), + comps: makeComponent(t, `{"fleet": 42}`), + cfg: map[string]interface{}{ + "fleet": 1, + }, + wantComps: nil, + wantErr: func(t *testing.T, err error) { + assert.ErrorContains(t, err, "'fleet' node isn't a map") + }, + }, + { + name: "endpoint-no-fleet.ssl", + compModifier: EndpointTLSComponentModifier(log), + comps: makeComponent(t, `{"fleet": {}}`), + cfg: map[string]interface{}{ + "fleet": map[string]interface{}{}, + }, + wantComps: makeComponent(t, `{"fleet": {}}`), + }, + { + name: "endpoint-wrong-fleet.ssl", + compModifier: EndpointTLSComponentModifier(log), + comps: makeComponent(t, `{"fleet": {"ssl": 42}}`), + cfg: map[string]interface{}{ + "fleet": map[string]interface{}{}, + }, + wantComps: nil, + wantErr: func(t *testing.T, err error) { + assert.ErrorContains(t, err, "'ssl' node isn't a map") + }, + }, + { + name: "endpoint-wrong-fleet.ssl.key_passphrase_path", + compModifier: EndpointTLSComponentModifier(log), + comps: makeComponent(t, ` + {"fleet": {"ssl": + {"key_passphrase_path": 42}}}`), + cfg: map[string]interface{}{ + "fleet": map[string]interface{}{}, + }, + wantComps: nil, + wantErr: func(t *testing.T, err error) { + assert.ErrorContains(t, err, "'key_passphrase_path' isn't a string") + }, + }, + { + name: "endpoint-wrong-fleet.ssl.key", + compModifier: EndpointTLSComponentModifier(log), + comps: makeComponent(t, ` +{"fleet": {"ssl": { + "key_passphrase_path": "/path/to/passphrase", + "key": 42}}}`), + cfg: map[string]interface{}{ + "fleet": map[string]interface{}{}, + }, + wantComps: nil, + wantErr: func(t *testing.T, err error) { + assert.ErrorContains(t, err, "'key' isn't a string") + }, + }, + { + name: "endpoint-wrong-fleet.ssl.certificate", + compModifier: EndpointTLSComponentModifier(log), + comps: makeComponent(t, ` + {"fleet": {"ssl": { + "key_passphrase_path": "/path/to/passphrase", + "key": "", + "certificate": 42}}}`), + cfg: map[string]interface{}{ + "fleet": map[string]interface{}{}, + }, + wantComps: nil, + wantErr: func(t *testing.T, err error) { + assert.ErrorContains(t, err, "'certificate' isn't a string") + }, + }, + + { + name: "endpoint-mTLS-passphrase", + compModifier: EndpointTLSComponentModifier(log), + comps: makeComponent(t, fmt.Sprintf(`{ + "fleet": { + "ssl": { + "certificate": %q, + "key": %q, + "key_passphrase_path": %q + } + } + }`, certPath, certKeyPath, certKeyPassPath)), + cfg: map[string]interface{}{ + "fleet": map[string]interface{}{ + "ssl": map[string]interface{}{ + "certificate": certPath, + "key": certKeyPath, + "key_passphrase_path": certKeyPassPath, + }, + }, + }, + wantComps: makeComponent(t, fmt.Sprintf(`{ + "fleet": { + "ssl": { + "certificate": %q, + "key": %q + } + } + }`, pair.Cert, pair.Key)), + }, + { + name: "endpoint-mTLS-passphrase-no-key", + compModifier: EndpointTLSComponentModifier(log), + comps: makeComponent(t, fmt.Sprintf(`{ + "fleet": { + "ssl": { + "certificate": %q, + "key_passphrase_path": %q + } + } + }`, certPath, certKeyPassPath)), + cfg: map[string]interface{}{ + "fleet": map[string]interface{}{ + "ssl": map[string]interface{}{ + "certificate": certPath, + "key_passphrase_path": certKeyPassPath, + }, + }, + }, + wantComps: nil, + wantErr: func(t *testing.T, err error) { + assert.ErrorContains(t, err, "'key' isn't present") + }, + }, + { + name: "endpoint-mTLS-passphrase-no-certificate", + compModifier: EndpointTLSComponentModifier(log), + comps: makeComponent(t, `{ + "fleet": { + "ssl": { + "key": "/path/to/key", + "key_passphrase_path": "/path/to/key_passphrase_path" + } + } + }`), + cfg: map[string]interface{}{ + "fleet": map[string]interface{}{ + "ssl": map[string]interface{}{ + "key": "/path/to/cert", + "key_passphrase_path": "/path/to/key_passphrase_path", + }, + }, + }, + wantComps: nil, + wantErr: func(t *testing.T, err error) { + assert.ErrorContains(t, err, "'certificate' isn't present") + }, + }, + { + name: "endpoint-mTLS-no-passphrase", + compModifier: EndpointTLSComponentModifier(log), + comps: makeComponent(t, `{ + "fleet": { + "ssl": { + "certificate": "/path/to/cert", + "key": "/path/to/key" + } + } + }`), + cfg: map[string]interface{}{ + "fleet": map[string]interface{}{ + "ssl": map[string]interface{}{ + "certificate": "/path/to/cert", + "key": "/path/to/key", + }, + }, + }, + wantComps: makeComponent(t, `{ + "fleet": { + "ssl": { + "certificate": "/path/to/cert", + "key": "/path/to/key" + } + } + }`), + }, + { + name: "endpoint-mTLS-empty-passphrase", + compModifier: EndpointTLSComponentModifier(log), + comps: makeComponent(t, `{ + "fleet": { + "ssl": { + "key_passphrase_path": "", + "certificate": "/path/to/cert", + "key": "/path/to/key" + } + } + }`), + cfg: map[string]interface{}{ + "fleet": map[string]interface{}{ + "ssl": map[string]interface{}{ + "key_passphrase_path": "", + "certificate": "/path/to/cert", + "key": "/path/to/key", + }, + }, + }, + wantComps: makeComponent(t, `{ + "fleet": { + "ssl": { + "key_passphrase_path": "", + "certificate": "/path/to/cert", + "key": "/path/to/key" + } + } + }`), + }, + { + name: "endpoint-TLS", + compModifier: EndpointTLSComponentModifier(log), + comps: makeComponent(t, `{ + "fleet": { + "ssl": { + "certificate_authorities": ["/path/to/ca1", "/path/to/ca2"] + } + } + }`), + cfg: map[string]interface{}{ + "fleet": map[string]interface{}{ + "ssl": map[string]interface{}{ + "certificate_authorities": []string{"/path/to/ca1", "/path/to/ca2"}, + }, + }, + }, + wantComps: makeComponent(t, `{ + "fleet": { + "ssl": { + "certificate_authorities": ["/path/to/ca1", "/path/to/ca2"] + } + } + }`), + }, + }, + } + + for name, tcs := range tests { + t.Run(name, func(t *testing.T) { + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + comps, err := tc.compModifier(tc.comps, tc.cfg) + + if tc.wantErr != nil { + tc.wantErr(t, err) + } else { + assert.NoError(t, err) + } + + // Cumbersome comparison of the source config encoded in protobuf, + // cmp panics protobufs comparison otherwise. + if len(tc.wantComps) > 0 && + len(tc.wantComps[0].Units) > 0 && + comps[0].Units[0].Config != nil && + comps[0].Units[0].Config.Source != nil { + + unitCgf := comps[0].Units[0].Config.Source.AsMap() + wantUnitCfg := tc.wantComps[0].Units[0].Config.Source.AsMap() + + assert.Equal(t, wantUnitCfg, unitCgf, "unit config do not match") + } + }) + } + }) + } +} + +func makeComponent(t *testing.T, sourceCfg string) []component.Component { + return []component.Component{ + { + ID: "ClientCertKey", + InputSpec: &component.InputRuntimeSpec{ + InputType: "endpoint", + }, + Units: []component.Unit{ + { + ID: "34534", + Type: client.UnitTypeInput, + Config: &proto.UnitExpectedConfig{ + Type: "endpoint", + Source: func() *structpb.Struct { + var source structpb.Struct + err := source.UnmarshalJSON([]byte(sourceCfg)) + require.NoError(t, err, "could not create component source config") + return &source + }(), + }, + }, + }, + }, + } +} diff --git a/internal/pkg/agent/application/endpoint_signed_component_modifier.go b/internal/pkg/agent/application/endpoint_signed_component_modifier.go deleted file mode 100644 index 39aac35604a..00000000000 --- a/internal/pkg/agent/application/endpoint_signed_component_modifier.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License 2.0; -// you may not use this file except in compliance with the Elastic License 2.0. - -package application - -import ( - "github.com/elastic/elastic-agent-client/v7/pkg/client" - "github.com/elastic/elastic-agent/internal/pkg/agent/application/coordinator" - "github.com/elastic/elastic-agent/pkg/component" -) - -// EndpointSignedComponentModifier copies "signed" properties to the top level "signed" for the endpoint input. -// Enpoint team want to be able to validate the signature and parse the signed configuration (not trust the agent). -// Endpoint uses uninstall_token_hash in order to verify uninstall command token -// and signing_key in order validate the action signature. -// Example: -// -// { -// .... -// "signed": { -// "data": "eyJpZCI6ImFhZWM4OTYwLWJiYjAtMTFlZC1hYzBkLTVmNjI0YTQxZjM4OCIsImFnZW50Ijp7InByb3RlY3Rpb24iOnsiZW5hYmxlZCI6dHJ1ZSwidW5pbnN0YWxsX3Rva2VuX2hhc2giOiIiLCJzaWduaW5nX2tleSI6Ik1Ga3dFd1lIS29aSXpqMENBUVlJS29aSXpqMERBUWNEUWdBRW1tckhDSTdtZ2tuZUJlYVJkc2VkQXZBU2l0UHRLbnpPdUlzeHZJRWdGTkFLVlg3MWpRTTVmalo1eUdsSDB0TmJuR2JrU2pVM0VEVUZsOWllQ1J0ME5nPT0ifX19", -// "signature": "MEUCIQCWoScyJW0dejHFxXBTEcSCOZiBHRVMjuJRPwFCwOdA1QIgKrtKUBzkvVeljRtJyMXfD8zIvWjrMzqhSkgjNESPW5E=" -// }, -// "revision": 1, -// "type": "endpoint" -// } -func EndpointSignedComponentModifier() coordinator.ComponentsModifier { - return func(comps []component.Component, cfg map[string]interface{}) ([]component.Component, error) { - const signedKey = "signed" - for i, comp := range comps { - if comp.InputSpec != nil && (comp.InputSpec.InputType == endpoint) { - for j, unit := range comp.Units { - if unit.Type == client.UnitTypeInput && (unit.Config.Type == endpoint) { - unitCfgMap := unit.Config.Source.AsMap() - if signed, ok := cfg[signedKey]; ok { - unitCfgMap[signedKey] = signed - } - - unitCfg, err := component.ExpectedConfig(unitCfgMap) - if err != nil { - return nil, err - } - - unit.Config = unitCfg - comp.Units[j] = unit - } - } - } - comps[i] = comp - } - return comps, nil - } -} diff --git a/internal/pkg/agent/application/endpoint_signed_component_modifier_test.go b/internal/pkg/agent/application/endpoint_signed_component_modifier_test.go deleted file mode 100644 index fcc16321d5c..00000000000 --- a/internal/pkg/agent/application/endpoint_signed_component_modifier_test.go +++ /dev/null @@ -1,137 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License 2.0; -// you may not use this file except in compliance with the Elastic License 2.0. - -package application - -import ( - "testing" - - "github.com/elastic/elastic-agent-client/v7/pkg/client" - "github.com/elastic/elastic-agent-client/v7/pkg/proto" - - "github.com/elastic/elastic-agent/pkg/component" - - "github.com/google/go-cmp/cmp" - "github.com/google/go-cmp/cmp/cmpopts" - "google.golang.org/protobuf/types/known/structpb" -) - -func TestEndpointSignedComponentModifier(t *testing.T) { - compModifier := EndpointSignedComponentModifier() - - tests := []struct { - name string - comps []component.Component - cfg map[string]interface{} - wantComps []component.Component - wantErr error - }{ - { - name: "nil", - }, - { - name: "non endpoint", - comps: []component.Component{ - { - ID: "asdfasd", - InputSpec: &component.InputRuntimeSpec{ - InputType: "osquery", - }, - Units: []component.Unit{ - { - ID: "34534", - Type: client.UnitTypeInput, - }, - }, - }, - }, - wantComps: []component.Component{ - { - ID: "asdfasd", - InputSpec: &component.InputRuntimeSpec{ - InputType: "osquery", - }, - Units: []component.Unit{ - { - ID: "34534", - Type: client.UnitTypeInput, - }, - }, - }, - }, - }, - { - name: "endpoint", - comps: []component.Component{ - { - ID: "asdfasd", - InputSpec: &component.InputRuntimeSpec{ - InputType: "endpoint", - }, - Units: []component.Unit{ - { - ID: "34534", - Type: client.UnitTypeInput, - Config: &proto.UnitExpectedConfig{ - Type: "endpoint", - Source: &structpb.Struct{}, - }, - }, - }, - }, - }, - cfg: map[string]interface{}{ - "signed": map[string]interface{}{ - "data": "eyJpZCI6ImFhZWM4OTYwLWJiYjAtMTFlZC1hYzBkLTVmNjI0YTQxZjM4OCIsImFnZW50Ijp7InByb3RlY3Rpb24iOnsiZW5hYmxlZCI6dHJ1ZSwidW5pbnN0YWxsX3Rva2VuX2hhc2giOiIiLCJzaWduaW5nX2tleSI6Ik1Ga3dFd1lIS29aSXpqMENBUVlJS29aSXpqMERBUWNEUWdBRW1tckhDSTdtZ2tuZUJlYVJkc2VkQXZBU2l0UHRLbnpPdUlzeHZJRWdGTkFLVlg3MWpRTTVmalo1eUdsSDB0TmJuR2JrU2pVM0VEVUZsOWllQ1J0ME5nPT0ifX19", - "signature": "MEUCIQCWoScyJW0dejHFxXBTEcSCOZiBHRVMjuJRPwFCwOdA1QIgKrtKUBzkvVeljRtJyMXfD8zIvWjrMzqhSkgjNESPW5E=", - }, - }, - wantComps: []component.Component{ - { - ID: "asdfasd", - InputSpec: &component.InputRuntimeSpec{ - InputType: "endpoint", - }, - Units: []component.Unit{ - { - ID: "34534", - Type: client.UnitTypeInput, - Config: &proto.UnitExpectedConfig{ - Source: func() *structpb.Struct { - var source structpb.Struct - err := source.UnmarshalJSON([]byte(`{"signed":{"data":"eyJpZCI6ImFhZWM4OTYwLWJiYjAtMTFlZC1hYzBkLTVmNjI0YTQxZjM4OCIsImFnZW50Ijp7InByb3RlY3Rpb24iOnsiZW5hYmxlZCI6dHJ1ZSwidW5pbnN0YWxsX3Rva2VuX2hhc2giOiIiLCJzaWduaW5nX2tleSI6Ik1Ga3dFd1lIS29aSXpqMENBUVlJS29aSXpqMERBUWNEUWdBRW1tckhDSTdtZ2tuZUJlYVJkc2VkQXZBU2l0UHRLbnpPdUlzeHZJRWdGTkFLVlg3MWpRTTVmalo1eUdsSDB0TmJuR2JrU2pVM0VEVUZsOWllQ1J0ME5nPT0ifX19", "signature":"MEUCIQCWoScyJW0dejHFxXBTEcSCOZiBHRVMjuJRPwFCwOdA1QIgKrtKUBzkvVeljRtJyMXfD8zIvWjrMzqhSkgjNESPW5E="}}`)) - if err != nil { - t.Fatal(err) - } - return &source - }(), - }, - }, - }, - }, - }, - }, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - comps, err := compModifier(tc.comps, tc.cfg) - - diff := cmp.Diff(tc.wantErr, err, cmpopts.EquateErrors()) - if diff != "" { - t.Fatal(diff) - } - - // Cumbersome comparison of the source config encoded in protobuf, cmp panics protobufs comparison otherwise - if len(tc.wantComps) > 0 && len(tc.wantComps[0].Units) > 0 && comps[0].Units[0].Config != nil && comps[0].Units[0].Config.Source != nil { - m := comps[0].Units[0].Config.Source.AsMap() - wantM := tc.wantComps[0].Units[0].Config.Source.AsMap() - - diff = cmp.Diff(wantM, m) - if diff != "" { - t.Fatal(diff) - } - } - }) - } -} From ebdc753a2062df8104a3c07da454330af9d85e58 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Mon, 30 Sep 2024 18:16:53 +0200 Subject: [PATCH 02/13] add cache and tests --- .../endpoint_component_modifier.go | 129 ++++++++---- .../endpoint_component_modifier_test.go | 196 ++++++++++++++---- 2 files changed, 248 insertions(+), 77 deletions(-) diff --git a/internal/pkg/agent/application/endpoint_component_modifier.go b/internal/pkg/agent/application/endpoint_component_modifier.go index f1f3a4b6414..c6b35d5f4f0 100644 --- a/internal/pkg/agent/application/endpoint_component_modifier.go +++ b/internal/pkg/agent/application/endpoint_component_modifier.go @@ -9,6 +9,7 @@ import ( "errors" "fmt" "os" + "sync" "github.com/elastic/elastic-agent-client/v7/pkg/client" "github.com/elastic/elastic-agent-libs/transport/tlscommon" @@ -17,6 +18,20 @@ import ( "github.com/elastic/elastic-agent/pkg/core/logger" ) +// tlsCache is used to cache the decrypted client certificate and key. +// In environments with a high rate of config changes, such as in Kubernetes +// using autodiscover, loading files and decrypting them might have a +// non-negligible overhead. +type tlsCache struct { + mu *sync.Mutex + + // PassphrasePath is used as the cache key + PassphrasePath string + + Certificate string + Key string +} + // EndpointSignedComponentModifier copies "signed" properties to the top level "signed" for the endpoint input. // Enpoint team want to be able to validate the signature and parse the signed configuration (not trust the agent). // Endpoint uses uninstall_token_hash in order to verify uninstall command token @@ -66,6 +81,10 @@ func EndpointSignedComponentModifier() coordinator.ComponentsModifier { // It does so, ONLY for the client TLS configuration for mTLS used with // fleet-server. func EndpointTLSComponentModifier(log *logger.Logger) coordinator.ComponentsModifier { + return newEndpointTLSComponentModifier(log, &tlsCache{mu: &sync.Mutex{}}) +} + +func newEndpointTLSComponentModifier(log *logger.Logger, cache *tlsCache) func(comps []component.Component, cfg map[string]interface{}) ([]component.Component, error) { return func(comps []component.Component, cfg map[string]interface{}) ([]component.Component, error) { compIdx, unitIdx, ok := findEndpointUnit(comps, client.UnitTypeInput) if !ok { @@ -106,12 +125,12 @@ func EndpointTLSComponentModifier(log *logger.Logger) coordinator.ComponentsModi // if no key_passphrase_path, nothing to decrypt return comps, nil } - keyPassPathStr, ok := keyPassPathI.(string) + keyPassPath, ok := keyPassPathI.(string) if !ok { return nil, errors.New("EndpointTLSComponentModifier: 'key_passphrase_path' isn't a string") } - if keyPassPathStr == "" { - // the key shouldn't be empty, but if it's, nothing to decrypt + if keyPassPath == "" { + // key_passphrase_path shouldn't be empty, but if it's, nothing to decrypt return comps, nil } @@ -120,7 +139,7 @@ func EndpointTLSComponentModifier(log *logger.Logger) coordinator.ComponentsModi // if there is a key_passphrase_path, the key must be present return nil, errors.New("EndpointTLSComponentModifier: 'key_passphrase_path' present, but 'key' isn't present") } - keyStr, ok := keyI.(string) + keyPath, ok := keyI.(string) if !ok { return nil, fmt.Errorf("EndpointTLSComponentModifier: 'key' isn't a string, it's %T", keyI) } @@ -130,53 +149,23 @@ func EndpointTLSComponentModifier(log *logger.Logger) coordinator.ComponentsModi // if there is a key_passphrase_path, the certificate must be present return nil, errors.New("EndpointTLSComponentModifier: 'key_passphrase_path' present, but 'certificate' isn't present") } - certStr, ok := certI.(string) + certPath, ok := certI.(string) if !ok { return nil, errors.New("EndpointTLSComponentModifier: 'certificate' isn't a string") } - // all SSL config exists and the certificate key is passphrase protected, - // now decrypt the key - - pass, err := os.ReadFile(keyPassPathStr) - if err != nil { - return nil, fmt.Errorf("EndpointTLSComponentModifier: failed to read client certificate passphrase file: %w", err) - } - - // we don't really support encrypted certificates, but it's how - // tlscommon.LoadCertificate does. Thus, let's keep the same behaviour. - // Also, tlscommon.LoadCertificate 'loses' the type of the private key. - // It stores they private key as an interface and there is no way to - // retrieve the type os the private key without a type assertion. - // Therefore, instead of manually checking the type, which would mean - // to check for all supported private key types and keep it up to date, - // better to load the certificate and its key directly from the PEM file. - cert, err := tlscommon.ReadPEMFile(log, - certStr, string(pass)) + cert, key, err := loadCertificatesWithCache(log, cache, keyPassPath, certPath, keyPath) if err != nil { - return nil, fmt.Errorf("EndpointTLSComponentModifier: unable to load TLS certificate: %w", err) - } - key, err := tlscommon.ReadPEMFile(log, - keyStr, - string(pass)) - if err != nil { - return nil, fmt.Errorf("EndpointTLSComponentModifier: unable to load TLS certificate key: %w", err) + return nil, err } - // tlscommon.ReadPEMFile only removes the 'DEK-Info' header, not the - // 'Proc-Type', so remove it now. Create a pem.Block to avoid editing - // the PEM data manually: - keyBlock, _ := pem.Decode(key) - delete(keyBlock.Headers, "Proc-Type") - key = pem.EncodeToMemory(keyBlock) - // remove 'key_passphrase_path' as the certificate key isn't encrypted // anymore. delete(sslMap, "key_passphrase_path") // update the certificate and its key with their decrypted version. - sslMap["certificate"] = string(cert) - sslMap["key"] = string(key) + sslMap["certificate"] = cert + sslMap["key"] = key unitCfg, err := component.ExpectedConfig(unitCfgMap) if err != nil { @@ -190,6 +179,68 @@ func EndpointTLSComponentModifier(log *logger.Logger) coordinator.ComponentsModi } } +func loadCertificatesWithCache(log *logger.Logger, cache *tlsCache, keyPassPath string, certPath string, keyPath string) (string, string, error) { + cache.mu.Lock() + defer cache.mu.Unlock() + + // cache hit + // hot reload of TLS files isn't supported, thus using the path as key is + // fine. + if cache.PassphrasePath == keyPassPath { + return cache.Certificate, cache.Key, nil + } + + cert, key, err := loadCertificates(log, keyPassPath, certPath, keyPath) + if err != nil { + return "", "", err + } + + cache.PassphrasePath = keyPassPath + cache.Certificate = cert + cache.Key = key + + return cert, key, nil +} + +func loadCertificates(log *logger.Logger, keyPassPathStr string, certStr string, keyStr string) (string, string, error) { + // all SSL config exists and the certificate key is passphrase protected, + // now decrypt the key + + pass, err := os.ReadFile(keyPassPathStr) + if err != nil { + return "", "", fmt.Errorf("EndpointTLSComponentModifier: failed to read client certificate passphrase file: %w", err) + } + + // we don't really support encrypted certificates, but it's how + // tlscommon.LoadCertificate does. Thus, let's keep the same behaviour. + // Also, tlscommon.LoadCertificate 'loses' the type of the private key. + // It stores they private key as an interface and there is no way to + // retrieve the type os the private key without a type assertion. + // Therefore, instead of manually checking the type, which would mean + // to check for all supported private key types and keep it up to date, + // better to load the certificate and its key directly from the PEM file. + cert, err := tlscommon.ReadPEMFile(log, + certStr, string(pass)) + if err != nil { + return "", "", fmt.Errorf("EndpointTLSComponentModifier: unable to load TLS certificate: %w", err) + } + key, err := tlscommon.ReadPEMFile(log, + keyStr, + string(pass)) + if err != nil { + return "", "", fmt.Errorf("EndpointTLSComponentModifier: unable to load TLS certificate key: %w", err) + } + + // tlscommon.ReadPEMFile only removes the 'DEK-Info' header, not the + // 'Proc-Type', so remove it now. Create a pem.Block to avoid editing + // the PEM data manually: + keyBlock, _ := pem.Decode(key) + delete(keyBlock.Headers, "Proc-Type") + key = pem.EncodeToMemory(keyBlock) + + return string(cert), string(key), nil +} + // findEndpointUnit finds the endpoint component and its unit of type 'unitType'. // It returns the component and unit index and true if found, if not, it returns // 0, 0, false. diff --git a/internal/pkg/agent/application/endpoint_component_modifier_test.go b/internal/pkg/agent/application/endpoint_component_modifier_test.go index 6ce66e97602..2275bf9bbd6 100644 --- a/internal/pkg/agent/application/endpoint_component_modifier_test.go +++ b/internal/pkg/agent/application/endpoint_component_modifier_test.go @@ -11,6 +11,7 @@ import ( "fmt" "os" "path/filepath" + "sync" "testing" "github.com/stretchr/testify/assert" @@ -36,34 +37,8 @@ func TestEndpointComponentModifier(t *testing.T) { loggertest.PrintObservedLogs(obs.TakeAll(), t.Log) }() - passphrase := "secure_passphrase" - _, _, pair, err := certutil.NewRootCA() - require.NoError(t, err, "could not create TLS certificate") - agentChildDERKey, _ := pem.Decode(pair.Key) - require.NoError(t, err, "could not create tls.Certificates from child certificate") - encPem, err := x509.EncryptPEMBlock( //nolint:staticcheck // we need to drop support for this, but while we don't, it needs to be tested. - rand.Reader, - "EC PRIVATE KEY", - agentChildDERKey.Bytes, - []byte(passphrase), - x509.PEMCipherAES128) - require.NoError(t, err, "failed encrypting agent child certificate key block") - - certKeyEnc := pem.EncodeToMemory(encPem) - - // save to disk - tmpDir := t.TempDir() - certPath := filepath.Join(tmpDir, "cert.pem") - certKeyPath := filepath.Join(tmpDir, "key.pem") - certKeyPassPath := filepath.Join(tmpDir, "key_pass.pem") - - err = os.WriteFile(certPath, pair.Cert, 0400) - require.NoError(t, err, "could write certificate key") - err = os.WriteFile(certKeyPath, certKeyEnc, 0400) - require.NoError(t, err, "could write certificate key") - err = os.WriteFile(certKeyPassPath, []byte(passphrase), 0400) - require.NoError(t, err, "could write certificate key passphrase") + pair, certPath, certKeyPath, certKeyPassPath := prepareEncTLSCertificates(t) tests := map[string][]struct { name string @@ -150,7 +125,7 @@ func TestEndpointComponentModifier(t *testing.T) { Config: &proto.UnitExpectedConfig{ Source: func() *structpb.Struct { var source structpb.Struct - err = source.UnmarshalJSON([]byte(`{"signed":{"data":"eyJpZCI6ImFhZWM4OTYwLWJiYjAtMTFlZC1hYzBkLTVmNjI0YTQxZjM4OCIsImFnZW50Ijp7InByb3RlY3Rpb24iOnsiZW5hYmxlZCI6dHJ1ZSwidW5pbnN0YWxsX3Rva2VuX2hhc2giOiIiLCJzaWduaW5nX2tleSI6Ik1Ga3dFd1lIS29aSXpqMENBUVlJS29aSXpqMERBUWNEUWdBRW1tckhDSTdtZ2tuZUJlYVJkc2VkQXZBU2l0UHRLbnpPdUlzeHZJRWdGTkFLVlg3MWpRTTVmalo1eUdsSDB0TmJuR2JrU2pVM0VEVUZsOWllQ1J0ME5nPT0ifX19", "signature":"MEUCIQCWoScyJW0dejHFxXBTEcSCOZiBHRVMjuJRPwFCwOdA1QIgKrtKUBzkvVeljRtJyMXfD8zIvWjrMzqhSkgjNESPW5E="}}`)) + err := source.UnmarshalJSON([]byte(`{"signed":{"data":"eyJpZCI6ImFhZWM4OTYwLWJiYjAtMTFlZC1hYzBkLTVmNjI0YTQxZjM4OCIsImFnZW50Ijp7InByb3RlY3Rpb24iOnsiZW5hYmxlZCI6dHJ1ZSwidW5pbnN0YWxsX3Rva2VuX2hhc2giOiIiLCJzaWduaW5nX2tleSI6Ik1Ga3dFd1lIS29aSXpqMENBUVlJS29aSXpqMERBUWNEUWdBRW1tckhDSTdtZ2tuZUJlYVJkc2VkQXZBU2l0UHRLbnpPdUlzeHZJRWdGTkFLVlg3MWpRTTVmalo1eUdsSDB0TmJuR2JrU2pVM0VEVUZsOWllQ1J0ME5nPT0ifX19", "signature":"MEUCIQCWoScyJW0dejHFxXBTEcSCOZiBHRVMjuJRPwFCwOdA1QIgKrtKUBzkvVeljRtJyMXfD8zIvWjrMzqhSkgjNESPW5E="}}`)) require.NoError(t, err, "could not create want component source config") return &source }(), @@ -440,22 +415,167 @@ func TestEndpointComponentModifier(t *testing.T) { // Cumbersome comparison of the source config encoded in protobuf, // cmp panics protobufs comparison otherwise. - if len(tc.wantComps) > 0 && - len(tc.wantComps[0].Units) > 0 && - comps[0].Units[0].Config != nil && - comps[0].Units[0].Config.Source != nil { - - unitCgf := comps[0].Units[0].Config.Source.AsMap() - wantUnitCfg := tc.wantComps[0].Units[0].Config.Source.AsMap() - - assert.Equal(t, wantUnitCfg, unitCgf, "unit config do not match") - } + compareComponents(t, comps, tc.wantComps) }) } }) } } +func compareComponents(t *testing.T, got, want []component.Component) { + if len(want) > 0 && + len(want[0].Units) > 0 && + got[0].Units[0].Config != nil && + got[0].Units[0].Config.Source != nil { + + unitCgf := got[0].Units[0].Config.Source.AsMap() + wantUnitCfg := want[0].Units[0].Config.Source.AsMap() + + assert.Equal(t, wantUnitCfg, unitCgf, "unit config do not match") + } +} + +func TestEndpointTLSComponentModifier_cache_miss(t *testing.T) { + log, obs := loggertest.New("TestEndpointSignedComponentModifier") + defer func() { + if !t.Failed() { + return + } + + loggertest.PrintObservedLogs(obs.TakeAll(), t.Log) + }() + + pair, certPath, certKeyPath, certKeyPassPath := prepareEncTLSCertificates(t) + + comps := makeComponent(t, fmt.Sprintf(`{ + "fleet": { + "ssl": { + "certificate": %q, + "key": %q, + "key_passphrase_path": %q + } + } + }`, certPath, certKeyPath, certKeyPassPath)) + cfg := map[string]interface{}{ + "fleet": map[string]interface{}{ + "ssl": map[string]interface{}{ + "certificate": certPath, + "key": certKeyPath, + "key_passphrase_path": certKeyPassPath, + }, + }, + } + wantComps := makeComponent(t, fmt.Sprintf(`{ + "fleet": { + "ssl": { + "certificate": %q, + "key": %q + } + } + }`, pair.Cert, pair.Key)) + + cache := tlsCache{mu: &sync.Mutex{}} + modifier := newEndpointTLSComponentModifier(log, &cache) + got, err := modifier(comps, cfg) + require.NoError(t, err, "unexpected error") + + assert.Equal(t, certKeyPassPath, cache.PassphrasePath, "passphrase path did not match") + assert.Equal(t, string(pair.Cert), cache.Certificate, "certificate did not match") + assert.Equal(t, string(pair.Key), cache.Key, "key did not match") + + compareComponents(t, got, wantComps) +} + +func TestEndpointTLSComponentModifier_cache_hit(t *testing.T) { + log, obs := loggertest.New("TestEndpointSignedComponentModifier") + defer func() { + if !t.Failed() { + return + } + + loggertest.PrintObservedLogs(obs.TakeAll(), t.Log) + }() + + certPath := "/path/to/cert" + certKeyPath := "/path/to/key" + certKeyPassPath := "/path/to/key_passphrase_path" + cache := tlsCache{ + mu: &sync.Mutex{}, + + PassphrasePath: certKeyPassPath, + Certificate: "cached certificate", + Key: "cached key", + } + + comps := makeComponent(t, fmt.Sprintf(`{ + "fleet": { + "ssl": { + "certificate": %q, + "key": %q, + "key_passphrase_path": %q + } + } + }`, certPath, certKeyPath, certKeyPassPath)) + cfg := map[string]interface{}{ + "fleet": map[string]interface{}{ + "ssl": map[string]interface{}{ + "certificate": cache.Certificate, + "key": cache.Key, + "key_passphrase_path": cache.PassphrasePath, + }, + }, + } + + wantComps := makeComponent(t, fmt.Sprintf(`{ + "fleet": { + "ssl": { + "certificate": %q, + "key": %q + } + } + }`, cache.Certificate, cache.Key)) + + modifier := newEndpointTLSComponentModifier(log, &cache) + got, err := modifier(comps, cfg) + require.NoError(t, err, "unexpected error") + + assert.Equal(t, certKeyPassPath, cache.PassphrasePath, "passphrase should not have changed") + compareComponents(t, got, wantComps) +} + +func prepareEncTLSCertificates(t *testing.T) (certutil.Pair, string, string, string) { + passphrase := "secure_passphrase" + _, _, pair, err := certutil.NewRootCA() + require.NoError(t, err, "could not create TLS certificate") + agentChildDERKey, _ := pem.Decode(pair.Key) + require.NoError(t, err, "could not create tls.Certificates from child certificate") + + encPem, err := x509.EncryptPEMBlock( //nolint:staticcheck // we need to drop support for this, but while we don't, it needs to be tested. + rand.Reader, + "EC PRIVATE KEY", + agentChildDERKey.Bytes, + []byte(passphrase), + x509.PEMCipherAES128) + require.NoError(t, err, "failed encrypting agent child certificate key block") + + certKeyEnc := pem.EncodeToMemory(encPem) + + // save to disk + tmpDir := t.TempDir() + certPath := filepath.Join(tmpDir, "cert.pem") + certKeyPath := filepath.Join(tmpDir, "key.pem") + certKeyPassPath := filepath.Join(tmpDir, "key_pass.pem") + + err = os.WriteFile(certPath, pair.Cert, 0400) + require.NoError(t, err, "could write certificate key") + err = os.WriteFile(certKeyPath, certKeyEnc, 0400) + require.NoError(t, err, "could write certificate key") + err = os.WriteFile(certKeyPassPath, []byte(passphrase), 0400) + require.NoError(t, err, "could write certificate key passphrase") + + return pair, certPath, certKeyPath, certKeyPassPath +} + func makeComponent(t *testing.T, sourceCfg string) []component.Component { return []component.Component{ { From af275d3a31aea3550cd64d75489448215d13f187 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Mon, 30 Sep 2024 18:19:07 +0200 Subject: [PATCH 03/13] fix cache miss test --- .../agent/application/endpoint_component_modifier_test.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/internal/pkg/agent/application/endpoint_component_modifier_test.go b/internal/pkg/agent/application/endpoint_component_modifier_test.go index 2275bf9bbd6..ff9f8a50c0f 100644 --- a/internal/pkg/agent/application/endpoint_component_modifier_test.go +++ b/internal/pkg/agent/application/endpoint_component_modifier_test.go @@ -445,6 +445,13 @@ func TestEndpointTLSComponentModifier_cache_miss(t *testing.T) { loggertest.PrintObservedLogs(obs.TakeAll(), t.Log) }() + cache := tlsCache{ + mu: &sync.Mutex{}, + + PassphrasePath: "/old/path/to/key-passphrase", + Certificate: "cached certificate", + Key: "cached key", + } pair, certPath, certKeyPath, certKeyPassPath := prepareEncTLSCertificates(t) comps := makeComponent(t, fmt.Sprintf(`{ @@ -474,7 +481,6 @@ func TestEndpointTLSComponentModifier_cache_miss(t *testing.T) { } }`, pair.Cert, pair.Key)) - cache := tlsCache{mu: &sync.Mutex{}} modifier := newEndpointTLSComponentModifier(log, &cache) got, err := modifier(comps, cfg) require.NoError(t, err, "unexpected error") From 988b83afa09ac842230997863cb593bb3020b6bf Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Wed, 2 Oct 2024 17:24:19 +0200 Subject: [PATCH 04/13] fix linter --- .../pkg/agent/application/endpoint_component_modifier_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/pkg/agent/application/endpoint_component_modifier_test.go b/internal/pkg/agent/application/endpoint_component_modifier_test.go index ff9f8a50c0f..4e713559149 100644 --- a/internal/pkg/agent/application/endpoint_component_modifier_test.go +++ b/internal/pkg/agent/application/endpoint_component_modifier_test.go @@ -504,7 +504,7 @@ func TestEndpointTLSComponentModifier_cache_hit(t *testing.T) { certPath := "/path/to/cert" certKeyPath := "/path/to/key" - certKeyPassPath := "/path/to/key_passphrase_path" + certKeyPassPath := "/path/to/key_passphrase_path" //nolint:gosec // not a real key cache := tlsCache{ mu: &sync.Mutex{}, From 3de25cc98d8d35c3d98f0d30bc751a373957a5ee Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Mon, 7 Oct 2024 11:46:36 +0200 Subject: [PATCH 05/13] update elastic-agent-libs --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b683324f1b9..8de046fd345 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/dolmen-go/contextio v0.0.0-20200217195037-68fc5150bcd5 github.com/elastic/elastic-agent-autodiscover v0.8.2 github.com/elastic/elastic-agent-client/v7 v7.16.0 - github.com/elastic/elastic-agent-libs v0.11.0 + github.com/elastic/elastic-agent-libs v0.12.0 github.com/elastic/elastic-agent-system-metrics v0.11.2 github.com/elastic/elastic-transport-go/v8 v8.6.0 github.com/elastic/go-elasticsearch/v8 v8.15.0 diff --git a/go.sum b/go.sum index d2943b24072..a8791d6cd7f 100644 --- a/go.sum +++ b/go.sum @@ -241,8 +241,8 @@ github.com/elastic/elastic-agent-autodiscover v0.8.2 h1:Fs2FhR33AMBPfm5/jz4drVza github.com/elastic/elastic-agent-autodiscover v0.8.2/go.mod h1:VZnU53EVaFTxR8Xf6YsLN8FHD5DKQzHSPlKax9/4w+o= github.com/elastic/elastic-agent-client/v7 v7.16.0 h1:yKGq2+CxAuW8Kh0EoNl202tqAyQKfBcPRawVKs2Jve0= github.com/elastic/elastic-agent-client/v7 v7.16.0/go.mod h1:6h+f9QdIr3GO2ODC0Y8+aEXRwzbA5W4eV4dd/67z7nI= -github.com/elastic/elastic-agent-libs v0.11.0 h1:m9rnNE3BkBF2XJoqubqEbu/kbtKEBZ7pHCjDlxfVRH0= -github.com/elastic/elastic-agent-libs v0.11.0/go.mod h1:5CR02awPrBr+tfmjBBK+JI+dMmHNQjpVY24J0wjbC7M= +github.com/elastic/elastic-agent-libs v0.12.0 h1:xfVVCYIaI6XEPVpJNq7HQav7O/VxLj+YbQK/poWb7wA= +github.com/elastic/elastic-agent-libs v0.12.0/go.mod h1:5CR02awPrBr+tfmjBBK+JI+dMmHNQjpVY24J0wjbC7M= github.com/elastic/elastic-agent-system-metrics v0.11.2 h1:UjSBcY6U3H3venB5zAPEFNjAb4Bb38EiVqaA4zALEnM= github.com/elastic/elastic-agent-system-metrics v0.11.2/go.mod h1:saqLKe9fuyuAo6IADAnnuy1kaBI7VNlxfwMo8KzSRyQ= github.com/elastic/elastic-transport-go/v8 v8.6.0 h1:Y2S/FBjx1LlCv5m6pWAF2kDJAHoSjSRSJCApolgfthA= From 3d5a8f3f26f19527c45cc4e715f7605dae42e432 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Mon, 7 Oct 2024 11:46:49 +0200 Subject: [PATCH 06/13] better comments --- .../pkg/agent/application/endpoint_component_modifier.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/pkg/agent/application/endpoint_component_modifier.go b/internal/pkg/agent/application/endpoint_component_modifier.go index c6b35d5f4f0..704c51218b7 100644 --- a/internal/pkg/agent/application/endpoint_component_modifier.go +++ b/internal/pkg/agent/application/endpoint_component_modifier.go @@ -25,7 +25,10 @@ import ( type tlsCache struct { mu *sync.Mutex - // PassphrasePath is used as the cache key + // PassphrasePath is used as the cache key. + // Watching the file for changes and reloading the file if any change is + // detected isn't supported, therefore it's safe to use the + // keyPassphrasePath as cache key. PassphrasePath string Certificate string @@ -184,8 +187,6 @@ func loadCertificatesWithCache(log *logger.Logger, cache *tlsCache, keyPassPath defer cache.mu.Unlock() // cache hit - // hot reload of TLS files isn't supported, thus using the path as key is - // fine. if cache.PassphrasePath == keyPassPath { return cache.Certificate, cache.Key, nil } From 268976087a19b11ee34f4f9876267b49955eb9a1 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Wed, 9 Oct 2024 10:27:57 +0200 Subject: [PATCH 07/13] fix comment --- .../pkg/agent/application/endpoint_component_modifier.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/internal/pkg/agent/application/endpoint_component_modifier.go b/internal/pkg/agent/application/endpoint_component_modifier.go index 704c51218b7..7a9bb7d7836 100644 --- a/internal/pkg/agent/application/endpoint_component_modifier.go +++ b/internal/pkg/agent/application/endpoint_component_modifier.go @@ -157,6 +157,8 @@ func newEndpointTLSComponentModifier(log *logger.Logger, cache *tlsCache) func(c return nil, errors.New("EndpointTLSComponentModifier: 'certificate' isn't a string") } + // all TLS config exists and the certificate key is passphrase protected, + // now load and decrypt the key. cert, key, err := loadCertificatesWithCache(log, cache, keyPassPath, certPath, keyPath) if err != nil { return nil, err @@ -204,9 +206,6 @@ func loadCertificatesWithCache(log *logger.Logger, cache *tlsCache, keyPassPath } func loadCertificates(log *logger.Logger, keyPassPathStr string, certStr string, keyStr string) (string, string, error) { - // all SSL config exists and the certificate key is passphrase protected, - // now decrypt the key - pass, err := os.ReadFile(keyPassPathStr) if err != nil { return "", "", fmt.Errorf("EndpointTLSComponentModifier: failed to read client certificate passphrase file: %w", err) From 3c6082be1baa48b66f3d7c94bcc2d3f3f8004ed2 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Wed, 9 Oct 2024 12:02:54 +0200 Subject: [PATCH 08/13] mage notice --- NOTICE.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index 6f2aebf44a5..8414ca710ae 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -1264,11 +1264,11 @@ SOFTWARE -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-libs -Version: v0.11.0 +Version: v0.12.0 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.11.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.12.0/LICENSE: Apache License Version 2.0, January 2004 From 7ba4a21a8c0b9f578a2168bdc3ffb532ade97119 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Wed, 9 Oct 2024 18:01:37 +0200 Subject: [PATCH 09/13] update elastic-agent-libs --- NOTICE.txt | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/NOTICE.txt b/NOTICE.txt index 8414ca710ae..780e23a5797 100644 --- a/NOTICE.txt +++ b/NOTICE.txt @@ -1264,11 +1264,11 @@ SOFTWARE -------------------------------------------------------------------------------- Dependency : github.com/elastic/elastic-agent-libs -Version: v0.12.0 +Version: v0.12.1 Licence type (autodetected): Apache-2.0 -------------------------------------------------------------------------------- -Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.12.0/LICENSE: +Contents of probable licence file $GOMODCACHE/github.com/elastic/elastic-agent-libs@v0.12.1/LICENSE: Apache License Version 2.0, January 2004 diff --git a/go.mod b/go.mod index 3ed87b805fa..fdb7ac289f9 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/dolmen-go/contextio v0.0.0-20200217195037-68fc5150bcd5 github.com/elastic/elastic-agent-autodiscover v0.9.0 github.com/elastic/elastic-agent-client/v7 v7.16.0 - github.com/elastic/elastic-agent-libs v0.12.0 + github.com/elastic/elastic-agent-libs v0.12.1 github.com/elastic/elastic-agent-system-metrics v0.11.3 github.com/elastic/elastic-transport-go/v8 v8.6.0 github.com/elastic/go-elasticsearch/v8 v8.15.0 diff --git a/go.sum b/go.sum index 7f58740d47b..01c8d39b8ce 100644 --- a/go.sum +++ b/go.sum @@ -248,8 +248,8 @@ github.com/elastic/elastic-agent-autodiscover v0.9.0 h1:+iWIKh0u3e8I+CJa3FfWe9h0 github.com/elastic/elastic-agent-autodiscover v0.9.0/go.mod h1:5iUxLHhVdaGSWYTveSwfJEY4RqPXTG13LPiFoxcpFd4= github.com/elastic/elastic-agent-client/v7 v7.16.0 h1:yKGq2+CxAuW8Kh0EoNl202tqAyQKfBcPRawVKs2Jve0= github.com/elastic/elastic-agent-client/v7 v7.16.0/go.mod h1:6h+f9QdIr3GO2ODC0Y8+aEXRwzbA5W4eV4dd/67z7nI= -github.com/elastic/elastic-agent-libs v0.12.0 h1:xfVVCYIaI6XEPVpJNq7HQav7O/VxLj+YbQK/poWb7wA= -github.com/elastic/elastic-agent-libs v0.12.0/go.mod h1:5CR02awPrBr+tfmjBBK+JI+dMmHNQjpVY24J0wjbC7M= +github.com/elastic/elastic-agent-libs v0.12.1 h1:5jkxMx15Bna8cq7/Sz/XUIVUXfNWiJ80iSk4ICQ7KJ0= +github.com/elastic/elastic-agent-libs v0.12.1/go.mod h1:5CR02awPrBr+tfmjBBK+JI+dMmHNQjpVY24J0wjbC7M= github.com/elastic/elastic-agent-system-metrics v0.11.3 h1:LDzRwP8kxvsYEtMDgMSKZs1TgPcSEukit+/EAP5Y28A= github.com/elastic/elastic-agent-system-metrics v0.11.3/go.mod h1:saqLKe9fuyuAo6IADAnnuy1kaBI7VNlxfwMo8KzSRyQ= github.com/elastic/elastic-transport-go/v8 v8.6.0 h1:Y2S/FBjx1LlCv5m6pWAF2kDJAHoSjSRSJCApolgfthA= From b04b42c3b7ddde2f6760cd481ad947dc9558eed0 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Fri, 11 Oct 2024 09:39:08 +0200 Subject: [PATCH 10/13] debug test --- testing/integration/logs_ingestion_test.go | 31 +++++++++++-- .../integration/metrics_monitoring_test.go | 46 +++++++++++++++++-- .../monitoring_probe_reload_test.go | 4 +- 3 files changed, 73 insertions(+), 8 deletions(-) diff --git a/testing/integration/logs_ingestion_test.go b/testing/integration/logs_ingestion_test.go index 56a2fcf2e29..b6af1b58857 100644 --- a/testing/integration/logs_ingestion_test.go +++ b/testing/integration/logs_ingestion_test.go @@ -53,8 +53,11 @@ func TestLogIngestionFleetManaged(t *testing.T) { Sudo: true, }) - ctx, cancel := testcontext.WithDeadline(t, context.Background(), time.Now().Add(10*time.Minute)) - defer cancel() + if d, ok := t.Deadline(); ok { + t.Logf("test global timeout: %s", d.Sub(time.Now())) + } else { + t.Logf("test global timeout not defined") + } agentFixture, err := define.NewFixtureFromLocalBuild(t, define.Version()) require.NoError(t, err) @@ -92,6 +95,14 @@ func TestLogIngestionFleetManaged(t *testing.T) { Force: true, } + ctx, cancel := testcontext.WithTimeout(t, context.Background(), 5*time.Minute) + defer cancel() + if d, ok := ctx.Deadline(); ok { + t.Logf("context timeout: %s", d.Sub(time.Now())) + } else { + t.Fatal("context without deadline after calling context.WithDeadline") + } + // 2. Install the Elastic-Agent with the policy that // was just created. policy, err := tools.InstallAgentWithPolicy( @@ -111,10 +122,24 @@ func TestLogIngestionFleetManaged(t *testing.T) { // 4. Ensure healthy state at startup checkHealthAtStartup(t, ctx, agentFixture) + ctx, cancel = testcontext.WithTimeout(t, context.Background(), 10*time.Minute) + defer cancel() + if d, ok := ctx.Deadline(); ok { + t.Logf("context timeout: %s", d.Sub(time.Now())) + } else { + t.Fatal("context without deadline after calling context.WithDeadline") + } t.Run("Monitoring logs are shipped", func(t *testing.T) { testMonitoringLogsAreShipped(t, ctx, info, agentFixture, policy) }) + ctx, cancel = testcontext.WithTimeout(t, context.Background(), 10*time.Minute) + defer cancel() + if d, ok := ctx.Deadline(); ok { + t.Logf("context timeout: %s", d.Sub(time.Now())) + } else { + t.Fatal("context without deadline after calling context.WithDeadline") + } t.Run("Normal logs with flattened data_stream are shipped", func(t *testing.T) { testFlattenedDatastreamFleetPolicy(t, ctx, info, policy) }) @@ -404,7 +429,7 @@ func testFlattenedDatastreamFleetPolicy( require.Eventually( t, ensureDocumentsInES(t, ctx, info.ESClient, dsType, dsDataset, dsNamespace, numEvents), - 120*time.Second, + 3*time.Minute, time.Second, "could not get all expected documents form ES") } diff --git a/testing/integration/metrics_monitoring_test.go b/testing/integration/metrics_monitoring_test.go index 08eda658ce6..ee5858dc4dd 100644 --- a/testing/integration/metrics_monitoring_test.go +++ b/testing/integration/metrics_monitoring_test.go @@ -7,6 +7,7 @@ package integration import ( + "bytes" "context" "fmt" "testing" @@ -17,6 +18,7 @@ import ( "github.com/stretchr/testify/suite" "github.com/elastic/elastic-agent-libs/kibana" + "github.com/elastic/elastic-agent/pkg/control/v2/client" atesting "github.com/elastic/elastic-agent/pkg/testing" "github.com/elastic/elastic-agent/pkg/testing/define" "github.com/elastic/elastic-agent/pkg/testing/tools" @@ -76,15 +78,53 @@ func (runner *MetricsRunner) SetupSuite() { _, err = tools.InstallPackageFromDefaultFile(ctx, runner.info.KibanaClient, "system", "1.53.1", "system_integration_setup.json", uuid.Must(uuid.NewV4()).String(), policyResp.ID) require.NoError(runner.T(), err) - } func (runner *MetricsRunner) TestBeatsMetrics() { UnitOutputName := "default" ctx, cancel := context.WithTimeout(context.Background(), time.Minute*20) defer cancel() - agentStatus, err := runner.agentFixture.ExecStatus(ctx) - require.NoError(runner.T(), err) + + var agentStatus atesting.AgentStatusOutput + f := runner.agentFixture + stateBuff := bytes.Buffer{} + require.Eventuallyf(runner.T(), func() bool { + stateBuff.Reset() + + status, err := f.ExecStatus(ctx) + if err != nil { + stateBuff.WriteString(fmt.Sprintf("failed to get agent status: %v", + err)) + return false + } + + if client.State(status.State) != client.Healthy { + stateBuff.WriteString(fmt.Sprintf( + "agent isn't healthy: %s-%s", + client.State(status.State), status.Message)) + return false + } + + if len(status.Components) == 0 { + stateBuff.WriteString(fmt.Sprintf( + "healthy but without components: agent status: %s-%s", + client.State(status.State), status.Message)) + return false + } + + for _, c := range status.Components { + + state := client.State(c.State) + if state != client.Healthy { + stateBuff.WriteString(fmt.Sprintf( + "%s not health: %s: %s", + c.Name, state, c.Message)) + return false + } + } + + return true + }, 2*time.Minute, 15*time.Second, "agent never became healthy. Last status: %v", &stateBuff) componentIds := []string{ fmt.Sprintf("system/metrics-%s", UnitOutputName), diff --git a/testing/integration/monitoring_probe_reload_test.go b/testing/integration/monitoring_probe_reload_test.go index 134a0143ca4..3a6807eff2b 100644 --- a/testing/integration/monitoring_probe_reload_test.go +++ b/testing/integration/monitoring_probe_reload_test.go @@ -172,12 +172,12 @@ func (runner *MonitoringRunner) AllComponentsHealthy(ctx context.Context) { } for _, comp := range status.Components { - runner.T().Logf("component state: %s", comp.Message) + runner.T().Logf("%s current state: %s", comp.Name, comp.Message) if comp.State != int(cproto.State_HEALTHY) { compDebugName = comp.Name allHealthy = false } } return allHealthy - }, runner.healthCheckTime, runner.healthCheckRefreshTime, "install never became healthy: components did not return a healthy state: %s", compDebugName) + }, runner.healthCheckTime, runner.healthCheckRefreshTime, "install never became healthy: components did not return a healthy state: %q", compDebugName) } From 25d7ae85cfe836db4575c17b8bd588136912f5d6 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Mon, 14 Oct 2024 14:12:57 +0200 Subject: [PATCH 11/13] fix test --- .../integration/metrics_monitoring_test.go | 47 ++++++++++++++----- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/testing/integration/metrics_monitoring_test.go b/testing/integration/metrics_monitoring_test.go index ee5858dc4dd..bb7813422e9 100644 --- a/testing/integration/metrics_monitoring_test.go +++ b/testing/integration/metrics_monitoring_test.go @@ -9,6 +9,7 @@ package integration import ( "bytes" "context" + "encoding/json" "fmt" "testing" "time" @@ -81,6 +82,8 @@ func (runner *MetricsRunner) SetupSuite() { } func (runner *MetricsRunner) TestBeatsMetrics() { + t := runner.T() + UnitOutputName := "default" ctx, cancel := context.WithTimeout(context.Background(), time.Minute*20) defer cancel() @@ -88,31 +91,32 @@ func (runner *MetricsRunner) TestBeatsMetrics() { var agentStatus atesting.AgentStatusOutput f := runner.agentFixture stateBuff := bytes.Buffer{} - require.Eventuallyf(runner.T(), func() bool { + require.Eventuallyf(t, func() bool { stateBuff.Reset() - status, err := f.ExecStatus(ctx) + var err error + agentStatus, err = f.ExecStatus(ctx) if err != nil { stateBuff.WriteString(fmt.Sprintf("failed to get agent status: %v", err)) return false } - if client.State(status.State) != client.Healthy { + if client.State(agentStatus.State) != client.Healthy { stateBuff.WriteString(fmt.Sprintf( "agent isn't healthy: %s-%s", - client.State(status.State), status.Message)) + client.State(agentStatus.State), agentStatus.Message)) return false } - if len(status.Components) == 0 { + if len(agentStatus.Components) == 0 { stateBuff.WriteString(fmt.Sprintf( "healthy but without components: agent status: %s-%s", - client.State(status.State), status.Message)) + client.State(agentStatus.State), agentStatus.Message)) return false } - for _, c := range status.Components { + for _, c := range agentStatus.Components { state := client.State(c.State) if state != client.Healthy { @@ -135,19 +139,36 @@ func (runner *MetricsRunner) TestBeatsMetrics() { "filestream-monitoring", } - require.Eventually(runner.T(), func() bool { + now := time.Now() + var query map[string]any + defer func() { + if t.Failed() { + bs, err := json.Marshal(query) + if err != nil { + // nothing we can do, just log the map + t.Errorf("executed at %s: %v", + now.Format(time.RFC3339Nano), query) + return + } + t.Errorf("executed at %s: query: %s", + now.Format(time.RFC3339Nano), string(bs)) + } + }() + + t.Logf("starting to ES for metrics at %s", now.Format(time.RFC3339Nano)) + require.Eventually(t, func() bool { for _, cid := range componentIds { - query := genESQuery(agentStatus.Info.ID, cid) + query = genESQuery(agentStatus.Info.ID, cid) + now = time.Now() res, err := estools.PerformQueryForRawQuery(ctx, query, "metrics-elastic_agent*", runner.info.ESClient) - require.NoError(runner.T(), err) - runner.T().Logf("Fetched metrics for %s, got %d hits", cid, res.Hits.Total.Value) + require.NoError(t, err) + t.Logf("Fetched metrics for %s, got %d hits", cid, res.Hits.Total.Value) if res.Hits.Total.Value < 1 { return false } - } return true - }, time.Minute*10, time.Second*10, "could not fetch metrics for all known beats in default install: %v", componentIds) + }, time.Minute*10, time.Second*10, "could not fetch metrics for all known components in default install: %v", componentIds) } func genESQuery(agentID string, componentID string) map[string]interface{} { From 782808e61c7f098d59e5bff8503b5b828178fe39 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Mon, 14 Oct 2024 14:31:42 +0200 Subject: [PATCH 12/13] Revert "debug test" This reverts commit b04b42c3b7ddde2f6760cd481ad947dc9558eed0. --- testing/integration/logs_ingestion_test.go | 31 ++----------- .../integration/metrics_monitoring_test.go | 46 ++----------------- .../monitoring_probe_reload_test.go | 4 +- 3 files changed, 8 insertions(+), 73 deletions(-) diff --git a/testing/integration/logs_ingestion_test.go b/testing/integration/logs_ingestion_test.go index b6af1b58857..56a2fcf2e29 100644 --- a/testing/integration/logs_ingestion_test.go +++ b/testing/integration/logs_ingestion_test.go @@ -53,11 +53,8 @@ func TestLogIngestionFleetManaged(t *testing.T) { Sudo: true, }) - if d, ok := t.Deadline(); ok { - t.Logf("test global timeout: %s", d.Sub(time.Now())) - } else { - t.Logf("test global timeout not defined") - } + ctx, cancel := testcontext.WithDeadline(t, context.Background(), time.Now().Add(10*time.Minute)) + defer cancel() agentFixture, err := define.NewFixtureFromLocalBuild(t, define.Version()) require.NoError(t, err) @@ -95,14 +92,6 @@ func TestLogIngestionFleetManaged(t *testing.T) { Force: true, } - ctx, cancel := testcontext.WithTimeout(t, context.Background(), 5*time.Minute) - defer cancel() - if d, ok := ctx.Deadline(); ok { - t.Logf("context timeout: %s", d.Sub(time.Now())) - } else { - t.Fatal("context without deadline after calling context.WithDeadline") - } - // 2. Install the Elastic-Agent with the policy that // was just created. policy, err := tools.InstallAgentWithPolicy( @@ -122,24 +111,10 @@ func TestLogIngestionFleetManaged(t *testing.T) { // 4. Ensure healthy state at startup checkHealthAtStartup(t, ctx, agentFixture) - ctx, cancel = testcontext.WithTimeout(t, context.Background(), 10*time.Minute) - defer cancel() - if d, ok := ctx.Deadline(); ok { - t.Logf("context timeout: %s", d.Sub(time.Now())) - } else { - t.Fatal("context without deadline after calling context.WithDeadline") - } t.Run("Monitoring logs are shipped", func(t *testing.T) { testMonitoringLogsAreShipped(t, ctx, info, agentFixture, policy) }) - ctx, cancel = testcontext.WithTimeout(t, context.Background(), 10*time.Minute) - defer cancel() - if d, ok := ctx.Deadline(); ok { - t.Logf("context timeout: %s", d.Sub(time.Now())) - } else { - t.Fatal("context without deadline after calling context.WithDeadline") - } t.Run("Normal logs with flattened data_stream are shipped", func(t *testing.T) { testFlattenedDatastreamFleetPolicy(t, ctx, info, policy) }) @@ -429,7 +404,7 @@ func testFlattenedDatastreamFleetPolicy( require.Eventually( t, ensureDocumentsInES(t, ctx, info.ESClient, dsType, dsDataset, dsNamespace, numEvents), - 3*time.Minute, + 120*time.Second, time.Second, "could not get all expected documents form ES") } diff --git a/testing/integration/metrics_monitoring_test.go b/testing/integration/metrics_monitoring_test.go index bb7813422e9..b4d885a19cb 100644 --- a/testing/integration/metrics_monitoring_test.go +++ b/testing/integration/metrics_monitoring_test.go @@ -7,7 +7,6 @@ package integration import ( - "bytes" "context" "encoding/json" "fmt" @@ -19,7 +18,6 @@ import ( "github.com/stretchr/testify/suite" "github.com/elastic/elastic-agent-libs/kibana" - "github.com/elastic/elastic-agent/pkg/control/v2/client" atesting "github.com/elastic/elastic-agent/pkg/testing" "github.com/elastic/elastic-agent/pkg/testing/define" "github.com/elastic/elastic-agent/pkg/testing/tools" @@ -79,6 +77,7 @@ func (runner *MetricsRunner) SetupSuite() { _, err = tools.InstallPackageFromDefaultFile(ctx, runner.info.KibanaClient, "system", "1.53.1", "system_integration_setup.json", uuid.Must(uuid.NewV4()).String(), policyResp.ID) require.NoError(runner.T(), err) + } func (runner *MetricsRunner) TestBeatsMetrics() { @@ -88,47 +87,8 @@ func (runner *MetricsRunner) TestBeatsMetrics() { ctx, cancel := context.WithTimeout(context.Background(), time.Minute*20) defer cancel() - var agentStatus atesting.AgentStatusOutput - f := runner.agentFixture - stateBuff := bytes.Buffer{} - require.Eventuallyf(t, func() bool { - stateBuff.Reset() - - var err error - agentStatus, err = f.ExecStatus(ctx) - if err != nil { - stateBuff.WriteString(fmt.Sprintf("failed to get agent status: %v", - err)) - return false - } - - if client.State(agentStatus.State) != client.Healthy { - stateBuff.WriteString(fmt.Sprintf( - "agent isn't healthy: %s-%s", - client.State(agentStatus.State), agentStatus.Message)) - return false - } - - if len(agentStatus.Components) == 0 { - stateBuff.WriteString(fmt.Sprintf( - "healthy but without components: agent status: %s-%s", - client.State(agentStatus.State), agentStatus.Message)) - return false - } - - for _, c := range agentStatus.Components { - - state := client.State(c.State) - if state != client.Healthy { - stateBuff.WriteString(fmt.Sprintf( - "%s not health: %s: %s", - c.Name, state, c.Message)) - return false - } - } - - return true - }, 2*time.Minute, 15*time.Second, "agent never became healthy. Last status: %v", &stateBuff) + agentStatus, err := runner.agentFixture.ExecStatus(ctx) + require.NoError(t, err, "could not to get agent status") componentIds := []string{ fmt.Sprintf("system/metrics-%s", UnitOutputName), diff --git a/testing/integration/monitoring_probe_reload_test.go b/testing/integration/monitoring_probe_reload_test.go index 3a6807eff2b..134a0143ca4 100644 --- a/testing/integration/monitoring_probe_reload_test.go +++ b/testing/integration/monitoring_probe_reload_test.go @@ -172,12 +172,12 @@ func (runner *MonitoringRunner) AllComponentsHealthy(ctx context.Context) { } for _, comp := range status.Components { - runner.T().Logf("%s current state: %s", comp.Name, comp.Message) + runner.T().Logf("component state: %s", comp.Message) if comp.State != int(cproto.State_HEALTHY) { compDebugName = comp.Name allHealthy = false } } return allHealthy - }, runner.healthCheckTime, runner.healthCheckRefreshTime, "install never became healthy: components did not return a healthy state: %q", compDebugName) + }, runner.healthCheckTime, runner.healthCheckRefreshTime, "install never became healthy: components did not return a healthy state: %s", compDebugName) } From 812381685033c8c1fa05cf5bac57bea5bf3ec473 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Tue, 15 Oct 2024 16:43:44 +0200 Subject: [PATCH 13/13] make cache key from all paths --- .../endpoint_component_modifier.go | 16 +++++++------- .../endpoint_component_modifier_test.go | 21 +++++++++++-------- 2 files changed, 21 insertions(+), 16 deletions(-) diff --git a/internal/pkg/agent/application/endpoint_component_modifier.go b/internal/pkg/agent/application/endpoint_component_modifier.go index 7a9bb7d7836..bc9b4b1db07 100644 --- a/internal/pkg/agent/application/endpoint_component_modifier.go +++ b/internal/pkg/agent/application/endpoint_component_modifier.go @@ -25,16 +25,16 @@ import ( type tlsCache struct { mu *sync.Mutex - // PassphrasePath is used as the cache key. - // Watching the file for changes and reloading the file if any change is - // detected isn't supported, therefore it's safe to use the - // keyPassphrasePath as cache key. - PassphrasePath string + CacheKey string Certificate string Key string } +func (tlsCache) MakeKey(keyPassPath, certPath, keyPath string) string { + return keyPassPath + certPath + keyPath +} + // EndpointSignedComponentModifier copies "signed" properties to the top level "signed" for the endpoint input. // Enpoint team want to be able to validate the signature and parse the signed configuration (not trust the agent). // Endpoint uses uninstall_token_hash in order to verify uninstall command token @@ -188,8 +188,10 @@ func loadCertificatesWithCache(log *logger.Logger, cache *tlsCache, keyPassPath cache.mu.Lock() defer cache.mu.Unlock() + cacheKey := cache.MakeKey(keyPassPath, certPath, keyPath) + // cache hit - if cache.PassphrasePath == keyPassPath { + if cache.CacheKey == cacheKey { return cache.Certificate, cache.Key, nil } @@ -198,7 +200,7 @@ func loadCertificatesWithCache(log *logger.Logger, cache *tlsCache, keyPassPath return "", "", err } - cache.PassphrasePath = keyPassPath + cache.CacheKey = cacheKey cache.Certificate = cert cache.Key = key diff --git a/internal/pkg/agent/application/endpoint_component_modifier_test.go b/internal/pkg/agent/application/endpoint_component_modifier_test.go index 4e713559149..424b2709f52 100644 --- a/internal/pkg/agent/application/endpoint_component_modifier_test.go +++ b/internal/pkg/agent/application/endpoint_component_modifier_test.go @@ -448,11 +448,12 @@ func TestEndpointTLSComponentModifier_cache_miss(t *testing.T) { cache := tlsCache{ mu: &sync.Mutex{}, - PassphrasePath: "/old/path/to/key-passphrase", - Certificate: "cached certificate", - Key: "cached key", + CacheKey: "/old-cache-key", + Certificate: "cached certificate", + Key: "cached key", } pair, certPath, certKeyPath, certKeyPassPath := prepareEncTLSCertificates(t) + cackeKey := cache.MakeKey(certKeyPassPath, certPath, certKeyPath) comps := makeComponent(t, fmt.Sprintf(`{ "fleet": { @@ -485,7 +486,7 @@ func TestEndpointTLSComponentModifier_cache_miss(t *testing.T) { got, err := modifier(comps, cfg) require.NoError(t, err, "unexpected error") - assert.Equal(t, certKeyPassPath, cache.PassphrasePath, "passphrase path did not match") + assert.Equal(t, cackeKey, cache.CacheKey, "passphrase path did not match") assert.Equal(t, string(pair.Cert), cache.Certificate, "certificate did not match") assert.Equal(t, string(pair.Key), cache.Key, "key did not match") @@ -505,13 +506,15 @@ func TestEndpointTLSComponentModifier_cache_hit(t *testing.T) { certPath := "/path/to/cert" certKeyPath := "/path/to/key" certKeyPassPath := "/path/to/key_passphrase_path" //nolint:gosec // not a real key + cache := tlsCache{ mu: &sync.Mutex{}, - PassphrasePath: certKeyPassPath, - Certificate: "cached certificate", - Key: "cached key", + Certificate: "cached certificate", + Key: "cached key", } + cacheKey := cache.MakeKey(certKeyPassPath, certPath, certKeyPath) + cache.CacheKey = cacheKey comps := makeComponent(t, fmt.Sprintf(`{ "fleet": { @@ -527,7 +530,7 @@ func TestEndpointTLSComponentModifier_cache_hit(t *testing.T) { "ssl": map[string]interface{}{ "certificate": cache.Certificate, "key": cache.Key, - "key_passphrase_path": cache.PassphrasePath, + "key_passphrase_path": cache.CacheKey, }, }, } @@ -545,7 +548,7 @@ func TestEndpointTLSComponentModifier_cache_hit(t *testing.T) { got, err := modifier(comps, cfg) require.NoError(t, err, "unexpected error") - assert.Equal(t, certKeyPassPath, cache.PassphrasePath, "passphrase should not have changed") + assert.Equal(t, cacheKey, cache.CacheKey, "passphrase should not have changed") compareComponents(t, got, wantComps) }