From e013cd5a663d10ddf9fe96fbf8c2217b3895100e Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Thu, 6 Oct 2022 14:18:00 +0200 Subject: [PATCH 1/8] updated query --- internal/pkg/dl/migration.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/internal/pkg/dl/migration.go b/internal/pkg/dl/migration.go index 48e169691..9e3b4ba1a 100644 --- a/internal/pkg/dl/migration.go +++ b/internal/pkg/dl/migration.go @@ -217,13 +217,14 @@ func migrateToV8_5(ctx context.Context, bulker bulk.Bulk) error { // this change fixes. func migrateAgentOutputs() (string, string, []byte, error) { const ( - migrationName = "AgentOutputs" - fieldOutputs = "outputs" - fieldRetiredAt = "retiredAt" + migrationName = "AgentOutputs" + fieldOutputs = "outputs" + fieldDefaultAPIKeyID = "default_api_key_id" + fieldRetiredAt = "retiredAt" ) query := dsl.NewRoot() - query.Query().Bool().MustNot().Exists(fieldOutputs) + query.Query().Bool().Must().Exists(fieldDefaultAPIKeyID) fields := map[string]interface{}{fieldRetiredAt: timeNow().UTC().Format(time.RFC3339)} painless := ` From a2753f033b59083868e7adf0dedf1f84af8290d3 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Thu, 6 Oct 2022 14:33:19 +0200 Subject: [PATCH 2/8] lint --- internal/pkg/dl/migration.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/pkg/dl/migration.go b/internal/pkg/dl/migration.go index 9e3b4ba1a..a957b85d1 100644 --- a/internal/pkg/dl/migration.go +++ b/internal/pkg/dl/migration.go @@ -219,7 +219,7 @@ func migrateAgentOutputs() (string, string, []byte, error) { const ( migrationName = "AgentOutputs" fieldOutputs = "outputs" - fieldDefaultAPIKeyID = "default_api_key_id" + fieldDefaultAPIKeyID = "default_api_key_id" // nolint:gosec,G101 // this is not a credential fieldRetiredAt = "retiredAt" ) From b5580ccaa324e3391b9cf6f4190a37d0e3d8af2d Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Thu, 6 Oct 2022 15:00:28 +0200 Subject: [PATCH 3/8] i dont know what i'm doing --- internal/pkg/dl/migration.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/internal/pkg/dl/migration.go b/internal/pkg/dl/migration.go index a957b85d1..35cabeffe 100644 --- a/internal/pkg/dl/migration.go +++ b/internal/pkg/dl/migration.go @@ -223,8 +223,10 @@ func migrateAgentOutputs() (string, string, []byte, error) { fieldRetiredAt = "retiredAt" ) - query := dsl.NewRoot() - query.Query().Bool().Must().Exists(fieldDefaultAPIKeyID) + root := dsl.NewRoot() + tmpl := dsl.NewTmpl() + + root.Query().Bool().Must().Term(fieldDefaultAPIKeyID, tmpl.Bind(fieldDefaultAPIKeyID), nil) fields := map[string]interface{}{fieldRetiredAt: timeNow().UTC().Format(time.RFC3339)} painless := ` @@ -258,13 +260,15 @@ ctx._source.default_api_key=""; ctx._source.default_api_key_id=""; ctx._source.policy_output_permissions_hash=""; ` - query.Param("script", map[string]interface{}{ + root.Param("script", map[string]interface{}{ "lang": "painless", "source": painless, "params": fields, }) - body, err := query.MarshalJSON() + tmpl.MustResolve(root) + + body, err := tmpl.Render(map[string]interface{}{fieldDefaultAPIKeyID: ""}) if err != nil { return migrationName, FleetAgents, nil, fmt.Errorf("could not marshal ES query: %w", err) } From 5df0c857fce91468289eb8036d612896bc80144e Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Thu, 6 Oct 2022 15:12:30 +0200 Subject: [PATCH 4/8] i dont know what i'm doing --- internal/pkg/dl/migration.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/pkg/dl/migration.go b/internal/pkg/dl/migration.go index 35cabeffe..97cc4a9df 100644 --- a/internal/pkg/dl/migration.go +++ b/internal/pkg/dl/migration.go @@ -226,7 +226,7 @@ func migrateAgentOutputs() (string, string, []byte, error) { root := dsl.NewRoot() tmpl := dsl.NewTmpl() - root.Query().Bool().Must().Term(fieldDefaultAPIKeyID, tmpl.Bind(fieldDefaultAPIKeyID), nil) + root.Query().Bool().MustNot().Term(fieldDefaultAPIKeyID, tmpl.Bind(fieldDefaultAPIKeyID), nil) fields := map[string]interface{}{fieldRetiredAt: timeNow().UTC().Format(time.RFC3339)} painless := ` From e392eb719f51b11d340cbc699fe950a3f8c41c95 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Thu, 6 Oct 2022 15:15:17 +0200 Subject: [PATCH 5/8] no template yaay --- internal/pkg/dl/migration.go | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/internal/pkg/dl/migration.go b/internal/pkg/dl/migration.go index 97cc4a9df..eaad643b1 100644 --- a/internal/pkg/dl/migration.go +++ b/internal/pkg/dl/migration.go @@ -223,10 +223,8 @@ func migrateAgentOutputs() (string, string, []byte, error) { fieldRetiredAt = "retiredAt" ) - root := dsl.NewRoot() - tmpl := dsl.NewTmpl() - - root.Query().Bool().MustNot().Term(fieldDefaultAPIKeyID, tmpl.Bind(fieldDefaultAPIKeyID), nil) + query := dsl.NewRoot() + query.Query().Bool().Must().Term(fieldDefaultAPIKeyID, "", nil) fields := map[string]interface{}{fieldRetiredAt: timeNow().UTC().Format(time.RFC3339)} painless := ` @@ -260,15 +258,13 @@ ctx._source.default_api_key=""; ctx._source.default_api_key_id=""; ctx._source.policy_output_permissions_hash=""; ` - root.Param("script", map[string]interface{}{ + query.Param("script", map[string]interface{}{ "lang": "painless", "source": painless, "params": fields, }) - tmpl.MustResolve(root) - - body, err := tmpl.Render(map[string]interface{}{fieldDefaultAPIKeyID: ""}) + body, err := query.MarshalJSON() if err != nil { return migrationName, FleetAgents, nil, fmt.Errorf("could not marshal ES query: %w", err) } From 31fe3193155078e84d98409f634f60298d930c9f Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Thu, 6 Oct 2022 15:15:29 +0200 Subject: [PATCH 6/8] no template yaay --- internal/pkg/dl/migration.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/pkg/dl/migration.go b/internal/pkg/dl/migration.go index eaad643b1..d59ff7a8b 100644 --- a/internal/pkg/dl/migration.go +++ b/internal/pkg/dl/migration.go @@ -224,7 +224,7 @@ func migrateAgentOutputs() (string, string, []byte, error) { ) query := dsl.NewRoot() - query.Query().Bool().Must().Term(fieldDefaultAPIKeyID, "", nil) + query.Query().Bool().MustNot().Term(fieldDefaultAPIKeyID, "", nil) fields := map[string]interface{}{fieldRetiredAt: timeNow().UTC().Format(time.RFC3339)} painless := ` From c60482c8c900150bdbb0521d64903884db26c5f8 Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Thu, 6 Oct 2022 15:44:49 +0200 Subject: [PATCH 7/8] set deprecated fields to null and test it --- internal/pkg/dl/migration.go | 6 +- internal/pkg/dl/migration_integration_test.go | 72 ++++++++++++++----- 2 files changed, 58 insertions(+), 20 deletions(-) diff --git a/internal/pkg/dl/migration.go b/internal/pkg/dl/migration.go index d59ff7a8b..16d684199 100644 --- a/internal/pkg/dl/migration.go +++ b/internal/pkg/dl/migration.go @@ -254,9 +254,9 @@ ctx._source['` + fieldOutputs + `']['default'].permissions_hash=ctx._source.poli // Erase deprecated fields ctx._source.default_api_key_history=null; -ctx._source.default_api_key=""; -ctx._source.default_api_key_id=""; -ctx._source.policy_output_permissions_hash=""; +ctx._source.default_api_key=null; +ctx._source.default_api_key_id=null; +ctx._source.policy_output_permissions_hash=null; ` query.Param("script", map[string]interface{}{ "lang": "painless", diff --git a/internal/pkg/dl/migration_integration_test.go b/internal/pkg/dl/migration_integration_test.go index 55a2e61d7..ac59ab708 100644 --- a/internal/pkg/dl/migration_integration_test.go +++ b/internal/pkg/dl/migration_integration_test.go @@ -155,12 +155,29 @@ func TestMigrateOutputs_withDefaultAPIKeyHistory(t *testing.T) { for i, id := range agentIDs { wantOutputType := "elasticsearch" //nolint:goconst // test cases have some duplication - got, err := FindAgent( - context.Background(), bulker, QueryAgentByID, FieldID, id, WithIndexName(index)) - if err != nil { - assert.NoError(t, err, "failed to find agent ID %q", id) // we want to continue even if a single agent fails - continue - } + res, err := SearchWithOneParam(context.Background(), bulker, QueryAgentByID, index, FieldID, id) + require.NoError(t, err) + require.Len(t, res.Hits, 1) + + var got model.Agent + err = res.Hits[0].Unmarshal(&got) + require.NoError(t, err, "could not unmarshal ES document into model.Agent") + + gotDeprecatedFields := struct { + // Deprecated. Use Outputs instead. API key the Elastic Agent uses to authenticate with elasticsearch + DefaultAPIKey *string `json:"default_api_key,omitempty"` + + // Deprecated. Use Outputs instead. Default API Key History + DefaultAPIKeyHistory []model.ToRetireAPIKeyIdsItems `json:"default_api_key_history,omitempty"` + + // Deprecated. Use Outputs instead. ID of the API key the Elastic Agent uses to authenticate with elasticsearch + DefaultAPIKeyID *string `json:"default_api_key_id,omitempty"` + + // Deprecated. Use Outputs instead. The policy output permissions hash + PolicyOutputPermissionsHash *string `json:"policy_output_permissions_hash,omitempty"` + }{} + err = res.Hits[0].Unmarshal(&gotDeprecatedFields) + require.NoError(t, err, "could not unmarshal ES document into gotDeprecatedFields") wantToRetireAPIKeyIds := []model.ToRetireAPIKeyIdsItems{ { @@ -196,10 +213,10 @@ func TestMigrateOutputs_withDefaultAPIKeyHistory(t *testing.T) { } // Assert deprecated fields - assert.Empty(t, got.DefaultAPIKey) - assert.Empty(t, got.DefaultAPIKey) - assert.Empty(t, got.PolicyOutputPermissionsHash) - assert.Nil(t, got.DefaultAPIKeyHistory) + assert.Nil(t, gotDeprecatedFields.DefaultAPIKey) + assert.Nil(t, gotDeprecatedFields.DefaultAPIKeyID) + assert.Nil(t, gotDeprecatedFields.PolicyOutputPermissionsHash) + assert.Nil(t, gotDeprecatedFields.DefaultAPIKeyHistory) } } @@ -249,9 +266,30 @@ func TestMigrateOutputs_nil_DefaultAPIKeyHistory(t *testing.T) { migratedAgents, err := migrate(context.Background(), bulker, migrateAgentOutputs) require.NoError(t, err) - got, err := FindAgent( - context.Background(), bulker, QueryAgentByID, FieldID, agentID, WithIndexName(index)) - require.NoError(t, err, "failed to find agent ID %q", agentID) // we want to continue even if a single agent fails + res, err := SearchWithOneParam( + context.Background(), bulker, QueryAgentByID, index, FieldID, agentID) + require.NoError(t, err, "failed to find agent ID %q", agentID) + require.Len(t, res.Hits, 1) + + var got model.Agent + err = res.Hits[0].Unmarshal(&got) + require.NoError(t, err, "could not unmarshal ES document into model.Agent") + + gotDeprecatedFields := struct { + // Deprecated. Use Outputs instead. API key the Elastic Agent uses to authenticate with elasticsearch + DefaultAPIKey *string `json:"default_api_key,omitempty"` + + // Deprecated. Use Outputs instead. Default API Key History + DefaultAPIKeyHistory []model.ToRetireAPIKeyIdsItems `json:"default_api_key_history,omitempty"` + + // Deprecated. Use Outputs instead. ID of the API key the Elastic Agent uses to authenticate with elasticsearch + DefaultAPIKeyID *string `json:"default_api_key_id,omitempty"` + + // Deprecated. Use Outputs instead. The policy output permissions hash + PolicyOutputPermissionsHash *string `json:"policy_output_permissions_hash,omitempty"` + }{} + err = res.Hits[0].Unmarshal(&gotDeprecatedFields) + require.NoError(t, err, "could not unmarshal ES document into gotDeprecatedFields") assert.Equal(t, 1, migratedAgents) @@ -273,10 +311,10 @@ func TestMigrateOutputs_nil_DefaultAPIKeyHistory(t *testing.T) { } // Assert deprecated fields - assert.Empty(t, got.DefaultAPIKey) - assert.Empty(t, got.DefaultAPIKey) - assert.Empty(t, got.PolicyOutputPermissionsHash) - assert.Nil(t, got.DefaultAPIKeyHistory) + assert.Nil(t, gotDeprecatedFields.DefaultAPIKey) + assert.Nil(t, gotDeprecatedFields.DefaultAPIKey) + assert.Nil(t, gotDeprecatedFields.PolicyOutputPermissionsHash) + assert.Nil(t, gotDeprecatedFields.DefaultAPIKeyHistory) } func TestMigrateOutputs_no_agent_document(t *testing.T) { From 786b6b4680825e3c2128c3c5e449c56465509eec Mon Sep 17 00:00:00 2001 From: Anderson Queiroz Date: Thu, 6 Oct 2022 15:55:01 +0200 Subject: [PATCH 8/8] query for not null DefaultAPIKeyID and ensure it does not migrate twice --- internal/pkg/dl/migration.go | 2 +- internal/pkg/dl/migration_integration_test.go | 25 +++++++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/internal/pkg/dl/migration.go b/internal/pkg/dl/migration.go index 16d684199..f38d3c770 100644 --- a/internal/pkg/dl/migration.go +++ b/internal/pkg/dl/migration.go @@ -224,7 +224,7 @@ func migrateAgentOutputs() (string, string, []byte, error) { ) query := dsl.NewRoot() - query.Query().Bool().MustNot().Term(fieldDefaultAPIKeyID, "", nil) + query.Query().Bool().Must().Exists(fieldDefaultAPIKeyID) fields := map[string]interface{}{fieldRetiredAt: timeNow().UTC().Format(time.RFC3339)} painless := ` diff --git a/internal/pkg/dl/migration_integration_test.go b/internal/pkg/dl/migration_integration_test.go index ac59ab708..fdbfd8a7e 100644 --- a/internal/pkg/dl/migration_integration_test.go +++ b/internal/pkg/dl/migration_integration_test.go @@ -220,6 +220,31 @@ func TestMigrateOutputs_withDefaultAPIKeyHistory(t *testing.T) { } } +func TestMigrateOutputs_dontMigrateTwice(t *testing.T) { + now, err := time.Parse(time.RFC3339, nowStr) + require.NoError(t, err, "could not parse time "+nowStr) + timeNow = func() time.Time { + return now + } + + index, bulker := ftesting.SetupCleanIndex(context.Background(), t, FleetAgents) + apiKey := bulk.APIKey{ + ID: "testAgent_", + Key: "testAgent_key_", + } + + agentIDs := createSomeAgents(t, 25, apiKey, index, bulker) + + migratedAgents, err := migrate(context.Background(), bulker, migrateAgentOutputs) + require.NoError(t, err) + assert.Equal(t, len(agentIDs), migratedAgents) + + migratedAgents2, err := migrate(context.Background(), bulker, migrateAgentOutputs) + require.NoError(t, err) + + assert.Equal(t, 0, migratedAgents2) +} + func TestMigrateOutputs_nil_DefaultAPIKeyHistory(t *testing.T) { wantOutputType := "elasticsearch"