From b2f3e20ace214ae8a8295646d2fc2c6228e3d423 Mon Sep 17 00:00:00 2001 From: Michal Pristas Date: Thu, 6 Oct 2022 17:56:24 +0200 Subject: [PATCH] Updated migration query to match items with deprecated field present (#1959) Co-authored-by: Anderson Queiroz --- internal/pkg/dl/migration.go | 15 +-- internal/pkg/dl/migration_integration_test.go | 97 +++++++++++++++---- 2 files changed, 88 insertions(+), 24 deletions(-) diff --git a/internal/pkg/dl/migration.go b/internal/pkg/dl/migration.go index 48e169691..f38d3c770 100644 --- a/internal/pkg/dl/migration.go +++ b/internal/pkg/dl/migration.go @@ -217,13 +217,14 @@ func migrateToV8_5(ctx context.Context, bulker bulk.Bulk) error { // this change fixes. func migrateAgentOutputs() (string, string, []byte, error) { const ( - migrationName = "AgentOutputs" - fieldOutputs = "outputs" - fieldRetiredAt = "retiredAt" + migrationName = "AgentOutputs" + fieldOutputs = "outputs" + fieldDefaultAPIKeyID = "default_api_key_id" // nolint:gosec,G101 // this is not a credential + fieldRetiredAt = "retiredAt" ) query := dsl.NewRoot() - query.Query().Bool().MustNot().Exists(fieldOutputs) + query.Query().Bool().Must().Exists(fieldDefaultAPIKeyID) fields := map[string]interface{}{fieldRetiredAt: timeNow().UTC().Format(time.RFC3339)} painless := ` @@ -253,9 +254,9 @@ ctx._source['` + fieldOutputs + `']['default'].permissions_hash=ctx._source.poli // Erase deprecated fields ctx._source.default_api_key_history=null; -ctx._source.default_api_key=""; -ctx._source.default_api_key_id=""; -ctx._source.policy_output_permissions_hash=""; +ctx._source.default_api_key=null; +ctx._source.default_api_key_id=null; +ctx._source.policy_output_permissions_hash=null; ` query.Param("script", map[string]interface{}{ "lang": "painless", diff --git a/internal/pkg/dl/migration_integration_test.go b/internal/pkg/dl/migration_integration_test.go index 55a2e61d7..fdbfd8a7e 100644 --- a/internal/pkg/dl/migration_integration_test.go +++ b/internal/pkg/dl/migration_integration_test.go @@ -155,12 +155,29 @@ func TestMigrateOutputs_withDefaultAPIKeyHistory(t *testing.T) { for i, id := range agentIDs { wantOutputType := "elasticsearch" //nolint:goconst // test cases have some duplication - got, err := FindAgent( - context.Background(), bulker, QueryAgentByID, FieldID, id, WithIndexName(index)) - if err != nil { - assert.NoError(t, err, "failed to find agent ID %q", id) // we want to continue even if a single agent fails - continue - } + res, err := SearchWithOneParam(context.Background(), bulker, QueryAgentByID, index, FieldID, id) + require.NoError(t, err) + require.Len(t, res.Hits, 1) + + var got model.Agent + err = res.Hits[0].Unmarshal(&got) + require.NoError(t, err, "could not unmarshal ES document into model.Agent") + + gotDeprecatedFields := struct { + // Deprecated. Use Outputs instead. API key the Elastic Agent uses to authenticate with elasticsearch + DefaultAPIKey *string `json:"default_api_key,omitempty"` + + // Deprecated. Use Outputs instead. Default API Key History + DefaultAPIKeyHistory []model.ToRetireAPIKeyIdsItems `json:"default_api_key_history,omitempty"` + + // Deprecated. Use Outputs instead. ID of the API key the Elastic Agent uses to authenticate with elasticsearch + DefaultAPIKeyID *string `json:"default_api_key_id,omitempty"` + + // Deprecated. Use Outputs instead. The policy output permissions hash + PolicyOutputPermissionsHash *string `json:"policy_output_permissions_hash,omitempty"` + }{} + err = res.Hits[0].Unmarshal(&gotDeprecatedFields) + require.NoError(t, err, "could not unmarshal ES document into gotDeprecatedFields") wantToRetireAPIKeyIds := []model.ToRetireAPIKeyIdsItems{ { @@ -196,11 +213,36 @@ func TestMigrateOutputs_withDefaultAPIKeyHistory(t *testing.T) { } // Assert deprecated fields - assert.Empty(t, got.DefaultAPIKey) - assert.Empty(t, got.DefaultAPIKey) - assert.Empty(t, got.PolicyOutputPermissionsHash) - assert.Nil(t, got.DefaultAPIKeyHistory) + assert.Nil(t, gotDeprecatedFields.DefaultAPIKey) + assert.Nil(t, gotDeprecatedFields.DefaultAPIKeyID) + assert.Nil(t, gotDeprecatedFields.PolicyOutputPermissionsHash) + assert.Nil(t, gotDeprecatedFields.DefaultAPIKeyHistory) + } +} + +func TestMigrateOutputs_dontMigrateTwice(t *testing.T) { + now, err := time.Parse(time.RFC3339, nowStr) + require.NoError(t, err, "could not parse time "+nowStr) + timeNow = func() time.Time { + return now + } + + index, bulker := ftesting.SetupCleanIndex(context.Background(), t, FleetAgents) + apiKey := bulk.APIKey{ + ID: "testAgent_", + Key: "testAgent_key_", } + + agentIDs := createSomeAgents(t, 25, apiKey, index, bulker) + + migratedAgents, err := migrate(context.Background(), bulker, migrateAgentOutputs) + require.NoError(t, err) + assert.Equal(t, len(agentIDs), migratedAgents) + + migratedAgents2, err := migrate(context.Background(), bulker, migrateAgentOutputs) + require.NoError(t, err) + + assert.Equal(t, 0, migratedAgents2) } func TestMigrateOutputs_nil_DefaultAPIKeyHistory(t *testing.T) { @@ -249,9 +291,30 @@ func TestMigrateOutputs_nil_DefaultAPIKeyHistory(t *testing.T) { migratedAgents, err := migrate(context.Background(), bulker, migrateAgentOutputs) require.NoError(t, err) - got, err := FindAgent( - context.Background(), bulker, QueryAgentByID, FieldID, agentID, WithIndexName(index)) - require.NoError(t, err, "failed to find agent ID %q", agentID) // we want to continue even if a single agent fails + res, err := SearchWithOneParam( + context.Background(), bulker, QueryAgentByID, index, FieldID, agentID) + require.NoError(t, err, "failed to find agent ID %q", agentID) + require.Len(t, res.Hits, 1) + + var got model.Agent + err = res.Hits[0].Unmarshal(&got) + require.NoError(t, err, "could not unmarshal ES document into model.Agent") + + gotDeprecatedFields := struct { + // Deprecated. Use Outputs instead. API key the Elastic Agent uses to authenticate with elasticsearch + DefaultAPIKey *string `json:"default_api_key,omitempty"` + + // Deprecated. Use Outputs instead. Default API Key History + DefaultAPIKeyHistory []model.ToRetireAPIKeyIdsItems `json:"default_api_key_history,omitempty"` + + // Deprecated. Use Outputs instead. ID of the API key the Elastic Agent uses to authenticate with elasticsearch + DefaultAPIKeyID *string `json:"default_api_key_id,omitempty"` + + // Deprecated. Use Outputs instead. The policy output permissions hash + PolicyOutputPermissionsHash *string `json:"policy_output_permissions_hash,omitempty"` + }{} + err = res.Hits[0].Unmarshal(&gotDeprecatedFields) + require.NoError(t, err, "could not unmarshal ES document into gotDeprecatedFields") assert.Equal(t, 1, migratedAgents) @@ -273,10 +336,10 @@ func TestMigrateOutputs_nil_DefaultAPIKeyHistory(t *testing.T) { } // Assert deprecated fields - assert.Empty(t, got.DefaultAPIKey) - assert.Empty(t, got.DefaultAPIKey) - assert.Empty(t, got.PolicyOutputPermissionsHash) - assert.Nil(t, got.DefaultAPIKeyHistory) + assert.Nil(t, gotDeprecatedFields.DefaultAPIKey) + assert.Nil(t, gotDeprecatedFields.DefaultAPIKey) + assert.Nil(t, gotDeprecatedFields.PolicyOutputPermissionsHash) + assert.Nil(t, gotDeprecatedFields.DefaultAPIKeyHistory) } func TestMigrateOutputs_no_agent_document(t *testing.T) {