From 705849fedde0e919a5e6b2b13b57466bb742d3ee Mon Sep 17 00:00:00 2001 From: Toby Brain Date: Mon, 10 Oct 2022 13:56:45 +1100 Subject: [PATCH 1/4] Utilise the migrate template API when applying changes to the deployment template id --- ec/acc/deployment_template_migration_test.go | 96 ++++++++++++++++ .../deploymentresource/apm_expanders.go | 9 ++ .../elasticsearch_expanders.go | 26 ++++- .../enterprise_search_expanders.go | 9 ++ ec/ecresource/deploymentresource/expanders.go | 106 ++++++++++++------ .../deploymentresource/expanders_test.go | 95 ++++------------ .../integrations_server_expanders.go | 9 ++ .../deploymentresource/kibana_expanders.go | 9 ++ ...nt-update-aws-cross-cluster-search-v2.json | 92 +++++++++++++++ .../deploymentresource/testutil_func.go | 16 +++ 10 files changed, 353 insertions(+), 114 deletions(-) create mode 100644 ec/acc/deployment_template_migration_test.go create mode 100644 ec/ecresource/deploymentresource/testdata/deployment-update-aws-cross-cluster-search-v2.json diff --git a/ec/acc/deployment_template_migration_test.go b/ec/acc/deployment_template_migration_test.go new file mode 100644 index 000000000..9b01f3d9d --- /dev/null +++ b/ec/acc/deployment_template_migration_test.go @@ -0,0 +1,96 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package acc + +import ( + "testing" + + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/acctest" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" +) + +func TestAccDeployment_template_migration(t *testing.T) { + resName := "ec_deployment.compute_optimized" + randomName := prefix + acctest.RandStringFromCharSet(10, acctest.CharSetAlphaNum) + basicCfg := "testdata/deployment_compute_optimized_1.tf" + region := getRegion() + cfg := fixtureAccDeploymentResourceBasicDefaults(t, basicCfg, randomName, region, computeOpTemplate) + secondConfigCfg := fixtureAccDeploymentResourceBasicDefaults(t, basicCfg, randomName, region, memoryOpTemplate) + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + ProviderFactories: testAccProviderFactory, + CheckDestroy: testAccDeploymentDestroy, + Steps: []resource.TestStep{ + { + // Create a Compute Optimized deployment with the default settings. + Config: cfg, + Check: resource.ComposeAggregateTestCheckFunc( + resource.TestCheckResourceAttr(resName, "deployment_template_id", setDefaultTemplate(region, computeOpTemplate)), + resource.TestCheckResourceAttr(resName, "elasticsearch.#", "1"), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.#", "1"), + resource.TestCheckResourceAttrSet(resName, "elasticsearch.0.topology.0.instance_configuration_id"), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.0.size", "8g"), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.0.size_resource", "memory"), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.0.node_type_data", ""), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.0.node_type_ingest", ""), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.0.node_type_master", ""), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.0.node_type_ml", ""), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.0.id", "hot_content"), + resource.TestCheckResourceAttrSet(resName, "elasticsearch.0.topology.0.node_roles.#"), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.0.zone_count", "2"), + resource.TestCheckResourceAttr(resName, "kibana.#", "1"), + resource.TestCheckResourceAttr(resName, "kibana.0.topology.0.zone_count", "1"), + resource.TestCheckResourceAttr(resName, "kibana.0.topology.#", "1"), + resource.TestCheckResourceAttrSet(resName, "kibana.0.topology.0.instance_configuration_id"), + resource.TestCheckResourceAttr(resName, "kibana.0.topology.0.size", "1g"), + resource.TestCheckResourceAttr(resName, "kibana.0.topology.0.size_resource", "memory"), + resource.TestCheckResourceAttr(resName, "apm.#", "0"), + resource.TestCheckResourceAttr(resName, "enterprise_search.#", "0"), + ), + }, + { + // Change the deployment to memory optimized + Config: secondConfigCfg, + Check: resource.ComposeAggregateTestCheckFunc( + resource.TestCheckResourceAttr(resName, "deployment_template_id", setDefaultTemplate(region, memoryOpTemplate)), + resource.TestCheckResourceAttr(resName, "elasticsearch.#", "1"), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.#", "1"), + resource.TestCheckResourceAttrSet(resName, "elasticsearch.0.topology.0.instance_configuration_id"), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.0.size", "8g"), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.0.size_resource", "memory"), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.0.node_type_data", ""), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.0.node_type_ingest", ""), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.0.node_type_master", ""), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.0.node_type_ml", ""), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.0.id", "hot_content"), + resource.TestCheckResourceAttrSet(resName, "elasticsearch.0.topology.0.node_roles.#"), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.0.zone_count", "2"), + resource.TestCheckResourceAttr(resName, "kibana.#", "1"), + resource.TestCheckResourceAttr(resName, "kibana.0.topology.0.zone_count", "1"), + resource.TestCheckResourceAttr(resName, "kibana.0.topology.#", "1"), + resource.TestCheckResourceAttrSet(resName, "kibana.0.topology.0.instance_configuration_id"), + resource.TestCheckResourceAttr(resName, "kibana.0.topology.0.size", "1g"), + resource.TestCheckResourceAttr(resName, "kibana.0.topology.0.size_resource", "memory"), + resource.TestCheckResourceAttr(resName, "apm.#", "0"), + resource.TestCheckResourceAttr(resName, "enterprise_search.#", "0"), + ), + }, + }, + }) +} diff --git a/ec/ecresource/deploymentresource/apm_expanders.go b/ec/ecresource/deploymentresource/apm_expanders.go index a86654b85..0c118723b 100644 --- a/ec/ecresource/deploymentresource/apm_expanders.go +++ b/ec/ecresource/deploymentresource/apm_expanders.go @@ -202,3 +202,12 @@ func apmResource(res *models.DeploymentTemplateInfoV2) *models.ApmPayload { } return res.DeploymentTemplate.Resources.Apm[0] } + +// apmResourceFromUpdate returns the ApmPayload from a deployment +// update request or an empty version of the payload. +func apmResourceFromUpdate(res *models.DeploymentUpdateResources) *models.ApmPayload { + if len(res.Apm) == 0 { + return nil + } + return res.Apm[0] +} diff --git a/ec/ecresource/deploymentresource/elasticsearch_expanders.go b/ec/ecresource/deploymentresource/elasticsearch_expanders.go index 18506823e..24da2e967 100644 --- a/ec/ecresource/deploymentresource/elasticsearch_expanders.go +++ b/ec/ecresource/deploymentresource/elasticsearch_expanders.go @@ -374,20 +374,34 @@ func matchEsTopologyID(id string, topologies []*models.ElasticsearchClusterTopol ) } +func emptyEsResource() *models.ElasticsearchPayload { + return &models.ElasticsearchPayload{ + Plan: &models.ElasticsearchClusterPlan{ + Elasticsearch: &models.ElasticsearchConfiguration{}, + }, + Settings: &models.ElasticsearchClusterSettings{}, + } +} + // esResource returns the ElaticsearchPayload from a deployment // template or an empty version of the payload. func esResource(res *models.DeploymentTemplateInfoV2) *models.ElasticsearchPayload { if len(res.DeploymentTemplate.Resources.Elasticsearch) == 0 { - return &models.ElasticsearchPayload{ - Plan: &models.ElasticsearchClusterPlan{ - Elasticsearch: &models.ElasticsearchConfiguration{}, - }, - Settings: &models.ElasticsearchClusterSettings{}, - } + return emptyEsResource() } return res.DeploymentTemplate.Resources.Elasticsearch[0] } +// esResourceFromUpdate returns the ElaticsearchPayload from a deployment +// update request or an empty version of the payload. +func esResourceFromUpdate(res *models.DeploymentUpdateResources) *models.ElasticsearchPayload { + if len(res.Elasticsearch) == 0 { + return emptyEsResource() + } + + return res.Elasticsearch[0] +} + func unsetElasticsearchCuration(payload *models.ElasticsearchPayload) { if payload.Plan.Elasticsearch != nil { payload.Plan.Elasticsearch.Curation = nil diff --git a/ec/ecresource/deploymentresource/enterprise_search_expanders.go b/ec/ecresource/deploymentresource/enterprise_search_expanders.go index 68a559443..01de017ed 100644 --- a/ec/ecresource/deploymentresource/enterprise_search_expanders.go +++ b/ec/ecresource/deploymentresource/enterprise_search_expanders.go @@ -208,3 +208,12 @@ func essResource(res *models.DeploymentTemplateInfoV2) *models.EnterpriseSearchP } return res.DeploymentTemplate.Resources.EnterpriseSearch[0] } + +// essResourceFromUpdate returns the EnterpriseSearchPayload from a deployment +// update request or an empty version of the payload. +func essResourceFromUpdate(res *models.DeploymentUpdateResources) *models.EnterpriseSearchPayload { + if len(res.EnterpriseSearch) == 0 { + return nil + } + return res.EnterpriseSearch[0] +} diff --git a/ec/ecresource/deploymentresource/expanders.go b/ec/ecresource/deploymentresource/expanders.go index 1f4048179..4f1411f28 100644 --- a/ec/ecresource/deploymentresource/expanders.go +++ b/ec/ecresource/deploymentresource/expanders.go @@ -24,6 +24,7 @@ import ( semver "github.com/blang/semver/v4" "github.com/elastic/cloud-sdk-go/pkg/api" "github.com/elastic/cloud-sdk-go/pkg/api/deploymentapi/deptemplateapi" + "github.com/elastic/cloud-sdk-go/pkg/client/deployments" "github.com/elastic/cloud-sdk-go/pkg/models" "github.com/elastic/cloud-sdk-go/pkg/multierror" "github.com/elastic/cloud-sdk-go/pkg/util/ec" @@ -123,6 +124,70 @@ func createResourceToModel(d *schema.ResourceData, client *api.API) (*models.Dep return &result, nil } +func getBaseUpdatePayloads(d *schema.ResourceData, client *api.API) (*models.DeploymentUpdateResources, error) { + prevDtId, dtIdIf := d.GetChange("deployment_template_id") + dtId := dtIdIf.(string) + template, err := deptemplateapi.Get(deptemplateapi.GetParams{ + API: client, + TemplateID: dtId, + Region: d.Get("region").(string), + HideInstanceConfigurations: true, + }) + + if err != nil { + return nil, err + } + + baseUpdatePayloads := &models.DeploymentUpdateResources{ + Apm: template.DeploymentTemplate.Resources.Apm, + Appsearch: template.DeploymentTemplate.Resources.Appsearch, + Elasticsearch: template.DeploymentTemplate.Resources.Elasticsearch, + EnterpriseSearch: template.DeploymentTemplate.Resources.EnterpriseSearch, + IntegrationsServer: template.DeploymentTemplate.Resources.IntegrationsServer, + Kibana: template.DeploymentTemplate.Resources.Kibana, + } + + // If the deployment template has changed then we should use the template migration API + // to build the base update payloads + if d.HasChange("deployment_template_id") && prevDtId.(string) != "" { + // Get an update request from the template migration API + migrateUpdateRequest, err := client.V1API.Deployments.MigrateDeploymentTemplate( + deployments.NewMigrateDeploymentTemplateParams().WithDeploymentID(d.Id()).WithTemplateID(dtId), + client.AuthWriter, + ) + + if err != nil { + return nil, err + } + + if len(migrateUpdateRequest.Payload.Resources.Apm) > 0 { + baseUpdatePayloads.Apm = migrateUpdateRequest.Payload.Resources.Apm + } + + if len(migrateUpdateRequest.Payload.Resources.Appsearch) > 0 { + baseUpdatePayloads.Appsearch = migrateUpdateRequest.Payload.Resources.Appsearch + } + + if len(migrateUpdateRequest.Payload.Resources.Elasticsearch) > 0 { + baseUpdatePayloads.Elasticsearch = migrateUpdateRequest.Payload.Resources.Elasticsearch + } + + if len(migrateUpdateRequest.Payload.Resources.EnterpriseSearch) > 0 { + baseUpdatePayloads.EnterpriseSearch = migrateUpdateRequest.Payload.Resources.EnterpriseSearch + } + + if len(migrateUpdateRequest.Payload.Resources.IntegrationsServer) > 0 { + baseUpdatePayloads.IntegrationsServer = migrateUpdateRequest.Payload.Resources.IntegrationsServer + } + + if len(migrateUpdateRequest.Payload.Resources.Kibana) > 0 { + baseUpdatePayloads.Kibana = migrateUpdateRequest.Payload.Resources.Kibana + } + } + + return baseUpdatePayloads, nil +} + func updateResourceToModel(d *schema.ResourceData, client *api.API) (*models.DeploymentUpdateRequest, error) { var result = models.DeploymentUpdateRequest{ Name: d.Get("name").(string), @@ -133,38 +198,19 @@ func updateResourceToModel(d *schema.ResourceData, client *api.API) (*models.Dep Metadata: &models.DeploymentUpdateMetadata{}, } - dtID := d.Get("deployment_template_id").(string) - version := d.Get("version").(string) - template, err := deptemplateapi.Get(deptemplateapi.GetParams{ - API: client, - TemplateID: dtID, - Region: d.Get("region").(string), - HideInstanceConfigurations: true, - }) + updatePayloads, err := getBaseUpdatePayloads(d, client) if err != nil { return nil, err } + dtID := d.Get("deployment_template_id").(string) + version := d.Get("version").(string) es := d.Get("elasticsearch").([]interface{}) kibana := d.Get("kibana").([]interface{}) apm := d.Get("apm").([]interface{}) integrationsServer := d.Get("integrations_server").([]interface{}) enterpriseSearch := d.Get("enterprise_search").([]interface{}) - // When the deployment template is changed, we need to unset the missing - // resource topologies to account for a new instance_configuration_id and - // a different default value. - prevDT, _ := d.GetChange("deployment_template_id") - if d.HasChange("deployment_template_id") && prevDT.(string) != "" { - // If the deployment_template_id is changed, then we unset the - // Elasticsearch topology to account for the case where the - // instance_configuration_id changes, i.e. Hot / Warm, etc. - - // This might not be necessary going forward as we move to - // tiered Elasticsearch nodes. - unsetTopology(es) - } - useNodeRoles, err := compatibleWithNodeRoles(version) if err != nil { return nil, err @@ -178,7 +224,7 @@ func updateResourceToModel(d *schema.ResourceData, client *api.API) (*models.Dep merr := multierror.NewPrefixed("invalid configuration") esRes, err := expandEsResources( es, enrichElasticsearchTemplate( - esResource(template), dtID, version, useNodeRoles, + esResourceFromUpdate(updatePayloads), dtID, version, useNodeRoles, ), ) if err != nil { @@ -191,25 +237,25 @@ func updateResourceToModel(d *schema.ResourceData, client *api.API) (*models.Dep // to "partial". ensurePartialSnapshotStrategy(esRes) - kibanaRes, err := expandKibanaResources(kibana, kibanaResource(template)) + kibanaRes, err := expandKibanaResources(kibana, kibanaResourceFromUpdate(updatePayloads)) if err != nil { merr = merr.Append(err) } result.Resources.Kibana = append(result.Resources.Kibana, kibanaRes...) - apmRes, err := expandApmResources(apm, apmResource(template)) + apmRes, err := expandApmResources(apm, apmResourceFromUpdate(updatePayloads)) if err != nil { merr = merr.Append(err) } result.Resources.Apm = append(result.Resources.Apm, apmRes...) - integrationsServerRes, err := expandIntegrationsServerResources(integrationsServer, integrationsServerResource(template)) + integrationsServerRes, err := expandIntegrationsServerResources(integrationsServer, integrationsServerResourceFromUpdate(updatePayloads)) if err != nil { merr = merr.Append(err) } result.Resources.IntegrationsServer = append(result.Resources.IntegrationsServer, integrationsServerRes...) - enterpriseSearchRes, err := expandEssResources(enterpriseSearch, essResource(template)) + enterpriseSearchRes, err := expandEssResources(enterpriseSearch, essResourceFromUpdate(updatePayloads)) if err != nil { merr = merr.Append(err) } @@ -261,12 +307,6 @@ func enrichElasticsearchTemplate(tpl *models.ElasticsearchPayload, dt, version s return tpl } -func unsetTopology(rawRes []interface{}) { - for _, r := range rawRes { - delete(r.(map[string]interface{}), "topology") - } -} - func expandTags(raw map[string]interface{}) []*models.MetadataItem { result := make([]*models.MetadataItem, 0, len(raw)) for k, v := range raw { diff --git a/ec/ecresource/deploymentresource/expanders_test.go b/ec/ecresource/deploymentresource/expanders_test.go index 9e60fe7ee..f5998888f 100644 --- a/ec/ecresource/deploymentresource/expanders_test.go +++ b/ec/ecresource/deploymentresource/expanders_test.go @@ -2956,6 +2956,9 @@ func Test_updateResourceToModel(t *testing.T) { ccsTpl := func() io.ReadCloser { return fileAsResponseBody(t, "testdata/template-aws-cross-cluster-search-v2.json") } + ccsDeploymentUpdate := func() io.ReadCloser { + return fileAsResponseBody(t, "testdata/deployment-update-aws-cross-cluster-search-v2.json") + } deploymentEmptyRDWithTemplateChange := util.NewResourceData(t, util.ResDataParams{ ID: mock.ValidClusterID, State: newSampleLegacyDeployment(), @@ -3018,9 +3021,6 @@ func Test_updateResourceToModel(t *testing.T) { Schema: newSchema(), }) - emptyTpl := func() io.ReadCloser { - return fileAsResponseBody(t, "testdata/template-empty.json") - } deploymentChangeFromExplicitSizingToEmpty := util.NewResourceData(t, util.ResDataParams{ ID: mock.ValidClusterID, State: map[string]interface{}{ @@ -3071,56 +3071,6 @@ func Test_updateResourceToModel(t *testing.T) { Schema: newSchema(), }) - deploymentChangeToEmptyDT := util.NewResourceData(t, util.ResDataParams{ - ID: mock.ValidClusterID, - State: map[string]interface{}{ - "name": "my_deployment_name", - "deployment_template_id": "aws-io-optimized-v2", - "region": "us-east-1", - "version": "7.9.2", - "elasticsearch": []interface{}{ - map[string]interface{}{ - "topology": []interface{}{map[string]interface{}{ - "id": "hot_content", - "size": "16g", - }}, - }, - map[string]interface{}{ - "topology": []interface{}{map[string]interface{}{ - "id": "coordinating", - "size": "16g", - }}, - }, - }, - "kibana": []interface{}{map[string]interface{}{ - "topology": []interface{}{map[string]interface{}{ - "size": "2g", - }}, - }}, - "apm": []interface{}{map[string]interface{}{ - "topology": []interface{}{map[string]interface{}{ - "size": "1g", - }}, - }}, - "enterprise_search": []interface{}{map[string]interface{}{ - "topology": []interface{}{map[string]interface{}{ - "size": "8g", - }}, - }}, - }, - Change: map[string]interface{}{ - "name": "my_deployment_name", - "deployment_template_id": "empty-deployment-template", - "region": "us-east-1", - "version": "7.9.2", - "elasticsearch": []interface{}{map[string]interface{}{}}, - "kibana": []interface{}{map[string]interface{}{}}, - "apm": []interface{}{map[string]interface{}{}}, - "enterprise_search": []interface{}{map[string]interface{}{}}, - }, - Schema: newSchema(), - }) - deploymentWithTags := util.NewResourceData(t, util.ResDataParams{ ID: mock.ValidClusterID, State: map[string]interface{}{ @@ -3699,8 +3649,11 @@ func Test_updateResourceToModel(t *testing.T) { { name: "toplogy change from hot / warm to cross cluster search", args: args{ - d: deploymentEmptyRDWithTemplateChange, - client: api.NewMock(mock.New200Response(ccsTpl())), + d: deploymentEmptyRDWithTemplateChange, + client: api.NewMock( + mock.New200Response(ccsTpl()), + mock.New200Response(ccsDeploymentUpdate()), + ), }, want: &models.DeploymentUpdateRequest{ Name: "my_deployment_name", @@ -3713,7 +3666,7 @@ func Test_updateResourceToModel(t *testing.T) { Tags: []*models.MetadataItem{}, }, Resources: &models.DeploymentUpdateResources{ - Elasticsearch: enrichWithEmptyTopologies(readerToESPayload(t, ccsTpl(), false), &models.ElasticsearchPayload{ + Elasticsearch: enrichWithEmptyTopologies(readerDeploymentUpdateToESPayload(t, ccsDeploymentUpdate(), false, "aws-cross-cluster-search-v2"), &models.ElasticsearchPayload{ Region: ec.String("us-east-1"), RefID: ec.String("main-elasticsearch"), Settings: &models.ElasticsearchClusterSettings{}, @@ -3726,16 +3679,18 @@ func Test_updateResourceToModel(t *testing.T) { }, ClusterTopology: []*models.ElasticsearchClusterTopologyElement{{ ID: "hot_content", + Elasticsearch: &models.ElasticsearchConfiguration{}, ZoneCount: 1, InstanceConfigurationID: "aws.ccs.r5d", Size: &models.TopologySize{ Resource: ec.String("memory"), - Value: ec.Int32(1024), + Value: ec.Int32(2048), }, NodeType: &models.ElasticsearchNodeType{ Data: ec.Bool(true), Ingest: ec.Bool(true), Master: ec.Bool(true), + Ml: ec.Bool(false), }, TopologyElementControl: &models.TopologyElementControl{ Min: &models.TopologySize{ @@ -3768,14 +3723,16 @@ func Test_updateResourceToModel(t *testing.T) { }, }, // The behavior of this change should be: - // * Resets the Elasticsearch topology: from 16g (due to unsetTopology call on DT change). // * Keeps the kibana toplogy size to 2g even though the topology element has been removed (saved value persists). // * Removes all other non present resources { name: "topology change with sizes not default from io optimized to cross cluster search", args: args{ - d: deploymentEmptyRDWithTemplateChangeWithDiffSize, - client: api.NewMock(mock.New200Response(ccsTpl())), + d: deploymentEmptyRDWithTemplateChangeWithDiffSize, + client: api.NewMock( + mock.New200Response(ccsTpl()), + mock.New200Response(ccsDeploymentUpdate()), + ), }, want: &models.DeploymentUpdateRequest{ Name: "my_deployment_name", @@ -3785,7 +3742,7 @@ func Test_updateResourceToModel(t *testing.T) { Tags: []*models.MetadataItem{}, }, Resources: &models.DeploymentUpdateResources{ - Elasticsearch: enrichWithEmptyTopologies(readerToESPayload(t, ccsTpl(), false), &models.ElasticsearchPayload{ + Elasticsearch: enrichWithEmptyTopologies(readerDeploymentUpdateToESPayload(t, ccsDeploymentUpdate(), false, "aws-cross-cluster-search-v2"), &models.ElasticsearchPayload{ Region: ec.String("us-east-1"), RefID: ec.String("main-elasticsearch"), Settings: &models.ElasticsearchClusterSettings{}, @@ -3798,12 +3755,12 @@ func Test_updateResourceToModel(t *testing.T) { }, ClusterTopology: []*models.ElasticsearchClusterTopologyElement{{ ID: "hot_content", + Elasticsearch: &models.ElasticsearchConfiguration{}, ZoneCount: 1, InstanceConfigurationID: "aws.ccs.r5d", Size: &models.TopologySize{ Resource: ec.String("memory"), - // This field's value is reset. - Value: ec.Int32(1024), + Value: ec.Int32(16384), }, NodeType: &models.ElasticsearchNodeType{ Data: ec.Bool(true), @@ -5029,18 +4986,6 @@ func Test_updateResourceToModel(t *testing.T) { }, }, }, - { - name: "topology change with invalid resources returns an error", - args: args{ - d: deploymentChangeToEmptyDT, - client: api.NewMock(mock.New200Response(emptyTpl())), - }, - err: multierror.NewPrefixed("invalid configuration", - errors.New("kibana specified but deployment template is not configured for it. Use a different template if you wish to add kibana"), - errors.New("apm specified but deployment template is not configured for it. Use a different template if you wish to add apm"), - errors.New("enterprise_search specified but deployment template is not configured for it. Use a different template if you wish to add enterprise_search"), - ), - }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/ec/ecresource/deploymentresource/integrations_server_expanders.go b/ec/ecresource/deploymentresource/integrations_server_expanders.go index d38fc64f0..de9718aa8 100644 --- a/ec/ecresource/deploymentresource/integrations_server_expanders.go +++ b/ec/ecresource/deploymentresource/integrations_server_expanders.go @@ -202,3 +202,12 @@ func integrationsServerResource(res *models.DeploymentTemplateInfoV2) *models.In } return res.DeploymentTemplate.Resources.IntegrationsServer[0] } + +// integrationsServerResourceFromUpdate returns the IntegrationsServerPayload from a deployment +// update request or an empty version of the payload. +func integrationsServerResourceFromUpdate(res *models.DeploymentUpdateResources) *models.IntegrationsServerPayload { + if len(res.IntegrationsServer) == 0 { + return nil + } + return res.IntegrationsServer[0] +} diff --git a/ec/ecresource/deploymentresource/kibana_expanders.go b/ec/ecresource/deploymentresource/kibana_expanders.go index 0133badb7..82dafc65e 100644 --- a/ec/ecresource/deploymentresource/kibana_expanders.go +++ b/ec/ecresource/deploymentresource/kibana_expanders.go @@ -192,3 +192,12 @@ func kibanaResource(res *models.DeploymentTemplateInfoV2) *models.KibanaPayload } return res.DeploymentTemplate.Resources.Kibana[0] } + +// kibanaResourceFromUpdate returns the KibanaPayload from a deployment +// update request or an empty version of the payload. +func kibanaResourceFromUpdate(res *models.DeploymentUpdateResources) *models.KibanaPayload { + if len(res.Kibana) == 0 { + return nil + } + return res.Kibana[0] +} diff --git a/ec/ecresource/deploymentresource/testdata/deployment-update-aws-cross-cluster-search-v2.json b/ec/ecresource/deploymentresource/testdata/deployment-update-aws-cross-cluster-search-v2.json new file mode 100644 index 000000000..30a46b06c --- /dev/null +++ b/ec/ecresource/deploymentresource/testdata/deployment-update-aws-cross-cluster-search-v2.json @@ -0,0 +1,92 @@ +{ + "resources": { + "apm": null, + "appsearch": null, + "elasticsearch": [ + { + "plan": { + "cluster_topology": [ + { + "id": "hot_content", + "instance_configuration_id": "aws.ccs.r5d", + "node_roles": [ + "master", + "ingest", + "remote_cluster_client", + "data_hot", + "transform", + "data_content" + ], + "node_type": { + "data": true, + "ingest": true, + "master": true + }, + "size": { + "resource": "memory", + "value": 1024 + }, + "topology_element_control": { + "min": { + "resource": "memory", + "value": 1024 + } + }, + "zone_count": 1 + }, + { + "id": "ml", + "instance_configuration_id": "aws.ml.m5d", + "node_roles": [ + "ml", + "remote_cluster_client" + ], + "node_type": { + "data": false, + "ingest": false, + "master": false, + "ml": true + }, + "size": { + "resource": "memory", + "value": 0 + }, + "topology_element_control": { + "min": { + "resource": "memory", + "value": 0 + } + }, + "zone_count": 1 + } + ], + "elasticsearch": {} + }, + "ref_id": "es-ref-id", + "region": "us-east-1", + "settings": {} + } + ], + "enterprise_search": null, + "kibana": [ + { + "elasticsearch_cluster_ref_id": "es-ref-id", + "plan": { + "cluster_topology": [ + { + "instance_configuration_id": "aws.kibana.r5d", + "size": { + "resource": "memory", + "value": 1024 + }, + "zone_count": 1 + } + ], + "kibana": {} + }, + "ref_id": "kibana-ref-id", + "region": "us-east-1" + } + ] + } +} \ No newline at end of file diff --git a/ec/ecresource/deploymentresource/testutil_func.go b/ec/ecresource/deploymentresource/testutil_func.go index c81799e72..945f45f1d 100644 --- a/ec/ecresource/deploymentresource/testutil_func.go +++ b/ec/ecresource/deploymentresource/testutil_func.go @@ -87,6 +87,22 @@ func enrichWithEmptyTopologies(tpl, want *models.ElasticsearchPayload) []*models return []*models.ElasticsearchPayload{tpl} } +func readerDeploymentUpdateToESPayload(t *testing.T, rc io.Reader, nr bool, tplID string) *models.ElasticsearchPayload { + t.Helper() + + var tpl models.DeploymentUpdateRequest + if err := json.NewDecoder(rc).Decode(&tpl); err != nil { + t.Fatal(err) + } + + return enrichElasticsearchTemplate( + tpl.Resources.Elasticsearch[0], + tplID, + "", + nr, + ) +} + func readerToESPayload(t *testing.T, rc io.Reader, nr bool) *models.ElasticsearchPayload { t.Helper() From 4157e74e45ae3b619b9edda703b74976a5bbeda1 Mon Sep 17 00:00:00 2001 From: Toby Brain Date: Mon, 10 Oct 2022 13:56:57 +1100 Subject: [PATCH 2/4] Add changelog --- .changelog/547.txt | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .changelog/547.txt diff --git a/.changelog/547.txt b/.changelog/547.txt new file mode 100644 index 000000000..4a383e3f1 --- /dev/null +++ b/.changelog/547.txt @@ -0,0 +1,3 @@ +```release-note:feature +resource/deployment: Utilise the template migration API to build the base update request when changing `deployment_template_id`. This results in more reliable changes between deployment templates. +``` From 72833d1ef6a3030979379a71d9517535d30ed0ad Mon Sep 17 00:00:00 2001 From: Toby Brain Date: Mon, 10 Oct 2022 20:19:08 +1100 Subject: [PATCH 3/4] Fix acceptance tests --- ec/acc/testdata/deployment_basic_defaults_hw_2.tf | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/ec/acc/testdata/deployment_basic_defaults_hw_2.tf b/ec/acc/testdata/deployment_basic_defaults_hw_2.tf index dba23d472..d3561ab25 100644 --- a/ec/acc/testdata/deployment_basic_defaults_hw_2.tf +++ b/ec/acc/testdata/deployment_basic_defaults_hw_2.tf @@ -9,7 +9,16 @@ resource "ec_deployment" "defaults" { version = data.ec_stack.latest.version deployment_template_id = "%s" - elasticsearch {} + elasticsearch { + topology { + id = "hot_content" + size = "4g" + } + topology { + id = "warm" + size = "4g" + } + } kibana {} -} \ No newline at end of file +} From a189ad61cac2715f5df24fc2dc85a2ecea782843 Mon Sep 17 00:00:00 2001 From: Toby Brain Date: Thu, 20 Oct 2022 19:55:35 +1100 Subject: [PATCH 4/4] Reset the instance_configuration_id so changes from the migration API are picked up --- ec/ecresource/deploymentresource/expanders.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/ec/ecresource/deploymentresource/expanders.go b/ec/ecresource/deploymentresource/expanders.go index 4f1411f28..2b28069ac 100644 --- a/ec/ecresource/deploymentresource/expanders.go +++ b/ec/ecresource/deploymentresource/expanders.go @@ -211,6 +211,13 @@ func updateResourceToModel(d *schema.ResourceData, client *api.API) (*models.Dep integrationsServer := d.Get("integrations_server").([]interface{}) enterpriseSearch := d.Get("enterprise_search").([]interface{}) + prevDT, _ := d.GetChange("deployment_template_id") + if d.HasChange("deployment_template_id") && prevDT != "" { + unsetInstanceConfigurations([][]interface{}{ + es, kibana, apm, integrationsServer, enterpriseSearch, + }) + } + useNodeRoles, err := compatibleWithNodeRoles(version) if err != nil { return nil, err @@ -283,6 +290,17 @@ func updateResourceToModel(d *schema.ResourceData, client *api.API) (*models.Dep return &result, nil } +func unsetInstanceConfigurations(rawResources [][]interface{}) { + for _, resource := range rawResources { + for _, r := range resource { + topologies := r.(map[string]interface{})["topology"].([]interface{}) + for _, topology := range topologies { + delete(topology.(map[string]interface{}), "instance_configuration_id") + } + } + } +} + func enrichElasticsearchTemplate(tpl *models.ElasticsearchPayload, dt, version string, useNodeRoles bool) *models.ElasticsearchPayload { if tpl.Plan.DeploymentTemplate == nil { tpl.Plan.DeploymentTemplate = &models.DeploymentTemplateReference{}