Utilise the migrate template API when applying changes to the deploym…

…ent template id (elastic#547)

* Utilise the migrate template API when applying changes to the deployment template id

* Add changelog

* Fix acceptance tests

* Reset the instance_configuration_id so changes from the migration API are picked up

.changelog/547.txt

@@ -0,0 +1,3 @@
+```release-note:feature
+resource/deployment: Utilise the template migration API to build the base update request when changing `deployment_template_id`. This results in more reliable changes between deployment templates.
+```

ec/acc/deployment_template_migration_test.go

@@ -0,0 +1,96 @@
+// Licensed to Elasticsearch B.V. under one or more contributor
+// license agreements. See the NOTICE file distributed with
+// this work for additional information regarding copyright
+// ownership. Elasticsearch B.V. licenses this file to you under
+// the Apache License, Version 2.0 (the "License"); you may
+// not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package acc
+
+import (
+	"testing"
+
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/acctest"
+	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource"
+)
+
+func TestAccDeployment_template_migration(t *testing.T) {
+	resName := "ec_deployment.compute_optimized"
+	randomName := prefix + acctest.RandStringFromCharSet(10, acctest.CharSetAlphaNum)
+	basicCfg := "testdata/deployment_compute_optimized_1.tf"
+	region := getRegion()
+	cfg := fixtureAccDeploymentResourceBasicDefaults(t, basicCfg, randomName, region, computeOpTemplate)
+	secondConfigCfg := fixtureAccDeploymentResourceBasicDefaults(t, basicCfg, randomName, region, memoryOpTemplate)
+
+	resource.ParallelTest(t, resource.TestCase{
+		PreCheck:          func() { testAccPreCheck(t) },
+		ProviderFactories: testAccProviderFactory,
+		CheckDestroy:      testAccDeploymentDestroy,
+		Steps: []resource.TestStep{
+			{
+				// Create a Compute Optimized deployment with the default settings.
+				Config: cfg,
+				Check: resource.ComposeAggregateTestCheckFunc(
+					resource.TestCheckResourceAttr(resName, "deployment_template_id", setDefaultTemplate(region, computeOpTemplate)),
+					resource.TestCheckResourceAttr(resName, "elasticsearch.#", "1"),
+					resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.#", "1"),
+					resource.TestCheckResourceAttrSet(resName, "elasticsearch.0.topology.0.instance_configuration_id"),
+					resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.0.size", "8g"),
+					resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.0.size_resource", "memory"),
+					resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.0.node_type_data", ""),
+					resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.0.node_type_ingest", ""),
+					resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.0.node_type_master", ""),
+					resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.0.node_type_ml", ""),
+					resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.0.id", "hot_content"),
+					resource.TestCheckResourceAttrSet(resName, "elasticsearch.0.topology.0.node_roles.#"),
+					resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.0.zone_count", "2"),
+					resource.TestCheckResourceAttr(resName, "kibana.#", "1"),
+					resource.TestCheckResourceAttr(resName, "kibana.0.topology.0.zone_count", "1"),
+					resource.TestCheckResourceAttr(resName, "kibana.0.topology.#", "1"),
+					resource.TestCheckResourceAttrSet(resName, "kibana.0.topology.0.instance_configuration_id"),
+					resource.TestCheckResourceAttr(resName, "kibana.0.topology.0.size", "1g"),
+					resource.TestCheckResourceAttr(resName, "kibana.0.topology.0.size_resource", "memory"),
+					resource.TestCheckResourceAttr(resName, "apm.#", "0"),
+					resource.TestCheckResourceAttr(resName, "enterprise_search.#", "0"),
+				),
+			},
+			{
+				// Change the deployment to memory optimized
+				Config: secondConfigCfg,
+				Check: resource.ComposeAggregateTestCheckFunc(
+					resource.TestCheckResourceAttr(resName, "deployment_template_id", setDefaultTemplate(region, memoryOpTemplate)),
+					resource.TestCheckResourceAttr(resName, "elasticsearch.#", "1"),
+					resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.#", "1"),
+					resource.TestCheckResourceAttrSet(resName, "elasticsearch.0.topology.0.instance_configuration_id"),
+					resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.0.size", "8g"),
+					resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.0.size_resource", "memory"),
+					resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.0.node_type_data", ""),
+					resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.0.node_type_ingest", ""),
+					resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.0.node_type_master", ""),
+					resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.0.node_type_ml", ""),
+					resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.0.id", "hot_content"),
+					resource.TestCheckResourceAttrSet(resName, "elasticsearch.0.topology.0.node_roles.#"),
+					resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.0.zone_count", "2"),
+					resource.TestCheckResourceAttr(resName, "kibana.#", "1"),
+					resource.TestCheckResourceAttr(resName, "kibana.0.topology.0.zone_count", "1"),
+					resource.TestCheckResourceAttr(resName, "kibana.0.topology.#", "1"),
+					resource.TestCheckResourceAttrSet(resName, "kibana.0.topology.0.instance_configuration_id"),
+					resource.TestCheckResourceAttr(resName, "kibana.0.topology.0.size", "1g"),
+					resource.TestCheckResourceAttr(resName, "kibana.0.topology.0.size_resource", "memory"),
+					resource.TestCheckResourceAttr(resName, "apm.#", "0"),
+					resource.TestCheckResourceAttr(resName, "enterprise_search.#", "0"),
+				),
+			},
+		},
+	})
+}

ec/acc/testdata/deployment_basic_defaults_hw_2.tf

@@ -9,7 +9,16 @@ resource "ec_deployment" "defaults" {
   version                = data.ec_stack.latest.version
   deployment_template_id = "%s"
 
-  elasticsearch {}
+  elasticsearch {
+    topology {
+      id   = "hot_content"
+      size = "4g"
+    }
+    topology {
+      id   = "warm"
+      size = "4g"
+    }
+  }
 
   kibana {}
-}
+}

ec/ecresource/deploymentresource/apm_expanders.go

@@ -202,3 +202,12 @@ func apmResource(res *models.DeploymentTemplateInfoV2) *models.ApmPayload {
 	}
 	return res.DeploymentTemplate.Resources.Apm[0]
 }
+
+// apmResourceFromUpdate returns the ApmPayload from a deployment
+// update request or an empty version of the payload.
+func apmResourceFromUpdate(res *models.DeploymentUpdateResources) *models.ApmPayload {
+	if len(res.Apm) == 0 {
+		return nil
+	}
+	return res.Apm[0]
+}

ec/ecresource/deploymentresource/elasticsearch_expanders.go

@@ -374,20 +374,34 @@ func matchEsTopologyID(id string, topologies []*models.ElasticsearchClusterTopol
 	)
 }
 
+func emptyEsResource() *models.ElasticsearchPayload {
+	return &models.ElasticsearchPayload{
+		Plan: &models.ElasticsearchClusterPlan{
+			Elasticsearch: &models.ElasticsearchConfiguration{},
+		},
+		Settings: &models.ElasticsearchClusterSettings{},
+	}
+}
+
 // esResource returns the ElaticsearchPayload from a deployment
 // template or an empty version of the payload.
 func esResource(res *models.DeploymentTemplateInfoV2) *models.ElasticsearchPayload {
 	if len(res.DeploymentTemplate.Resources.Elasticsearch) == 0 {
-		return &models.ElasticsearchPayload{
-			Plan: &models.ElasticsearchClusterPlan{
-				Elasticsearch: &models.ElasticsearchConfiguration{},
-			},
-			Settings: &models.ElasticsearchClusterSettings{},
-		}
+		return emptyEsResource()
 	}
 	return res.DeploymentTemplate.Resources.Elasticsearch[0]
 }
 
+// esResourceFromUpdate returns the ElaticsearchPayload from a deployment
+// update request or an empty version of the payload.
+func esResourceFromUpdate(res *models.DeploymentUpdateResources) *models.ElasticsearchPayload {
+	if len(res.Elasticsearch) == 0 {
+		return emptyEsResource()
+	}
+
+	return res.Elasticsearch[0]
+}
+
 func unsetElasticsearchCuration(payload *models.ElasticsearchPayload) {
 	if payload.Plan.Elasticsearch != nil {
 		payload.Plan.Elasticsearch.Curation = nil

ec/ecresource/deploymentresource/enterprise_search_expanders.go

@@ -208,3 +208,12 @@ func essResource(res *models.DeploymentTemplateInfoV2) *models.EnterpriseSearchP
 	}
 	return res.DeploymentTemplate.Resources.EnterpriseSearch[0]
 }
+
+// essResourceFromUpdate returns the EnterpriseSearchPayload from a deployment
+// update request or an empty version of the payload.
+func essResourceFromUpdate(res *models.DeploymentUpdateResources) *models.EnterpriseSearchPayload {
+	if len(res.EnterpriseSearch) == 0 {
+		return nil
+	}
+	return res.EnterpriseSearch[0]
+}

ec/ecresource/deploymentresource/expanders.go

@@ -24,6 +24,7 @@ import (
 	semver "github.com/blang/semver/v4"
 	"github.com/elastic/cloud-sdk-go/pkg/api"
 	"github.com/elastic/cloud-sdk-go/pkg/api/deploymentapi/deptemplateapi"
+	"github.com/elastic/cloud-sdk-go/pkg/client/deployments"
 	"github.com/elastic/cloud-sdk-go/pkg/models"
 	"github.com/elastic/cloud-sdk-go/pkg/multierror"
 	"github.com/elastic/cloud-sdk-go/pkg/util/ec"
@@ -123,6 +124,70 @@ func createResourceToModel(d *schema.ResourceData, client *api.API) (*models.Dep
 	return &result, nil
 }
 
+func getBaseUpdatePayloads(d *schema.ResourceData, client *api.API) (*models.DeploymentUpdateResources, error) {
+	prevDtId, dtIdIf := d.GetChange("deployment_template_id")
+	dtId := dtIdIf.(string)
+	template, err := deptemplateapi.Get(deptemplateapi.GetParams{
+		API:                        client,
+		TemplateID:                 dtId,
+		Region:                     d.Get("region").(string),
+		HideInstanceConfigurations: true,
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	baseUpdatePayloads := &models.DeploymentUpdateResources{
+		Apm:                template.DeploymentTemplate.Resources.Apm,
+		Appsearch:          template.DeploymentTemplate.Resources.Appsearch,
+		Elasticsearch:      template.DeploymentTemplate.Resources.Elasticsearch,
+		EnterpriseSearch:   template.DeploymentTemplate.Resources.EnterpriseSearch,
+		IntegrationsServer: template.DeploymentTemplate.Resources.IntegrationsServer,
+		Kibana:             template.DeploymentTemplate.Resources.Kibana,
+	}
+
+	// If the deployment template has changed then we should use the template migration API
+	// to build the base update payloads
+	if d.HasChange("deployment_template_id") && prevDtId.(string) != "" {
+		// Get an update request from the template migration API
+		migrateUpdateRequest, err := client.V1API.Deployments.MigrateDeploymentTemplate(
+			deployments.NewMigrateDeploymentTemplateParams().WithDeploymentID(d.Id()).WithTemplateID(dtId),
+			client.AuthWriter,
+		)
+
+		if err != nil {
+			return nil, err
+		}
+
+		if len(migrateUpdateRequest.Payload.Resources.Apm) > 0 {
+			baseUpdatePayloads.Apm = migrateUpdateRequest.Payload.Resources.Apm
+		}
+
+		if len(migrateUpdateRequest.Payload.Resources.Appsearch) > 0 {
+			baseUpdatePayloads.Appsearch = migrateUpdateRequest.Payload.Resources.Appsearch
+		}
+
+		if len(migrateUpdateRequest.Payload.Resources.Elasticsearch) > 0 {
+			baseUpdatePayloads.Elasticsearch = migrateUpdateRequest.Payload.Resources.Elasticsearch
+		}
+
+		if len(migrateUpdateRequest.Payload.Resources.EnterpriseSearch) > 0 {
+			baseUpdatePayloads.EnterpriseSearch = migrateUpdateRequest.Payload.Resources.EnterpriseSearch
+		}
+
+		if len(migrateUpdateRequest.Payload.Resources.IntegrationsServer) > 0 {
+			baseUpdatePayloads.IntegrationsServer = migrateUpdateRequest.Payload.Resources.IntegrationsServer
+		}
+
+		if len(migrateUpdateRequest.Payload.Resources.Kibana) > 0 {
+			baseUpdatePayloads.Kibana = migrateUpdateRequest.Payload.Resources.Kibana
+		}
+	}
+
+	return baseUpdatePayloads, nil
+}
+
 func updateResourceToModel(d *schema.ResourceData, client *api.API) (*models.DeploymentUpdateRequest, error) {
 	var result = models.DeploymentUpdateRequest{
 		Name:         d.Get("name").(string),
@@ -133,36 +198,24 @@ func updateResourceToModel(d *schema.ResourceData, client *api.API) (*models.Dep
 		Metadata:     &models.DeploymentUpdateMetadata{},
 	}
 
-	dtID := d.Get("deployment_template_id").(string)
-	version := d.Get("version").(string)
-	template, err := deptemplateapi.Get(deptemplateapi.GetParams{
-		API:                        client,
-		TemplateID:                 dtID,
-		Region:                     d.Get("region").(string),
-		HideInstanceConfigurations: true,
-	})
+	updatePayloads, err := getBaseUpdatePayloads(d, client)
 	if err != nil {
 		return nil, err
 	}
 
+	dtID := d.Get("deployment_template_id").(string)
+	version := d.Get("version").(string)
 	es := d.Get("elasticsearch").([]interface{})
 	kibana := d.Get("kibana").([]interface{})
 	apm := d.Get("apm").([]interface{})
 	integrationsServer := d.Get("integrations_server").([]interface{})
 	enterpriseSearch := d.Get("enterprise_search").([]interface{})
 
-	// When the deployment template is changed, we need to unset the missing
-	// resource topologies to account for a new instance_configuration_id and
-	// a different default value.
 	prevDT, _ := d.GetChange("deployment_template_id")
-	if d.HasChange("deployment_template_id") && prevDT.(string) != "" {
-		// If the deployment_template_id is changed, then we unset the
-		// Elasticsearch topology to account for the case where the
-		// instance_configuration_id changes, i.e. Hot / Warm, etc.
-
-		// This might not be necessary going forward as we move to
-		// tiered Elasticsearch nodes.
-		unsetTopology(es)
+	if d.HasChange("deployment_template_id") && prevDT != "" {
+		unsetInstanceConfigurations([][]interface{}{
+			es, kibana, apm, integrationsServer, enterpriseSearch,
+		})
 	}
 
 	useNodeRoles, err := compatibleWithNodeRoles(version)
@@ -178,7 +231,7 @@ func updateResourceToModel(d *schema.ResourceData, client *api.API) (*models.Dep
 	merr := multierror.NewPrefixed("invalid configuration")
 	esRes, err := expandEsResources(
 		es, enrichElasticsearchTemplate(
-			esResource(template), dtID, version, useNodeRoles,
+			esResourceFromUpdate(updatePayloads), dtID, version, useNodeRoles,
 		),
 	)
 	if err != nil {
@@ -191,25 +244,25 @@ func updateResourceToModel(d *schema.ResourceData, client *api.API) (*models.Dep
 	// to "partial".
 	ensurePartialSnapshotStrategy(esRes)
 
-	kibanaRes, err := expandKibanaResources(kibana, kibanaResource(template))
+	kibanaRes, err := expandKibanaResources(kibana, kibanaResourceFromUpdate(updatePayloads))
 	if err != nil {
 		merr = merr.Append(err)
 	}
 	result.Resources.Kibana = append(result.Resources.Kibana, kibanaRes...)
 
-	apmRes, err := expandApmResources(apm, apmResource(template))
+	apmRes, err := expandApmResources(apm, apmResourceFromUpdate(updatePayloads))
 	if err != nil {
 		merr = merr.Append(err)
 	}
 	result.Resources.Apm = append(result.Resources.Apm, apmRes...)
 
-	integrationsServerRes, err := expandIntegrationsServerResources(integrationsServer, integrationsServerResource(template))
+	integrationsServerRes, err := expandIntegrationsServerResources(integrationsServer, integrationsServerResourceFromUpdate(updatePayloads))
 	if err != nil {
 		merr = merr.Append(err)
 	}
 	result.Resources.IntegrationsServer = append(result.Resources.IntegrationsServer, integrationsServerRes...)
 
-	enterpriseSearchRes, err := expandEssResources(enterpriseSearch, essResource(template))
+	enterpriseSearchRes, err := expandEssResources(enterpriseSearch, essResourceFromUpdate(updatePayloads))
 	if err != nil {
 		merr = merr.Append(err)
 	}
@@ -237,6 +290,17 @@ func updateResourceToModel(d *schema.ResourceData, client *api.API) (*models.Dep
 	return &result, nil
 }
 
+func unsetInstanceConfigurations(rawResources [][]interface{}) {
+	for _, resource := range rawResources {
+		for _, r := range resource {
+			topologies := r.(map[string]interface{})["topology"].([]interface{})
+			for _, topology := range topologies {
+				delete(topology.(map[string]interface{}), "instance_configuration_id")
+			}
+		}
+	}
+}
+
 func enrichElasticsearchTemplate(tpl *models.ElasticsearchPayload, dt, version string, useNodeRoles bool) *models.ElasticsearchPayload {
 	if tpl.Plan.DeploymentTemplate == nil {
 		tpl.Plan.DeploymentTemplate = &models.DeploymentTemplateReference{}
@@ -261,12 +325,6 @@ func enrichElasticsearchTemplate(tpl *models.ElasticsearchPayload, dt, version s
 	return tpl
 }
 
-func unsetTopology(rawRes []interface{}) {
-	for _, r := range rawRes {
-		delete(r.(map[string]interface{}), "topology")
-	}
-}
-
 func expandTags(raw map[string]interface{}) []*models.MetadataItem {
 	result := make([]*models.MetadataItem, 0, len(raw))
 	for k, v := range raw {