From f8bd0711199f8ffb0197babdb923551f747f8a27 Mon Sep 17 00:00:00 2001 From: Marc Lopez Date: Wed, 14 Apr 2021 11:15:44 +0500 Subject: [PATCH 01/12] Add ec_depoyment resource autoscaling logic Signed-off-by: Marc Lopez --- .../elasticsearch_expanders.go | 105 ++++++++++++++---- .../elasticsearch_flatteners.go | 45 +++++++- .../deploymentresource/flatteners.go | 5 +- .../schema_elasticsearch.go | 61 ++++++++++ 4 files changed, 187 insertions(+), 29 deletions(-) diff --git a/ec/ecresource/deploymentresource/elasticsearch_expanders.go b/ec/ecresource/deploymentresource/elasticsearch_expanders.go index 90398155c..145900953 100644 --- a/ec/ecresource/deploymentresource/elasticsearch_expanders.go +++ b/ec/ecresource/deploymentresource/elasticsearch_expanders.go @@ -20,9 +20,11 @@ package deploymentresource import ( "encoding/json" "fmt" + "reflect" "strconv" "strings" + "github.com/elastic/cloud-sdk-go/pkg/api/deploymentapi/deploymentsize" "github.com/elastic/cloud-sdk-go/pkg/models" "github.com/elastic/cloud-sdk-go/pkg/util/ec" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" @@ -107,13 +109,23 @@ func expandEsResource(raw interface{}, res *models.ElasticsearchPayload) (*model } } + if auto := es["autoscale"]; auto != nil { + if autoscale := auto.(string); autoscale != "" { + autoscaleBool, err := strconv.ParseBool(autoscale) + if err != nil { + return nil, fmt.Errorf("failed parsing autoscale value: %w", err) + } + res.Plan.AutoscalingEnabled = &autoscaleBool + } + } + return res, nil } // expandEsTopology expands a flattened topology func expandEsTopology(raw interface{}, topologies []*models.ElasticsearchClusterTopologyElement) ([]*models.ElasticsearchClusterTopologyElement, error) { rawTopologies := raw.([]interface{}) - res := make([]*models.ElasticsearchClusterTopologyElement, 0) + res := topologies for _, rawTop := range rawTopologies { topology := rawTop.(map[string]interface{}) @@ -136,12 +148,6 @@ func expandEsTopology(raw interface{}, topologies []*models.ElasticsearchCluster elem.Size = size } - // This check will most likely will need to be updated to handle - // autoscaling values. This is already the case for Machine Learning. - if sizeIsEmpty(elem.Size) { - return nil, fmt.Errorf("elasticsearch topology %s: size cannot be zero", topologyID) - } - if zones, ok := topology["zone_count"]; ok { if z := zones.(int); z > 0 { elem.ZoneCount = int32(z) @@ -159,7 +165,71 @@ func expandEsTopology(raw interface{}, topologies []*models.ElasticsearchCluster } } - res = append(res, elem) + if autoscalingRaw := topology["autoscaling"]; autoscalingRaw != nil { + for _, autoscaleRaw := range autoscalingRaw.([]interface{}) { + autoscale := autoscaleRaw.(map[string]interface{}) + if elem.AutoscalingMax == nil { + elem.AutoscalingMax = new(models.TopologySize) + } + + if elem.AutoscalingMin == nil { + elem.AutoscalingMin = new(models.TopologySize) + } + + if minSizeRes := autoscale["min_size_resource"]; minSizeRes != nil { + if minSize := minSizeRes.(string); minSize != "" { + elem.AutoscalingMin.Resource = ec.String(minSize) + } + } + + if minSize := autoscale["min_size"]; minSize != nil { + if minSize := minSize.(string); minSize != "" { + val, err := deploymentsize.ParseGb(minSize) + if err != nil { + return nil, err + } + elem.AutoscalingMin.Value = &val + } + } + + if maxSizeRes := autoscale["max_size_resource"]; maxSizeRes != nil { + if maxSize := maxSizeRes.(string); maxSize != "" { + elem.AutoscalingMax.Resource = ec.String(maxSize) + } + } + + if maxSize := autoscale["max_size"]; maxSize != nil { + if maxSize := maxSize.(string); maxSize != "" { + val, err := deploymentsize.ParseGb(maxSize) + if err != nil { + return nil, err + } + elem.AutoscalingMax.Value = &val + } + } + + // Ensure that if the Min and Max are empty, they're nil. + if reflect.DeepEqual(elem.AutoscalingMin, new(models.TopologySize)) { + elem.AutoscalingMin = nil + } + if reflect.DeepEqual(elem.AutoscalingMax, new(models.TopologySize)) { + elem.AutoscalingMax = nil + } + + if policy := autoscale["policy_override_json"]; policy != nil { + if policyString := policy.(string); policyString != "" { + if err := json.Unmarshal([]byte(policyString), + &elem.AutoscalingPolicyOverrideJSON, + ); err != nil { + return nil, fmt.Errorf( + "elasticsearch topology %s: unable to load policy_override_json: %w", + topologyID, err, + ) + } + } + } + } + } } return res, nil @@ -339,18 +409,6 @@ func parseLegacyNodeType(topology map[string]interface{}, nodeType *models.Elast return nil } -func sizeIsEmpty(size *models.TopologySize) bool { - if size == nil { - return true - } - - if size.Value == nil || *size.Value == 0 { - return true - } - - return false -} - func updateNodeRolesOnDedicatedTiers(topologies []*models.ElasticsearchClusterTopologyElement) { dataTier, hasMasterTier, hasIngestTier := dedicatedTopoogies(topologies) // This case is not very likely since all deployments will have a data tier. @@ -378,13 +436,14 @@ func dedicatedTopoogies(topologies []*models.ElasticsearchClusterTopologyElement var hasMasterRole bool var hasIngestRole bool for _, role := range topology.NodeRoles { - if strings.HasPrefix(role, dataTierRolePrefix) { + sizeNonZero := *topology.Size.Value > 0 + if strings.HasPrefix(role, dataTierRolePrefix) && sizeNonZero { hasSomeDataRole = true } - if role == ingestDataTierRole { + if role == ingestDataTierRole && sizeNonZero { hasIngestRole = true } - if role == masterDataTierRole { + if role == masterDataTierRole && sizeNonZero { hasMasterRole = true } } diff --git a/ec/ecresource/deploymentresource/elasticsearch_flatteners.go b/ec/ecresource/deploymentresource/elasticsearch_flatteners.go index aa5c292f2..5cb4015de 100644 --- a/ec/ecresource/deploymentresource/elasticsearch_flatteners.go +++ b/ec/ecresource/deploymentresource/elasticsearch_flatteners.go @@ -20,6 +20,7 @@ package deploymentresource import ( "bytes" "encoding/json" + "fmt" "sort" "strconv" @@ -31,7 +32,7 @@ import ( // flattenEsResources takes in Elasticsearch resource models and returns its // flattened form. -func flattenEsResources(in []*models.ElasticsearchResourceInfo, name string, remotes models.RemoteResources) []interface{} { +func flattenEsResources(in []*models.ElasticsearchResourceInfo, name string, remotes models.RemoteResources) ([]interface{}, error) { result := make([]interface{}, 0, len(in)) for _, res := range in { m := make(map[string]interface{}) @@ -52,10 +53,18 @@ func flattenEsResources(in []*models.ElasticsearchResourceInfo, name string, rem } plan := res.Info.PlanInfo.Current.Plan - if topology := flattenEsTopology(plan); len(topology) > 0 { + topology, err := flattenEsTopology(plan) + if err != nil { + return nil, err + } + if len(topology) > 0 { m["topology"] = topology } + if plan.AutoscalingEnabled != nil { + m["autoscale"] = strconv.FormatBool(*plan.AutoscalingEnabled) + } + if meta := res.Info.Metadata; meta != nil && meta.CloudID != "" { m["cloud_id"] = meta.CloudID } @@ -88,10 +97,10 @@ func flattenEsResources(in []*models.ElasticsearchResourceInfo, name string, rem result = append(result, m) } - return result + return result, nil } -func flattenEsTopology(plan *models.ElasticsearchClusterPlan) []interface{} { +func flattenEsTopology(plan *models.ElasticsearchClusterPlan) ([]interface{}, error) { result := make([]interface{}, 0, len(plan.ClusterTopology)) for _, topology := range plan.ClusterTopology { var m = make(map[string]interface{}) @@ -142,6 +151,32 @@ func flattenEsTopology(plan *models.ElasticsearchClusterPlan) []interface{} { )) } + autoscaling := make(map[string]interface{}) + if ascale := topology.AutoscalingMax; ascale != nil { + autoscaling["max_size_resource"] = *ascale.Resource + autoscaling["max_size"] = util.MemoryToState(*ascale.Value) + } + + if ascale := topology.AutoscalingMin; ascale != nil { + autoscaling["min_size_resource"] = *ascale.Resource + autoscaling["min_size"] = util.MemoryToState(*ascale.Value) + } + + if topology.AutoscalingPolicyOverrideJSON != nil { + b, err := json.Marshal(topology.AutoscalingPolicyOverrideJSON) + if err != nil { + return nil, fmt.Errorf( + "elasticsearch topology %s: unable to persist policy_override_json: %w", + topology.ID, err, + ) + } + autoscaling["policy_override_json"] = string(b) + } + + if len(autoscaling) > 0 { + m["autoscaling"] = []interface{}{autoscaling} + } + result = append(result, m) } @@ -151,7 +186,7 @@ func flattenEsTopology(plan *models.ElasticsearchClusterPlan) []interface{} { b := result[j].(map[string]interface{}) return a["id"].(string) < b["id"].(string) }) - return result + return result, nil } func flattenEsConfig(cfg *models.ElasticsearchConfiguration) []interface{} { diff --git a/ec/ecresource/deploymentresource/flatteners.go b/ec/ecresource/deploymentresource/flatteners.go index 0d1d1bd49..130b757de 100644 --- a/ec/ecresource/deploymentresource/flatteners.go +++ b/ec/ecresource/deploymentresource/flatteners.go @@ -58,7 +58,10 @@ func modelToState(d *schema.ResourceData, res *models.DeploymentGetResponse, rem return err } - esFlattened := flattenEsResources(res.Resources.Elasticsearch, *res.Name, remotes) + esFlattened, err := flattenEsResources(res.Resources.Elasticsearch, *res.Name, remotes) + if err != nil { + return err + } if err := d.Set("elasticsearch", esFlattened); err != nil { return err } diff --git a/ec/ecresource/deploymentresource/schema_elasticsearch.go b/ec/ecresource/deploymentresource/schema_elasticsearch.go index 18db979e0..3cc7e62c9 100644 --- a/ec/ecresource/deploymentresource/schema_elasticsearch.go +++ b/ec/ecresource/deploymentresource/schema_elasticsearch.go @@ -20,6 +20,7 @@ package deploymentresource import ( "bytes" "fmt" + "strconv" "github.com/elastic/cloud-sdk-go/pkg/util/slice" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" @@ -29,6 +30,21 @@ import ( func newElasticsearchResource() *schema.Resource { return &schema.Resource{ Schema: map[string]*schema.Schema{ + "autoscale": { + Type: schema.TypeString, + Description: `Enable or disable autoscaling. Defaults to the setting coming from the deployment template. Accepted values are "true" or "false".`, + Computed: true, + Optional: true, + ValidateFunc: func(i interface{}, s string) ([]string, []error) { + if _, err := strconv.ParseBool(i.(string)); err != nil { + return nil, []error{ + fmt.Errorf("failed parsing autoscale value: %w", err), + } + } + return nil, nil + }, + }, + "ref_id": { Type: schema.TypeString, Description: "Optional ref_id to set on the Elasticsearch resource", @@ -147,6 +163,51 @@ func elasticsearchTopologySchema() *schema.Schema { Type: schema.TypeString, }, }, + + "autoscaling": { + Type: schema.TypeList, + Description: "Optional Elasticsearch autoscaling settings, such a maximum and minimum size and resources.", + Optional: true, + Computed: true, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "max_size_resource": { + Description: "Maximum resource type for the maximum autoscaling setting.", + Type: schema.TypeString, + Optional: true, + Computed: true, + }, + + "max_size": { + Description: "Maximum size value for the maximum autoscaling setting.", + Type: schema.TypeString, + Optional: true, + Computed: true, + }, + + "min_size_resource": { + Description: "Minimum resource type for the minimum autoscaling setting.", + Type: schema.TypeString, + Optional: true, + Computed: true, + }, + + "min_size": { + Description: "Minimum size value for the minimum autoscaling setting.", + Type: schema.TypeString, + Optional: true, + Computed: true, + }, + + "policy_override_json": { + Type: schema.TypeString, + Description: "Computed policy overrides set directly via the API or other clients.", + Computed: true, + }, + }, + }, + }, }, }, } From 33b432f3013d7e79e2812d6cf9ad2daaa484b144 Mon Sep 17 00:00:00 2001 From: Marc Lopez Date: Wed, 14 Apr 2021 11:16:14 +0500 Subject: [PATCH 02/12] Add ec_deployment documentation Signed-off-by: Marc Lopez --- docs/resources/ec_deployment.md | 39 ++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/docs/resources/ec_deployment.md b/docs/resources/ec_deployment.md index 77301409e..cddbfa481 100644 --- a/docs/resources/ec_deployment.md +++ b/docs/resources/ec_deployment.md @@ -45,7 +45,7 @@ resource "ec_deployment" "example_minimal" { } ``` -### Hot warm cold tiered deployment +### Tiered deployment with Autoscaling enabled ```hcl data "ec_stack" "latest" { @@ -59,19 +59,26 @@ resource "ec_deployment" "example_minimal" { deployment_template_id = "aws-io-optimized-v2" elasticsearch { + autoscale = "true" + topology { - id = "hot_content" + id = "cold" size = "8g" } topology { - id = "warm" - size = "16g" + id = "hot_content" + size = "8g" + + autoscaling { + // Optionally change the policy max size. + // max_size = "29g" + } } topology { - id = "cold" - size = "8g" + id = "warm" + size = "16g" } } @@ -207,7 +214,7 @@ The following arguments are supported: ### Resources -!> **Warning on removing explicit topology objects** Due to current limitations, if a topology object is removed from the configuration, the removal won't trigger any changes since the field is optional and computed. There is no way to determine if the block was removed, which results in a _"sticky"_ topology configuration. +!> **Warning on removing explicit topology objects** Due to current limitations, if a topology object is removed from the configuration, the removal won't trigger any changes since the field is optional and computed. There is no way to determine if the block was removed, which results in a _"sticky"_ topology configuration. To disable a topology element, set the `topology.size` to `"0g"`. To create a valid deployment, you must specify at least the resource type `elasticsearch`. The supported resources are listed below. @@ -215,7 +222,7 @@ A default topology from the deployment template is used for empty blocks: `elast The `ec_deployment` resource will opt-out all the resources except Elasticsearch, which inherits the default topology from the deployment template. For example, the [I/O Optimized template includes an Elasticsearch cluster 8 GB memory x 2 availability zones](https://www.elastic.co/guide/en/cloud/current/ec-getting-started-profiles.html#ec-getting-started-profiles-io). -To customize the size or settings of the deployment resource, use the `topology` block within each resource kind block. +To customize the size or settings of the deployment resource, use the `topology` block within each resource kind block. The `topology` blocks are ordered lists and should be defined in the Terraform configuration in an ascending manner by alphabetical order of the `id` field. #### Elasticsearch @@ -227,6 +234,7 @@ The required `elasticsearch` block supports the following arguments: * `remote_cluster` (Optional) Elasticsearch remote clusters to configure for the Elasticsearch resource. Can be set multiple times. * `snapshot_source` (Optional) Restores data from a snapshot of another deployment. * `extension` (Optional) Custom Elasticsearch bundles or plugins. Can be set multiple times. +* `autoscale` (Optional) Enable or disable autoscaling. Defaults to the setting coming from the deployment template. Accepted values are `"true"` or `"false"`. ##### Topology @@ -244,9 +252,23 @@ The optional `elasticsearch.topology` block supports the following arguments: * `node_type_master` - (Optional) The node type for the Elasticsearch cluster (master node). * `node_type_ingest` - (Optional) The node type for the Elasticsearch cluster (ingest node). * `node_type_ml` - (Optional) The node type for the Elasticsearch cluster (machine learning node). +* `autoscaling` - (Optional) Autoscaling policy defining the maximum and / or minimum total size for this topology element. For more information refer to the `autoscaling` block. ~> **Note when node_type_* fields set** After upgrading to a version that supports data tiers (7.10.0 or above), the `node_type_*` has no effect even if specified. The provider automatically migrates the `node_type_*` fields to the appropriate `node_roles` as set by the deployment template. After having upgraded to `7.10.0` or above, the fields should be removed from the terraform configuration, if explicitly configured. +##### Autoscaling + +The optional `elasticsearch.autoscaling` block supports the following arguments: + +* `min_size` - (Optional) Defines the minimum size the deployment will scale down to. When set, scale down will be enabled, please note that not all the tiers support this option. +* `min_size_resource` - (Optional) Defines the resource type the scale down will use (Defaults to `"memory"`). +* `max_size` - (Optional) Defines the maximum size the deployment will scale up to. When set, scaling up will be enabled. All tiers should support this option. +* `max_size_resource` - (Optional) Defines the resource type the scale up will use (Defaults to `"memory"`). + +-> Note that none of these settings will take effect unless `elasticsearch.autoscale` is set to `"true"`. + +Please refer to the [Deployment Autoscaling](https://www.elastic.co/guide/en/cloud/current/ec-autoscaling.html) documentation for an updated list of the Elasticsearch tiers supporting scale up and scale down. + ##### Config The optional `elasticsearch.config` block supports the following arguments: @@ -391,6 +413,7 @@ In addition to all the arguments above, the following attributes are exported: * `elasticsearch.#.topology.#.node_type_ingest` - Node type (ingest) for the Elasticsearch topology element. * `elasticsearch.#.topology.#.node_type_ml` - Node type (machine learning) for the Elasticsearch topology element. * `elasticsearch.#.topology.#.node_roles` - List of roles for the topology element. They are inferred from the deployment template. +* `elasticsearch.#.topology.#.autoscaling.#.policy_override_json` - Computed policy overrides set directly via the API or other clients. * `elasticsearch.#.snapshot_source.#.source_elasticsearch_cluster_id` - ID of the Elasticsearch cluster that will be used as the source of the snapshot. * `elasticsearch.#.snapshot_source.#.snapshot_name` - Name of the snapshot to restore. * `kibana.#.resource_id` - Kibana resource unique identifier. From f5a2cea34ed3700af2c39ad6bd24b5a1a52737ba Mon Sep 17 00:00:00 2001 From: Marc Lopez Date: Wed, 14 Apr 2021 11:17:03 +0500 Subject: [PATCH 03/12] Add ec_deployment datasource logic and docs Signed-off-by: Marc Lopez --- docs/data-sources/ec_deployment.md | 6 +++ .../deploymentdatasource/datasource.go | 5 +- .../flatteners_elasticsearch.go | 48 +++++++++++++++++-- .../schema_elasticsearch.go | 44 +++++++++++++++++ 4 files changed, 97 insertions(+), 6 deletions(-) diff --git a/docs/data-sources/ec_deployment.md b/docs/data-sources/ec_deployment.md index d5c37b36b..8e481b377 100644 --- a/docs/data-sources/ec_deployment.md +++ b/docs/data-sources/ec_deployment.md @@ -38,6 +38,7 @@ These will not be available for interpolation. * `observability.#.logs` - Defines whether logs are enabled or disabled. * `observability.#.metrics` - Defines whether metrics are enabled or disabled. * `elasticsearch` - Instance configuration of the Elasticsearch resource kind. + * `elasticsearch.#.autoscale` - Whether or not Elasticsearch autoscaling is enabled. * `elasticsearch.#.healthy` - Resource kind health status. * `elasticsearch.#.cloud_id` - The encoded Elasticsearch credentials to use in Beats or Logstash. See [Configure Beats and Logstash with Cloud ID](https://www.elastic.co/guide/en/cloud/current/ec-cloud-id.html) for more information. * `elasticsearch.#.http_endpoint` - HTTP endpoint for the resource kind. @@ -55,6 +56,11 @@ These will not be available for interpolation. * `elasticsearch.#.topology.#.node_type_master` - Defines whether this node can be elected master (<7.10.0). * `elasticsearch.#.topology.#.node_type_ingest` - Defines whether this node can run an ingest pipeline (<7.10.0). * `elasticsearch.#.topology.#.node_type_ml` - Defines whether this node can run ML jobs (<7.10.0). + * `elasticsearch.#.topology.#.autoscaling.#.max_size` - The maximum size for the scale up policy. + * `elasticsearch.#.topology.#.autoscaling.#.max_size_resource` - The maximum size resource for the scale up policy. + * `elasticsearch.#.topology.#.autoscaling.#.min_size` - The minimum size for the scale down policy. + * `elasticsearch.#.topology.#.autoscaling.#.min_size_resource` - The minimum size for the scale down policy. + * `elasticsearch.#.topology.#.autoscaling.#.policy_override_json` - The advanced policy overrides for the autoscaling policy. * `kibana` - Instance configuration of the Kibana type. * `kibana.#.elasticsearch_cluster_ref_id` - The user-specified ID of the Elasticsearch cluster to which this resource kind will link. * `kibana.#.healthy` - Resource kind health status. diff --git a/ec/ecdatasource/deploymentdatasource/datasource.go b/ec/ecdatasource/deploymentdatasource/datasource.go index ba9811f6d..9c3ed7ca9 100644 --- a/ec/ecdatasource/deploymentdatasource/datasource.go +++ b/ec/ecdatasource/deploymentdatasource/datasource.go @@ -110,7 +110,10 @@ func modelToState(d *schema.ResourceData, res *models.DeploymentGetResponse) err } } - elasticsearchFlattened := flattenElasticsearchResources(res.Resources.Elasticsearch) + elasticsearchFlattened, err := flattenElasticsearchResources(res.Resources.Elasticsearch) + if err != nil { + return err + } if err := d.Set("elasticsearch", elasticsearchFlattened); err != nil { return err } diff --git a/ec/ecdatasource/deploymentdatasource/flatteners_elasticsearch.go b/ec/ecdatasource/deploymentdatasource/flatteners_elasticsearch.go index 5b7a381b9..b97e59ed4 100644 --- a/ec/ecdatasource/deploymentdatasource/flatteners_elasticsearch.go +++ b/ec/ecdatasource/deploymentdatasource/flatteners_elasticsearch.go @@ -18,6 +18,10 @@ package deploymentdatasource import ( + "encoding/json" + "fmt" + "strconv" + "github.com/elastic/cloud-sdk-go/pkg/models" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" @@ -26,7 +30,7 @@ import ( // flattenElasticsearchResources takes in Elasticsearch resource models and returns its // flattened form. -func flattenElasticsearchResources(in []*models.ElasticsearchResourceInfo) []interface{} { +func flattenElasticsearchResources(in []*models.ElasticsearchResourceInfo) ([]interface{}, error) { var result = make([]interface{}, 0, len(in)) for _, res := range in { var m = make(map[string]interface{}) @@ -55,7 +59,15 @@ func flattenElasticsearchResources(in []*models.ElasticsearchResourceInfo) []int m["version"] = plan.Elasticsearch.Version } - m["topology"] = flattenElasticsearchTopology(plan) + if plan.AutoscalingEnabled != nil { + m["autoscale"] = strconv.FormatBool(*plan.AutoscalingEnabled) + } + + top, err := flattenElasticsearchTopology(plan) + if err != nil { + return nil, err + } + m["topology"] = top } if res.Info.Metadata != nil { @@ -69,10 +81,10 @@ func flattenElasticsearchResources(in []*models.ElasticsearchResourceInfo) []int result = append(result, m) } - return result + return result, nil } -func flattenElasticsearchTopology(plan *models.ElasticsearchClusterPlan) []interface{} { +func flattenElasticsearchTopology(plan *models.ElasticsearchClusterPlan) ([]interface{}, error) { var result = make([]interface{}, 0, len(plan.ClusterTopology)) for _, topology := range plan.ClusterTopology { var m = make(map[string]interface{}) @@ -114,10 +126,36 @@ func flattenElasticsearchTopology(plan *models.ElasticsearchClusterPlan) []inter )) } + autoscaling := make(map[string]interface{}) + if ascale := topology.AutoscalingMax; ascale != nil { + autoscaling["max_size_resource"] = *ascale.Resource + autoscaling["max_size"] = util.MemoryToState(*ascale.Value) + } + + if ascale := topology.AutoscalingMin; ascale != nil { + autoscaling["min_size_resource"] = *ascale.Resource + autoscaling["min_size"] = util.MemoryToState(*ascale.Value) + } + + if topology.AutoscalingPolicyOverrideJSON != nil { + b, err := json.Marshal(topology.AutoscalingPolicyOverrideJSON) + if err != nil { + return nil, fmt.Errorf( + "elasticsearch topology %s: unable to persist policy_override_json: %w", + topology.ID, err, + ) + } + autoscaling["policy_override_json"] = string(b) + } + + if len(autoscaling) > 0 { + m["autoscaling"] = []interface{}{autoscaling} + } + result = append(result, m) } - return result + return result, nil } func isSizePopulated(topology *models.ElasticsearchClusterTopologyElement) bool { diff --git a/ec/ecdatasource/deploymentdatasource/schema_elasticsearch.go b/ec/ecdatasource/deploymentdatasource/schema_elasticsearch.go index e86fc4345..b5aa53ee1 100644 --- a/ec/ecdatasource/deploymentdatasource/schema_elasticsearch.go +++ b/ec/ecdatasource/deploymentdatasource/schema_elasticsearch.go @@ -24,6 +24,10 @@ import ( func newElasticsearchResourceInfo() *schema.Resource { return &schema.Resource{ Schema: map[string]*schema.Schema{ + "autoscale": { + Type: schema.TypeString, + Computed: true, + }, "healthy": { Type: schema.TypeBool, Computed: true, @@ -107,6 +111,46 @@ func elasticsearchTopologySchema() *schema.Schema { Type: schema.TypeString, }, }, + + "autoscaling": { + Type: schema.TypeList, + Description: "Optional Elasticsearch autoscaling settings, such a maximum and minimum size and resources.", + Computed: true, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "max_size_resource": { + Description: "Maximum resource type for the maximum autoscaling setting.", + Type: schema.TypeString, + Computed: true, + }, + + "max_size": { + Description: "Maximum size value for the maximum autoscaling setting.", + Type: schema.TypeString, + Computed: true, + }, + + "min_size_resource": { + Description: "Minimum resource type for the minimum autoscaling setting.", + Type: schema.TypeString, + Computed: true, + }, + + "min_size": { + Description: "Minimum size value for the minimum autoscaling setting.", + Type: schema.TypeString, + Computed: true, + }, + + "policy_override_json": { + Type: schema.TypeString, + Description: "Computed policy overrides set directly via the API or other clients.", + Computed: true, + }, + }, + }, + }, }, }, } From ce2f84558bfb1853fca9db8b6a9bc4b933b29ce2 Mon Sep 17 00:00:00 2001 From: Marc Lopez Date: Wed, 14 Apr 2021 11:17:35 +0500 Subject: [PATCH 04/12] ec_deployment: datasource add unit tests Signed-off-by: Marc Lopez --- .../flatteners_elasticsearch_test.go | 66 ++++++++++++------- 1 file changed, 41 insertions(+), 25 deletions(-) diff --git a/ec/ecdatasource/deploymentdatasource/flatteners_elasticsearch_test.go b/ec/ecdatasource/deploymentdatasource/flatteners_elasticsearch_test.go index 16e85fdb4..a251cf143 100644 --- a/ec/ecdatasource/deploymentdatasource/flatteners_elasticsearch_test.go +++ b/ec/ecdatasource/deploymentdatasource/flatteners_elasticsearch_test.go @@ -34,6 +34,7 @@ func Test_flattenElasticsearchResources(t *testing.T) { name string args args want []interface{} + err string }{ { name: "empty resource list returns empty list", @@ -61,6 +62,7 @@ func Test_flattenElasticsearchResources(t *testing.T) { PlanInfo: &models.ElasticsearchClusterPlansInfo{ Current: &models.ElasticsearchClusterPlanInfo{ Plan: &models.ElasticsearchClusterPlan{ + AutoscalingEnabled: ec.Bool(true), Elasticsearch: &models.ElasticsearchConfiguration{ Version: "7.7.0", }, @@ -79,6 +81,14 @@ func Test_flattenElasticsearchResources(t *testing.T) { Master: ec.Bool(true), Ml: ec.Bool(false), }, + AutoscalingMax: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(15360), + }, + AutoscalingMin: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(1024), + }, }, { NodeCountPerZone: 1, @@ -102,35 +112,41 @@ func Test_flattenElasticsearchResources(t *testing.T) { }, }, }}, - want: []interface{}{ - map[string]interface{}{ - "ref_id": "main-elasticsearch", - "resource_id": mock.ValidClusterID, - "version": "7.7.0", - "cloud_id": "some CLOUD ID", - "http_endpoint": "http://somecluster.cloud.elastic.co:9200", - "https_endpoint": "https://somecluster.cloud.elastic.co:9243", - "healthy": true, - "status": "started", - "topology": []interface{}{ - map[string]interface{}{ - "instance_configuration_id": "aws.data.highio.i3", - "size": "2g", - "size_resource": "memory", - "node_type_data": true, - "node_type_ingest": true, - "node_type_master": true, - "node_type_ml": false, - "zone_count": int32(1), - }, - }, - }, - }, + want: []interface{}{map[string]interface{}{ + "autoscale": "true", + "ref_id": "main-elasticsearch", + "resource_id": mock.ValidClusterID, + "version": "7.7.0", + "cloud_id": "some CLOUD ID", + "http_endpoint": "http://somecluster.cloud.elastic.co:9200", + "https_endpoint": "https://somecluster.cloud.elastic.co:9243", + "healthy": true, + "status": "started", + "topology": []interface{}{map[string]interface{}{ + "instance_configuration_id": "aws.data.highio.i3", + "size": "2g", + "size_resource": "memory", + "node_type_data": true, + "node_type_ingest": true, + "node_type_master": true, + "node_type_ml": false, + "zone_count": int32(1), + "autoscaling": []interface{}{map[string]interface{}{ + "max_size": "15g", + "max_size_resource": "memory", + "min_size": "1g", + "min_size_resource": "memory", + }}, + }}, + }}, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := flattenElasticsearchResources(tt.args.in) + got, err := flattenElasticsearchResources(tt.args.in) + if err != nil && assert.EqualError(t, err, tt.err) { + t.Error(err) + } assert.Equal(t, tt.want, got) }) } From b77fc9f55ec61940fb7b84695917e0d5aae5033e Mon Sep 17 00:00:00 2001 From: Marc Lopez Date: Wed, 14 Apr 2021 11:18:24 +0500 Subject: [PATCH 05/12] ec_deployment: resource update DT testdata Signed-off-by: Marc Lopez --- .../testdata/deployment-aws-ccs.json | 1 + ...deployment-aws-io-optimized-extension.json | 1 + .../deployment-aws-io-optimized-tags.json | 1 + .../testdata/deployment-aws-io-optimized.json | 1 + .../deployment-azure-io-optimized.json | 1 + .../deployment-gcp-hot-warm-node_roles.json | 1 + .../testdata/deployment-gcp-hot-warm.json | 1 + .../testdata/deployment-gcp-io-optimized.json | 1 + .../testdata/template-aws-hot-warm-v2.json | 21 +++++++++++++++++++ .../template-aws-io-optimized-v2.json | 21 +++++++++++++++++++ 10 files changed, 50 insertions(+) diff --git a/ec/ecresource/deploymentresource/testdata/deployment-aws-ccs.json b/ec/ecresource/deploymentresource/testdata/deployment-aws-ccs.json index ea734151d..30be5047f 100644 --- a/ec/ecresource/deploymentresource/testdata/deployment-aws-ccs.json +++ b/ec/ecresource/deploymentresource/testdata/deployment-aws-ccs.json @@ -59,6 +59,7 @@ "attempt_start_time": "2020-10-14T06:37:45.588Z", "healthy": true, "plan": { + "autoscaling_enabled": false, "cluster_topology": [ { "id": "hot_content", diff --git a/ec/ecresource/deploymentresource/testdata/deployment-aws-io-optimized-extension.json b/ec/ecresource/deploymentresource/testdata/deployment-aws-io-optimized-extension.json index c24ff3894..852b65012 100644 --- a/ec/ecresource/deploymentresource/testdata/deployment-aws-io-optimized-extension.json +++ b/ec/ecresource/deploymentresource/testdata/deployment-aws-io-optimized-extension.json @@ -119,6 +119,7 @@ "attempt_start_time": "2020-10-14T05:02:24.559Z", "healthy": true, "plan": { + "autoscaling_enabled": false, "cluster_topology": [ { "id": "hot_content", diff --git a/ec/ecresource/deploymentresource/testdata/deployment-aws-io-optimized-tags.json b/ec/ecresource/deploymentresource/testdata/deployment-aws-io-optimized-tags.json index 96daa2b22..c3e40e358 100644 --- a/ec/ecresource/deploymentresource/testdata/deployment-aws-io-optimized-tags.json +++ b/ec/ecresource/deploymentresource/testdata/deployment-aws-io-optimized-tags.json @@ -126,6 +126,7 @@ "attempt_start_time": "2020-10-14T05:02:24.559Z", "healthy": true, "plan": { + "autoscaling_enabled": false, "cluster_topology": [ { "id": "hot_content", diff --git a/ec/ecresource/deploymentresource/testdata/deployment-aws-io-optimized.json b/ec/ecresource/deploymentresource/testdata/deployment-aws-io-optimized.json index b739df69a..8b6e5f8a8 100644 --- a/ec/ecresource/deploymentresource/testdata/deployment-aws-io-optimized.json +++ b/ec/ecresource/deploymentresource/testdata/deployment-aws-io-optimized.json @@ -119,6 +119,7 @@ "attempt_start_time": "2020-10-14T05:02:24.559Z", "healthy": true, "plan": { + "autoscaling_enabled": false, "cluster_topology": [ { "id": "hot_content", diff --git a/ec/ecresource/deploymentresource/testdata/deployment-azure-io-optimized.json b/ec/ecresource/deploymentresource/testdata/deployment-azure-io-optimized.json index 0a7631e1b..ca9dc9243 100644 --- a/ec/ecresource/deploymentresource/testdata/deployment-azure-io-optimized.json +++ b/ec/ecresource/deploymentresource/testdata/deployment-azure-io-optimized.json @@ -116,6 +116,7 @@ "attempt_start_time": "2020-10-13T10:34:03.551Z", "healthy": true, "plan": { + "autoscaling_enabled": false, "cluster_topology": [ { "id": "hot_content", diff --git a/ec/ecresource/deploymentresource/testdata/deployment-gcp-hot-warm-node_roles.json b/ec/ecresource/deploymentresource/testdata/deployment-gcp-hot-warm-node_roles.json index c26b433e8..61093c337 100644 --- a/ec/ecresource/deploymentresource/testdata/deployment-gcp-hot-warm-node_roles.json +++ b/ec/ecresource/deploymentresource/testdata/deployment-gcp-hot-warm-node_roles.json @@ -119,6 +119,7 @@ "attempt_start_time": "2020-10-14T05:43:55.416Z", "healthy": true, "plan": { + "autoscaling_enabled": false, "cluster_topology": [ { "id": "hot_content", diff --git a/ec/ecresource/deploymentresource/testdata/deployment-gcp-hot-warm.json b/ec/ecresource/deploymentresource/testdata/deployment-gcp-hot-warm.json index 3e495ed0f..55c3d94df 100644 --- a/ec/ecresource/deploymentresource/testdata/deployment-gcp-hot-warm.json +++ b/ec/ecresource/deploymentresource/testdata/deployment-gcp-hot-warm.json @@ -119,6 +119,7 @@ "attempt_start_time": "2020-10-14T05:43:55.416Z", "healthy": true, "plan": { + "autoscaling_enabled": false, "cluster_topology": [ { "id": "hot_content", diff --git a/ec/ecresource/deploymentresource/testdata/deployment-gcp-io-optimized.json b/ec/ecresource/deploymentresource/testdata/deployment-gcp-io-optimized.json index b2ae56ab3..92c231f23 100644 --- a/ec/ecresource/deploymentresource/testdata/deployment-gcp-io-optimized.json +++ b/ec/ecresource/deploymentresource/testdata/deployment-gcp-io-optimized.json @@ -133,6 +133,7 @@ "attempt_start_time": "2020-10-14T05:31:22.873Z", "healthy": true, "plan": { + "autoscaling_enabled": false, "cluster_topology": [ { "id": "hot_content", diff --git a/ec/ecresource/deploymentresource/testdata/template-aws-hot-warm-v2.json b/ec/ecresource/deploymentresource/testdata/template-aws-hot-warm-v2.json index 81f45767d..b08ee7eee 100644 --- a/ec/ecresource/deploymentresource/testdata/template-aws-hot-warm-v2.json +++ b/ec/ecresource/deploymentresource/testdata/template-aws-hot-warm-v2.json @@ -25,6 +25,7 @@ "elasticsearch": [ { "plan": { + "autoscaling_enabled": false, "cluster_topology": [ { "id": "coordinating", @@ -51,6 +52,10 @@ "zone_count": 2 }, { + "autoscaling_max": { + "resource": "memory", + "value": 118784 + }, "elasticsearch": { "node_attributes": { "data": "hot" @@ -84,6 +89,10 @@ "zone_count": 2 }, { + "autoscaling_max": { + "resource": "memory", + "value": 118784 + }, "elasticsearch": { "node_attributes": { "data": "warm" @@ -113,6 +122,10 @@ "zone_count": 2 }, { + "autoscaling_max": { + "resource": "memory", + "value": 59392 + }, "elasticsearch": { "node_attributes": { "data": "cold" @@ -166,6 +179,14 @@ "zone_count": 3 }, { + "autoscaling_max": { + "resource": "memory", + "value": 61440 + }, + "autoscaling_min": { + "resource": "memory", + "value": 0 + }, "id": "ml", "instance_configuration_id": "aws.ml.m5d", "node_roles": [ diff --git a/ec/ecresource/deploymentresource/testdata/template-aws-io-optimized-v2.json b/ec/ecresource/deploymentresource/testdata/template-aws-io-optimized-v2.json index 52c0a569f..28acf0f72 100644 --- a/ec/ecresource/deploymentresource/testdata/template-aws-io-optimized-v2.json +++ b/ec/ecresource/deploymentresource/testdata/template-aws-io-optimized-v2.json @@ -25,6 +25,7 @@ "elasticsearch": [ { "plan": { + "autoscaling_enabled": false, "cluster_topology": [ { "id": "coordinating", @@ -51,6 +52,10 @@ "zone_count": 2 }, { + "autoscaling_max": { + "resource": "memory", + "value": 118784 + }, "elasticsearch": { "node_attributes": { "data": "hot" @@ -84,6 +89,10 @@ "zone_count": 2 }, { + "autoscaling_max": { + "resource": "memory", + "value": 118784 + }, "elasticsearch": { "node_attributes": { "data": "warm" @@ -113,6 +122,10 @@ "zone_count": 2 }, { + "autoscaling_max": { + "resource": "memory", + "value": 59392 + }, "elasticsearch": { "node_attributes": { "data": "cold" @@ -166,6 +179,14 @@ "zone_count": 3 }, { + "autoscaling_max": { + "resource": "memory", + "value": 61440 + }, + "autoscaling_min": { + "resource": "memory", + "value": 0 + }, "id": "ml", "instance_configuration_id": "aws.ml.m5d", "node_roles": [ From 69a79f766dfeb02120ab0a59f079a9250ad168a3 Mon Sep 17 00:00:00 2001 From: Marc Lopez Date: Wed, 14 Apr 2021 11:19:01 +0500 Subject: [PATCH 06/12] ec_deployment: resource update import tests Signed-off-by: Marc Lopez --- ec/ecresource/deploymentresource/import_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ec/ecresource/deploymentresource/import_test.go b/ec/ecresource/deploymentresource/import_test.go index 9b09bc3a1..55f82a17a 100644 --- a/ec/ecresource/deploymentresource/import_test.go +++ b/ec/ecresource/deploymentresource/import_test.go @@ -107,6 +107,7 @@ func Test_importFunc(t *testing.T) { "deployment_template_id": "aws-cross-cluster-search-v2", "elasticsearch.#": "1", + "elasticsearch.0.autoscale": "", "elasticsearch.0.cloud_id": "", "elasticsearch.0.snapshot_source.#": "0", "elasticsearch.0.config.#": "0", @@ -152,6 +153,7 @@ func Test_importFunc(t *testing.T) { "deployment_template_id": "aws-cross-cluster-search-v2", "elasticsearch.#": "1", + "elasticsearch.0.autoscale": "", "elasticsearch.0.cloud_id": "", "elasticsearch.0.snapshot_source.#": "0", "elasticsearch.0.config.#": "0", @@ -197,6 +199,7 @@ func Test_importFunc(t *testing.T) { "deployment_template_id": "aws-cross-cluster-search-v2", "elasticsearch.#": "1", + "elasticsearch.0.autoscale": "", "elasticsearch.0.cloud_id": "", "elasticsearch.0.snapshot_source.#": "0", "elasticsearch.0.config.#": "0", From 3ea1c45c04661ae5a4def588243e2ba88fc597db Mon Sep 17 00:00:00 2001 From: Marc Lopez Date: Wed, 14 Apr 2021 11:20:06 +0500 Subject: [PATCH 07/12] ec_deployment: resource add flattener tests Signed-off-by: Marc Lopez --- .../elasticsearch_flatteners_test.go | 12 +- .../deploymentresource/flatteners_test.go | 104 +++++++++++++++++- 2 files changed, 113 insertions(+), 3 deletions(-) diff --git a/ec/ecresource/deploymentresource/elasticsearch_flatteners_test.go b/ec/ecresource/deploymentresource/elasticsearch_flatteners_test.go index 287d3172e..d29fa7549 100644 --- a/ec/ecresource/deploymentresource/elasticsearch_flatteners_test.go +++ b/ec/ecresource/deploymentresource/elasticsearch_flatteners_test.go @@ -37,6 +37,7 @@ func Test_flattenEsResource(t *testing.T) { name string args args want []interface{} + err string }{ { name: "empty resource list returns empty list", @@ -339,7 +340,10 @@ func Test_flattenEsResource(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := flattenEsResources(tt.args.in, tt.args.name, tt.args.remotes) + got, err := flattenEsResources(tt.args.in, tt.args.name, tt.args.remotes) + if err != nil && !assert.EqualError(t, err, tt.err) { + t.Error(err) + } assert.Equal(t, tt.want, got) }) } @@ -353,6 +357,7 @@ func Test_flattenEsTopology(t *testing.T) { name string args args want []interface{} + err string }{ { name: "no zombie topologies", @@ -397,7 +402,10 @@ func Test_flattenEsTopology(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got := flattenEsTopology(tt.args.plan) + got, err := flattenEsTopology(tt.args.plan) + if err != nil && !assert.EqualError(t, err, tt.err) { + t.Error(err) + } assert.Equal(t, tt.want, got) }) } diff --git a/ec/ecresource/deploymentresource/flatteners_test.go b/ec/ecresource/deploymentresource/flatteners_test.go index 5217e02f3..a36d38d66 100644 --- a/ec/ecresource/deploymentresource/flatteners_test.go +++ b/ec/ecresource/deploymentresource/flatteners_test.go @@ -67,6 +67,7 @@ func Test_modelToState(t *testing.T) { }}, }}, "elasticsearch": []interface{}{map[string]interface{}{ + "autoscale": "false", "cloud_id": "up2d:somecloudID", "http_endpoint": "http://1238f19957874af69306787dca662154.eastus2.azure.elastic-cloud.com:9200", "https_endpoint": "https://1238f19957874af69306787dca662154.eastus2.azure.elastic-cloud.com:9243", @@ -131,6 +132,7 @@ func Test_modelToState(t *testing.T) { }}, }}, "elasticsearch": []interface{}{map[string]interface{}{ + "autoscale": "false", "cloud_id": "up2d:someCloudID", "http_endpoint": "http://1239f7ee7196439ba2d105319ac5eba7.eu-central-1.aws.cloud.es.io:9200", "https_endpoint": "https://1239f7ee7196439ba2d105319ac5eba7.eu-central-1.aws.cloud.es.io:9243", @@ -203,6 +205,7 @@ func Test_modelToState(t *testing.T) { }}, }}, "elasticsearch": []interface{}{map[string]interface{}{ + "autoscale": "false", "cloud_id": "up2d:someCloudID", "http_endpoint": "http://1239f7ee7196439ba2d105319ac5eba7.eu-central-1.aws.cloud.es.io:9200", "https_endpoint": "https://1239f7ee7196439ba2d105319ac5eba7.eu-central-1.aws.cloud.es.io:9243", @@ -267,6 +270,7 @@ func Test_modelToState(t *testing.T) { }}, }}, "elasticsearch": []interface{}{map[string]interface{}{ + "autoscale": "false", "cloud_id": "up2d:someCloudID", "http_endpoint": "http://123695e76d914005bf90b717e668ad4b.asia-east1.gcp.elastic-cloud.com:9200", "https_endpoint": "https://123695e76d914005bf90b717e668ad4b.asia-east1.gcp.elastic-cloud.com:9243", @@ -304,6 +308,10 @@ func Test_modelToState(t *testing.T) { Schema: newSchema(), }) + gcpIOOptimizedAutoscaleRes := openDeploymentGet(t, "testdata/deployment-gcp-io-optimized-autoscale.json") + gcpIOOptimizedAutoscaleRD := schema.TestResourceDataRaw(t, newSchema(), nil) + gcpIOOptimizedAutoscaleRD.SetId(mock.ValidClusterID) + gcpHotWarmRes := openDeploymentGet(t, "testdata/deployment-gcp-hot-warm.json") gcpHotWarmRD := schema.TestResourceDataRaw(t, newSchema(), nil) gcpHotWarmRD.SetId(mock.ValidClusterID) @@ -331,6 +339,7 @@ func Test_modelToState(t *testing.T) { }}, }}, "elasticsearch": []interface{}{map[string]interface{}{ + "autoscale": "false", "cloud_id": "up2d-hot-warm:someCloudID", "http_endpoint": "http://123e837db6ee4391bb74887be35a7a91.us-central1.gcp.cloud.es.io:9200", "https_endpoint": "https://123e837db6ee4391bb74887be35a7a91.us-central1.gcp.cloud.es.io:9243", @@ -408,6 +417,7 @@ func Test_modelToState(t *testing.T) { }}, }}, "elasticsearch": []interface{}{map[string]interface{}{ + "autoscale": "false", "cloud_id": "up2d-hot-warm:someCloudID", "http_endpoint": "http://123e837db6ee4391bb74887be35a7a91.us-central1.gcp.cloud.es.io:9200", "https_endpoint": "https://123e837db6ee4391bb74887be35a7a91.us-central1.gcp.cloud.es.io:9243", @@ -474,6 +484,7 @@ func Test_modelToState(t *testing.T) { "region": "eu-west-1", "version": "7.9.2", "elasticsearch": []interface{}{map[string]interface{}{ + "autoscale": "false", "cloud_id": "ccs:someCloudID", "http_endpoint": "http://1230b3ae633b4f51a432d50971f7f1c1.eu-west-1.aws.found.io:9200", "https_endpoint": "https://1230b3ae633b4f51a432d50971f7f1c1.eu-west-1.aws.found.io:9243", @@ -776,7 +787,8 @@ func Test_modelToState(t *testing.T) { }}, }}, "elasticsearch": []interface{}{map[string]interface{}{ - "cloud_id": "up2d:someCloudID", + "autoscale": "false", + "cloud_id": "up2d:someCloudID", "extension": []interface{}{ map[string]interface{}{ "name": "custom-bundle", @@ -849,6 +861,96 @@ func Test_modelToState(t *testing.T) { args: args{d: gcpIOOptimizedRD, res: gcpIOOptimizedRes}, want: wantGcpIOOptimizedDeployment, }, + { + name: "flattens a gcp plan with autoscale set (io-optimized)", + args: args{d: gcpIOOptimizedRD, res: gcpIOOptimizedAutoscaleRes}, + want: util.NewResourceData(t, util.ResDataParams{ + ID: mock.ValidClusterID, + State: map[string]interface{}{ + "deployment_template_id": "gcp-io-optimized", + "id": "123b7b540dfc967a7a649c18e2fce4ed", + "name": "up2d", + "region": "gcp-asia-east1", + "version": "7.9.2", + "apm": []interface{}{map[string]interface{}{ + "elasticsearch_cluster_ref_id": "main-elasticsearch", + "ref_id": "main-apm", + "region": "gcp-asia-east1", + "resource_id": "12307c6c304949b8a9f3682b80900879", + "version": "7.9.2", + "http_endpoint": "http://12307c6c304949b8a9f3682b80900879.apm.asia-east1.gcp.elastic-cloud.com:80", + "https_endpoint": "https://12307c6c304949b8a9f3682b80900879.apm.asia-east1.gcp.elastic-cloud.com:443", + "topology": []interface{}{map[string]interface{}{ + "instance_configuration_id": "gcp.apm.1", + "size": "0.5g", + "size_resource": "memory", + "zone_count": 1, + }}, + }}, + "elasticsearch": []interface{}{map[string]interface{}{ + "autoscale": "true", + "cloud_id": "up2d:someCloudID", + "http_endpoint": "http://123695e76d914005bf90b717e668ad4b.asia-east1.gcp.elastic-cloud.com:9200", + "https_endpoint": "https://123695e76d914005bf90b717e668ad4b.asia-east1.gcp.elastic-cloud.com:9243", + "ref_id": "main-elasticsearch", + "region": "gcp-asia-east1", + "resource_id": "123695e76d914005bf90b717e668ad4b", + "topology": []interface{}{ + map[string]interface{}{ + "id": "hot_content", + "instance_configuration_id": "gcp.data.highio.1", + "node_type_data": "true", + "node_type_ingest": "true", + "node_type_master": "true", + "node_type_ml": "false", + "size": "8g", + "size_resource": "memory", + "zone_count": 2, + "autoscaling": []interface{}{map[string]interface{}{ + "max_size": "29g", + "max_size_resource": "memory", + "policy_override_json": `{"proactive_storage":{"forecast_window":"3 h"}}`, + }}, + }, + map[string]interface{}{ + "id": "ml", + "instance_configuration_id": "gcp.ml.1", + "node_type_data": "false", + "node_type_ingest": "false", + "node_type_master": "false", + "node_type_ml": "true", + "size": "1g", + "size_resource": "memory", + "zone_count": 1, + "autoscaling": []interface{}{map[string]interface{}{ + "max_size": "30g", + "max_size_resource": "memory", + + "min_size": "1g", + "min_size_resource": "memory", + }}, + }, + }, + }}, + "kibana": []interface{}{map[string]interface{}{ + "elasticsearch_cluster_ref_id": "main-elasticsearch", + "ref_id": "main-kibana", + "region": "gcp-asia-east1", + "resource_id": "12365046781e4d729a07df64fe67c8c6", + "version": "7.9.2", + "http_endpoint": "http://12365046781e4d729a07df64fe67c8c6.asia-east1.gcp.elastic-cloud.com:9200", + "https_endpoint": "https://12365046781e4d729a07df64fe67c8c6.asia-east1.gcp.elastic-cloud.com:9243", + "topology": []interface{}{map[string]interface{}{ + "instance_configuration_id": "gcp.kibana.1", + "size": "1g", + "size_resource": "memory", + "zone_count": 1, + }}, + }}, + }, + Schema: newSchema(), + }), + }, { name: "flattens a gcp plan (hot-warm)", args: args{d: gcpHotWarmRD, res: gcpHotWarmRes}, From 402f171d52ce9b814d5bc2195c0487e5130f9e84 Mon Sep 17 00:00:00 2001 From: Marc Lopez Date: Wed, 14 Apr 2021 13:06:47 +0500 Subject: [PATCH 08/12] Add missing test file Signed-off-by: Marc Lopez --- ...deployment-gcp-io-optimized-autoscale.json | 389 ++++++++++++++++++ 1 file changed, 389 insertions(+) create mode 100644 ec/ecresource/deploymentresource/testdata/deployment-gcp-io-optimized-autoscale.json diff --git a/ec/ecresource/deploymentresource/testdata/deployment-gcp-io-optimized-autoscale.json b/ec/ecresource/deploymentresource/testdata/deployment-gcp-io-optimized-autoscale.json new file mode 100644 index 000000000..ac6e414fa --- /dev/null +++ b/ec/ecresource/deploymentresource/testdata/deployment-gcp-io-optimized-autoscale.json @@ -0,0 +1,389 @@ +{ + "healthy": true, + "id": "1239e402d6df471ea374bd68e3f91cc5", + "name": "up2d", + "resources": { + "apm": [ + { + "elasticsearch_cluster_ref_id": "main-elasticsearch", + "id": "12307c6c304949b8a9f3682b80900879", + "info": { + "deployment_id": "1239e402d6df471ea374bd68e3f91cc5", + "elasticsearch_cluster": { + "elasticsearch_id": "123695e76d914005bf90b717e668ad4b" + }, + "external_links": [], + "healthy": true, + "id": "12307c6c304949b8a9f3682b80900879", + "metadata": { + "endpoint": "12307c6c304949b8a9f3682b80900879.apm.asia-east1.gcp.elastic-cloud.com", + "last_modified": "2020-10-14T05:31:29.441Z", + "ports": { + "http": 80, + "https": 443, + "transport_passthrough": 9400 + }, + "version": 7 + }, + "name": "up2d", + "plan_info": { + "current": { + "attempt_end_time": "2020-10-14T05:33:42.125Z", + "attempt_start_time": "2020-10-14T05:31:28.748Z", + "healthy": true, + "plan": { + "apm": { + "system_settings": { + "secret_token": "7g6LZFbwU6aCCVoLjw" + }, + "version": "7.9.2" + }, + "cluster_topology": [ + { + "apm": { + "system_settings": { + "debug_enabled": false, + "secret_token": "7g6LZFbwU6aCCVoLjw" + } + }, + "instance_configuration_id": "gcp.apm.1", + "size": { + "resource": "memory", + "value": 512 + }, + "zone_count": 1 + } + ], + "transient": { + "plan_configuration": { + "calm_wait_time": 5, + "extended_maintenance": false, + "move_allocators": [], + "move_instances": [], + "preferred_allocators": [], + "reallocate_instances": false, + "timeout": 2048 + }, + "strategy": { + "autodetect": {} + } + } + }, + "plan_attempt_id": "59fdecfc-40ab-43c7-83d1-7179e720c307", + "plan_attempt_log": [], + "plan_end_time": "0001-01-01T00:00:00.000Z" + }, + "healthy": true, + "history": [] + }, + "region": "gcp-asia-east1", + "status": "started" + }, + "ref_id": "main-apm", + "region": "gcp-asia-east1" + } + ], + "appsearch": [], + "elasticsearch": [ + { + "id": "123695e76d914005bf90b717e668ad4b", + "info": { + "associated_apm_clusters": [ + { + "apm_id": "12307c6c304949b8a9f3682b80900879", + "enabled": true + } + ], + "associated_appsearch_clusters": [], + "associated_enterprise_search_clusters": [], + "associated_kibana_clusters": [ + { + "enabled": true, + "kibana_id": "12365046781e4d729a07df64fe67c8c6" + } + ], + "cluster_id": "123695e76d914005bf90b717e668ad4b", + "cluster_name": "up2d", + "deployment_id": "1239e402d6df471ea374bd68e3f91cc5", + "elasticsearch": { + "blocking_issues": { + "cluster_level": [], + "healthy": true, + "index_level": [] + }, + "healthy": true + }, + "external_links": [], + "healthy": true, + "locked": false, + "metadata": { + "cloud_id": "up2d:someCloudID", + "endpoint": "123695e76d914005bf90b717e668ad4b.asia-east1.gcp.elastic-cloud.com", + "last_modified": "2020-10-14T05:33:43.403Z", + "ports": { + "http": 9200, + "https": 9243, + "transport_passthrough": 9400 + }, + "version": 25 + }, + "plan_info": { + "current": { + "attempt_end_time": "2020-10-14T05:32:31.743Z", + "attempt_start_time": "2020-10-14T05:31:22.873Z", + "healthy": true, + "plan": { + "autoscaling_enabled": true, + "cluster_topology": [ + { + "autoscaling_max": { + "resource": "memory", + "value": 29696 + }, + "autoscaling_policy_override_json": { + "proactive_storage": { + "forecast_window": "3 h" + } + }, + "id": "hot_content", + "elasticsearch": { + "node_attributes": { + "data": "hot" + }, + "system_settings": { + "auto_create_index": true, + "destructive_requires_name": false, + "enable_close_index": true, + "monitoring_collection_interval": -1, + "monitoring_history_duration": "3d", + "reindex_whitelist": [], + "scripting": { + "inline": { + "enabled": true + }, + "stored": { + "enabled": true + } + }, + "use_disk_threshold": true + } + }, + "instance_configuration_id": "gcp.data.highio.1", + "node_type": { + "data": true, + "ingest": true, + "master": true, + "ml": false + }, + "size": { + "resource": "memory", + "value": 8192 + }, + "topology_element_control": { + "min": { + "resource": "memory", + "value": 1024 + } + }, + "zone_count": 2 + }, + { + "id": "coordinating", + "elasticsearch": { + "system_settings": { + "auto_create_index": true, + "destructive_requires_name": false, + "enable_close_index": true, + "monitoring_collection_interval": -1, + "monitoring_history_duration": "3d", + "reindex_whitelist": [], + "scripting": { + "inline": { + "enabled": true + }, + "stored": { + "enabled": true + } + }, + "use_disk_threshold": true + } + }, + "instance_configuration_id": "gcp.coordinating.1", + "node_type": { + "data": false, + "ingest": true, + "master": false, + "ml": false + }, + "size": { + "resource": "memory", + "value": 0 + }, + "zone_count": 2 + }, + { + "id": "master", + "elasticsearch": { + "system_settings": { + "auto_create_index": true, + "destructive_requires_name": false, + "enable_close_index": true, + "monitoring_collection_interval": -1, + "monitoring_history_duration": "3d", + "reindex_whitelist": [], + "scripting": { + "inline": { + "enabled": true + }, + "stored": { + "enabled": true + } + }, + "use_disk_threshold": true + } + }, + "instance_configuration_id": "gcp.master.1", + "node_type": { + "data": false, + "ingest": false, + "master": true, + "ml": false + }, + "size": { + "resource": "memory", + "value": 0 + }, + "zone_count": 3 + }, + { + "autoscaling_max": { + "value": 30720, + "resource": "memory" + }, + "autoscaling_min": { + "value": 1024, + "resource": "memory" + }, + "id": "ml", + "elasticsearch": { + "system_settings": { + "auto_create_index": true, + "destructive_requires_name": false, + "enable_close_index": true, + "monitoring_collection_interval": -1, + "monitoring_history_duration": "3d", + "reindex_whitelist": [], + "scripting": { + "inline": { + "enabled": true + }, + "stored": { + "enabled": true + } + }, + "use_disk_threshold": true + } + }, + "instance_configuration_id": "gcp.ml.1", + "node_type": { + "data": false, + "ingest": false, + "master": false, + "ml": true + }, + "size": { + "resource": "memory", + "value": 1024 + }, + "zone_count": 1 + } + ], + "deployment_template": { + "id": "gcp-io-optimized" + }, + "elasticsearch": { + "version": "7.9.2" + }, + "tiebreaker_topology": { + "memory_per_node": 1024 + } + }, + "plan_attempt_id": "c369366f-ed22-428c-94a2-15810152fa3f", + "plan_attempt_log": [], + "plan_end_time": "0001-01-01T00:00:00.000Z" + }, + "healthy": true, + "history": [] + }, + "region": "gcp-asia-east1", + "status": "started", + "system_alerts": [] + }, + "ref_id": "main-elasticsearch", + "region": "gcp-asia-east1" + } + ], + "enterprise_search": [], + "kibana": [ + { + "elasticsearch_cluster_ref_id": "main-elasticsearch", + "id": "12365046781e4d729a07df64fe67c8c6", + "info": { + "cluster_id": "12365046781e4d729a07df64fe67c8c6", + "cluster_name": "up2d", + "deployment_id": "1239e402d6df471ea374bd68e3f91cc5", + "elasticsearch_cluster": { + "elasticsearch_id": "123695e76d914005bf90b717e668ad4b" + }, + "external_links": [], + "healthy": true, + "metadata": { + "endpoint": "12365046781e4d729a07df64fe67c8c6.asia-east1.gcp.elastic-cloud.com", + "last_modified": "2020-10-14T05:33:43.199Z", + "ports": { + "http": 9200, + "https": 9243, + "transport_passthrough": 9400 + }, + "version": 12 + }, + "plan_info": { + "current": { + "attempt_end_time": "2020-10-14T05:35:15.948Z", + "attempt_start_time": "2020-10-14T05:33:42.291Z", + "healthy": true, + "plan": { + "cluster_topology": [ + { + "instance_configuration_id": "gcp.kibana.1", + "kibana": { + "system_settings": {} + }, + "size": { + "resource": "memory", + "value": 1024 + }, + "zone_count": 1 + } + ], + "kibana": { + "system_settings": {}, + "version": "7.9.2" + } + }, + "plan_attempt_id": "8e2746b7-0779-4e64-9972-7340b6382de3", + "plan_attempt_log": [], + "plan_end_time": "0001-01-01T00:00:00.000Z" + }, + "healthy": true, + "history": [] + }, + "region": "gcp-asia-east1", + "status": "started" + }, + "ref_id": "main-kibana", + "region": "gcp-asia-east1" + } + ] + } +} \ No newline at end of file From 0508171c2e7787e75ce2e75885d7f1d736252d3d Mon Sep 17 00:00:00 2001 From: Marc Lopez Date: Wed, 14 Apr 2021 13:19:11 +0500 Subject: [PATCH 09/12] Update logic to always include all topology elements Signed-off-by: Marc Lopez --- .../elasticsearch_expanders.go | 32 ------------------- ec/ecresource/deploymentresource/schema.go | 1 - 2 files changed, 33 deletions(-) diff --git a/ec/ecresource/deploymentresource/elasticsearch_expanders.go b/ec/ecresource/deploymentresource/elasticsearch_expanders.go index 145900953..009bdf0e3 100644 --- a/ec/ecresource/deploymentresource/elasticsearch_expanders.go +++ b/ec/ecresource/deploymentresource/elasticsearch_expanders.go @@ -82,8 +82,6 @@ func expandEsResource(raw interface{}, res *models.ElasticsearchPayload) (*model return nil, err } res.Plan.ClusterTopology = topology - } else { - res.Plan.ClusterTopology = defaultEsTopologies(res.Plan.ClusterTopology) } // Fixes the node_roles field to remove the dedicated tier roles from the @@ -285,36 +283,6 @@ func expandSnapshotSource(raw interface{}, restore *models.RestoreSnapshotConfig } } -func discardEsZeroSize(topologies []*models.ElasticsearchClusterTopologyElement) (result []*models.ElasticsearchClusterTopologyElement) { - for _, topology := range topologies { - if topology.Size == nil || topology.Size.Value == nil || *topology.Size.Value == 0 { - continue - } - result = append(result, topology) - } - return result -} - -// defaultEsTopologies iterates over all the templated topology elements and -// sets the size to the default when the template size is smaller than the -// deployment template default, the same is done on the ZoneCount. It discards -// any elements where the size is == 0, since it means that different Instance -// configurations are available to configure but are not included in the -// default deployment template. -func defaultEsTopologies(topology []*models.ElasticsearchClusterTopologyElement) []*models.ElasticsearchClusterTopologyElement { - topology = discardEsZeroSize(topology) - for _, t := range topology { - if *t.Size.Value < minimumElasticsearchSize { - t.Size.Value = ec.Int32(minimumElasticsearchSize) - } - if t.ZoneCount < minimumZoneCount { - t.ZoneCount = minimumZoneCount - } - } - - return topology -} - func matchEsTopologyID(id string, topologies []*models.ElasticsearchClusterTopologyElement) (*models.ElasticsearchClusterTopologyElement, error) { for _, t := range topologies { if t.ID == id { diff --git a/ec/ecresource/deploymentresource/schema.go b/ec/ecresource/deploymentresource/schema.go index ff38074c1..a5b82a797 100644 --- a/ec/ecresource/deploymentresource/schema.go +++ b/ec/ecresource/deploymentresource/schema.go @@ -22,7 +22,6 @@ import ( ) const ( - minimumElasticsearchSize = 1024 minimumKibanaSize = 1024 minimumApmSize = 512 minimumEnterpriseSearchSize = 2048 From 47e333a2ea5ee01a208d4d0a18477046aac599b5 Mon Sep 17 00:00:00 2001 From: Marc Lopez Date: Wed, 14 Apr 2021 13:20:03 +0500 Subject: [PATCH 10/12] Update all expander tests Signed-off-by: Marc Lopez --- .../elasticsearch_expanders_test.go | 1722 +++++++---- .../deploymentresource/expanders_test.go | 2558 ++++++++++------- .../deploymentresource/testutil_func.go | 38 + 3 files changed, 2717 insertions(+), 1601 deletions(-) diff --git a/ec/ecresource/deploymentresource/elasticsearch_expanders_test.go b/ec/ecresource/deploymentresource/elasticsearch_expanders_test.go index 614d0a283..02619913c 100644 --- a/ec/ecresource/deploymentresource/elasticsearch_expanders_test.go +++ b/ec/ecresource/deploymentresource/elasticsearch_expanders_test.go @@ -39,22 +39,26 @@ func Test_expandEsResource(t *testing.T) { ) } - create710 := enrichElasticsearchTemplate( - esResource(parseDeploymentTemplate(t, tplPath)), - "aws-io-optimized-v2", - "7.10.0", - true, - ) + create710 := func() *models.ElasticsearchPayload { + return enrichElasticsearchTemplate( + esResource(parseDeploymentTemplate(t, tplPath)), + "aws-io-optimized-v2", + "7.10.0", + true, + ) + } - update711 := enrichElasticsearchTemplate( - esResource(parseDeploymentTemplate(t, tplPath)), - "aws-io-optimized-v2", - "7.11.0", - true, - ) + update711 := func() *models.ElasticsearchPayload { + return enrichElasticsearchTemplate( + esResource(parseDeploymentTemplate(t, tplPath)), + "aws-io-optimized-v2", + "7.11.0", + true, + ) + } hotWarmTplPath := "testdata/template-aws-hot-warm-v2.json" - hotWarmTpl := func() *models.ElasticsearchPayload { + hotWarmTpl770 := func() *models.ElasticsearchPayload { return enrichElasticsearchTemplate( esResource(parseDeploymentTemplate(t, hotWarmTplPath)), "aws-io-optimized-v2", @@ -103,55 +107,58 @@ func Test_expandEsResource(t *testing.T) { }, }, }, - want: []*models.ElasticsearchPayload{ - { - Region: ec.String("some-region"), - RefID: ec.String("main-elasticsearch"), - Settings: &models.ElasticsearchClusterSettings{ - DedicatedMastersThreshold: 6, - }, - Plan: &models.ElasticsearchClusterPlan{ - Elasticsearch: &models.ElasticsearchConfiguration{ - Version: "7.7.0", - }, - DeploymentTemplate: &models.DeploymentTemplateReference{ - ID: ec.String("aws-io-optimized-v2"), - }, - ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ - { - ID: "hot_content", - ZoneCount: 1, - InstanceConfigurationID: "aws.data.highio.i3", - Size: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(2048), - }, - NodeType: &models.ElasticsearchNodeType{ - Data: ec.Bool(true), - Ingest: ec.Bool(true), - Master: ec.Bool(true), - }, - Elasticsearch: &models.ElasticsearchConfiguration{ - NodeAttributes: map[string]string{ - "data": "hot", - }, + want: enrichWithEmptyTopologies(tp770(), &models.ElasticsearchPayload{ + Region: ec.String("some-region"), + RefID: ec.String("main-elasticsearch"), + Settings: &models.ElasticsearchClusterSettings{ + DedicatedMastersThreshold: 6, + }, + Plan: &models.ElasticsearchClusterPlan{ + AutoscalingEnabled: ec.Bool(false), + Elasticsearch: &models.ElasticsearchConfiguration{ + Version: "7.7.0", + }, + DeploymentTemplate: &models.DeploymentTemplateReference{ + ID: ec.String("aws-io-optimized-v2"), + }, + ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ + { + ID: "hot_content", + ZoneCount: 1, + InstanceConfigurationID: "aws.data.highio.i3", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(2048), + }, + NodeType: &models.ElasticsearchNodeType{ + Data: ec.Bool(true), + Ingest: ec.Bool(true), + Master: ec.Bool(true), + }, + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{ + "data": "hot", }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(1024), - }, + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(1024), }, }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, }, }, }, - }, + }), }, { name: "parses an ES resource with empty version (7.10.0) in state uses node_roles from the DT", args: args{ - dt: create710, + dt: create710(), ess: []interface{}{ map[string]interface{}{ "ref_id": "main-elasticsearch", @@ -165,58 +172,61 @@ func Test_expandEsResource(t *testing.T) { }, }, }, - want: []*models.ElasticsearchPayload{ - { - Region: ec.String("some-region"), - RefID: ec.String("main-elasticsearch"), - Settings: &models.ElasticsearchClusterSettings{ - DedicatedMastersThreshold: 6, - }, - Plan: &models.ElasticsearchClusterPlan{ - Elasticsearch: &models.ElasticsearchConfiguration{ - Version: "7.10.0", - }, - DeploymentTemplate: &models.DeploymentTemplateReference{ - ID: ec.String("aws-io-optimized-v2"), - }, - ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ - { - ID: "hot_content", - ZoneCount: 1, - InstanceConfigurationID: "aws.data.highio.i3", - Size: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(2048), - }, - NodeRoles: []string{ - "master", - "ingest", - "remote_cluster_client", - "data_hot", - "transform", - "data_content", - }, - Elasticsearch: &models.ElasticsearchConfiguration{ - NodeAttributes: map[string]string{ - "data": "hot", - }, + want: enrichWithEmptyTopologies(create710(), &models.ElasticsearchPayload{ + Region: ec.String("some-region"), + RefID: ec.String("main-elasticsearch"), + Settings: &models.ElasticsearchClusterSettings{ + DedicatedMastersThreshold: 6, + }, + Plan: &models.ElasticsearchClusterPlan{ + AutoscalingEnabled: ec.Bool(false), + Elasticsearch: &models.ElasticsearchConfiguration{ + Version: "7.10.0", + }, + DeploymentTemplate: &models.DeploymentTemplateReference{ + ID: ec.String("aws-io-optimized-v2"), + }, + ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ + { + ID: "hot_content", + ZoneCount: 1, + InstanceConfigurationID: "aws.data.highio.i3", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(2048), + }, + NodeRoles: []string{ + "master", + "ingest", + "remote_cluster_client", + "data_hot", + "transform", + "data_content", + }, + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{ + "data": "hot", }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(1024), - }, + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(1024), }, }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, }, }, }, - }, + }), }, { name: "parses an ES resource with version 7.11.0 has node_roles coming from the saved state", args: args{ - dt: update711, + dt: update711(), ess: []interface{}{ map[string]interface{}{ "ref_id": "main-elasticsearch", @@ -233,48 +243,51 @@ func Test_expandEsResource(t *testing.T) { }, }, }, - want: []*models.ElasticsearchPayload{ - { - Region: ec.String("some-region"), - RefID: ec.String("main-elasticsearch"), - Settings: &models.ElasticsearchClusterSettings{ - DedicatedMastersThreshold: 6, - }, - Plan: &models.ElasticsearchClusterPlan{ - Elasticsearch: &models.ElasticsearchConfiguration{ - Version: "7.11.0", - }, - DeploymentTemplate: &models.DeploymentTemplateReference{ - ID: ec.String("aws-io-optimized-v2"), - }, - ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ - { - ID: "hot_content", - ZoneCount: 1, - InstanceConfigurationID: "aws.data.highio.i3", - Size: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(2048), - }, - NodeRoles: []string{ - "a", "b", "c", - }, - Elasticsearch: &models.ElasticsearchConfiguration{ - NodeAttributes: map[string]string{ - "data": "hot", - }, + want: enrichWithEmptyTopologies(update711(), &models.ElasticsearchPayload{ + Region: ec.String("some-region"), + RefID: ec.String("main-elasticsearch"), + Settings: &models.ElasticsearchClusterSettings{ + DedicatedMastersThreshold: 6, + }, + Plan: &models.ElasticsearchClusterPlan{ + AutoscalingEnabled: ec.Bool(false), + Elasticsearch: &models.ElasticsearchConfiguration{ + Version: "7.11.0", + }, + DeploymentTemplate: &models.DeploymentTemplateReference{ + ID: ec.String("aws-io-optimized-v2"), + }, + ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ + { + ID: "hot_content", + ZoneCount: 1, + InstanceConfigurationID: "aws.data.highio.i3", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(2048), + }, + NodeRoles: []string{ + "a", "b", "c", + }, + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{ + "data": "hot", }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(1024), - }, + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(1024), }, }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, }, }, }, - }, + }), }, { name: "parses an ES resource with invalid id", @@ -309,55 +322,58 @@ func Test_expandEsResource(t *testing.T) { }, }, }, - want: []*models.ElasticsearchPayload{ - { - Region: ec.String("some-region"), - RefID: ec.String("main-elasticsearch"), - Settings: &models.ElasticsearchClusterSettings{ - DedicatedMastersThreshold: 6, - }, - Plan: &models.ElasticsearchClusterPlan{ - Elasticsearch: &models.ElasticsearchConfiguration{ - Version: "7.7.0", - }, - DeploymentTemplate: &models.DeploymentTemplateReference{ - ID: ec.String("aws-io-optimized-v2"), - }, - ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ - { - ID: "hot_content", - InstanceConfigurationID: "aws.data.highio.i3", - ZoneCount: 2, - Size: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(8192), - }, - NodeType: &models.ElasticsearchNodeType{ - Data: ec.Bool(true), - Ingest: ec.Bool(true), - Master: ec.Bool(true), - }, - Elasticsearch: &models.ElasticsearchConfiguration{ - NodeAttributes: map[string]string{ - "data": "hot", - }, + want: enrichWithEmptyTopologies(tp770(), &models.ElasticsearchPayload{ + Region: ec.String("some-region"), + RefID: ec.String("main-elasticsearch"), + Settings: &models.ElasticsearchClusterSettings{ + DedicatedMastersThreshold: 6, + }, + Plan: &models.ElasticsearchClusterPlan{ + AutoscalingEnabled: ec.Bool(false), + Elasticsearch: &models.ElasticsearchConfiguration{ + Version: "7.7.0", + }, + DeploymentTemplate: &models.DeploymentTemplateReference{ + ID: ec.String("aws-io-optimized-v2"), + }, + ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ + { + ID: "hot_content", + InstanceConfigurationID: "aws.data.highio.i3", + ZoneCount: 2, + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(8192), + }, + NodeType: &models.ElasticsearchNodeType{ + Data: ec.Bool(true), + Ingest: ec.Bool(true), + Master: ec.Bool(true), + }, + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{ + "data": "hot", }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(1024), - }, + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(1024), }, }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, }, }, }, - }, + }), }, { name: "parses an ES resource (HotWarm)", args: args{ - dt: hotWarmTpl(), + dt: hotWarmTpl770(), ess: []interface{}{ map[string]interface{}{ "ref_id": "main-elasticsearch", @@ -379,82 +395,89 @@ func Test_expandEsResource(t *testing.T) { }, }, }, - want: []*models.ElasticsearchPayload{ - { - Region: ec.String("some-region"), - RefID: ec.String("main-elasticsearch"), - Settings: &models.ElasticsearchClusterSettings{ - DedicatedMastersThreshold: 6, - Curation: nil, - }, - Plan: &models.ElasticsearchClusterPlan{ - Elasticsearch: &models.ElasticsearchConfiguration{ - Version: "7.7.0", - Curation: nil, - }, - DeploymentTemplate: &models.DeploymentTemplateReference{ - ID: ec.String("aws-hot-warm-v2"), - }, - ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ - { - ID: "hot_content", - Elasticsearch: &models.ElasticsearchConfiguration{ - NodeAttributes: map[string]string{ - "data": "hot", - }, - }, - ZoneCount: 1, - InstanceConfigurationID: "aws.data.highio.i3", - Size: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(2048), - }, - NodeType: &models.ElasticsearchNodeType{ - Data: ec.Bool(true), - Ingest: ec.Bool(true), - Master: ec.Bool(true), - }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(1024), - }, - }, - }, - { - ID: "warm", - Elasticsearch: &models.ElasticsearchConfiguration{ - NodeAttributes: map[string]string{ - "data": "warm", - }, - }, - ZoneCount: 1, - InstanceConfigurationID: "aws.data.highstorage.d2", - Size: &models.TopologySize{ + want: enrichWithEmptyTopologies(hotWarmTpl770(), &models.ElasticsearchPayload{ + Region: ec.String("some-region"), + RefID: ec.String("main-elasticsearch"), + Settings: &models.ElasticsearchClusterSettings{ + DedicatedMastersThreshold: 6, + Curation: nil, + }, + Plan: &models.ElasticsearchClusterPlan{ + AutoscalingEnabled: ec.Bool(false), + Elasticsearch: &models.ElasticsearchConfiguration{ + Version: "7.7.0", + Curation: nil, + }, + DeploymentTemplate: &models.DeploymentTemplateReference{ + ID: ec.String("aws-hot-warm-v2"), + }, + ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ + { + ID: "hot_content", + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{ + "data": "hot", + }, + }, + ZoneCount: 1, + InstanceConfigurationID: "aws.data.highio.i3", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(2048), + }, + NodeType: &models.ElasticsearchNodeType{ + Data: ec.Bool(true), + Ingest: ec.Bool(true), + Master: ec.Bool(true), + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ Resource: ec.String("memory"), - Value: ec.Int32(2048), + Value: ec.Int32(1024), }, - NodeType: &models.ElasticsearchNodeType{ - Data: ec.Bool(true), - Ingest: ec.Bool(true), - Master: ec.Bool(false), + }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, + }, + { + ID: "warm", + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{ + "data": "warm", }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(0), - }, + }, + ZoneCount: 1, + InstanceConfigurationID: "aws.data.highstorage.d2", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(2048), + }, + NodeType: &models.ElasticsearchNodeType{ + Data: ec.Bool(true), + Ingest: ec.Bool(true), + Master: ec.Bool(false), + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(0), }, }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, }, }, }, - }, + }), }, { name: "parses an ES resource with config (HotWarm)", args: args{ - dt: hotWarmTpl(), + dt: hotWarmTpl770(), ess: []interface{}{ map[string]interface{}{ "ref_id": "main-elasticsearch", @@ -480,165 +503,179 @@ func Test_expandEsResource(t *testing.T) { }, }, }, - want: []*models.ElasticsearchPayload{ - { - Region: ec.String("some-region"), - RefID: ec.String("main-elasticsearch"), - Settings: &models.ElasticsearchClusterSettings{ - DedicatedMastersThreshold: 6, - Curation: nil, - }, - Plan: &models.ElasticsearchClusterPlan{ - Elasticsearch: &models.ElasticsearchConfiguration{ - Version: "7.7.0", - Curation: nil, - UserSettingsYaml: "somesetting: true", - }, - DeploymentTemplate: &models.DeploymentTemplateReference{ - ID: ec.String("aws-hot-warm-v2"), - }, - ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ - { - ID: "hot_content", - Elasticsearch: &models.ElasticsearchConfiguration{ - NodeAttributes: map[string]string{ - "data": "hot", - }, - }, - ZoneCount: 1, - InstanceConfigurationID: "aws.data.highio.i3", - Size: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(2048), - }, - NodeType: &models.ElasticsearchNodeType{ - Data: ec.Bool(true), - Ingest: ec.Bool(true), - Master: ec.Bool(true), - }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(1024), - }, - }, - }, - { - ID: "warm", - Elasticsearch: &models.ElasticsearchConfiguration{ - NodeAttributes: map[string]string{ - "data": "warm", - }, - }, - ZoneCount: 1, - InstanceConfigurationID: "aws.data.highstorage.d2", - Size: &models.TopologySize{ + want: enrichWithEmptyTopologies(hotWarmTpl770(), &models.ElasticsearchPayload{ + Region: ec.String("some-region"), + RefID: ec.String("main-elasticsearch"), + Settings: &models.ElasticsearchClusterSettings{ + DedicatedMastersThreshold: 6, + Curation: nil, + }, + Plan: &models.ElasticsearchClusterPlan{ + AutoscalingEnabled: ec.Bool(false), + Elasticsearch: &models.ElasticsearchConfiguration{ + Version: "7.7.0", + Curation: nil, + UserSettingsYaml: "somesetting: true", + }, + DeploymentTemplate: &models.DeploymentTemplateReference{ + ID: ec.String("aws-hot-warm-v2"), + }, + ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ + { + ID: "hot_content", + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{ + "data": "hot", + }, + }, + ZoneCount: 1, + InstanceConfigurationID: "aws.data.highio.i3", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(2048), + }, + NodeType: &models.ElasticsearchNodeType{ + Data: ec.Bool(true), + Ingest: ec.Bool(true), + Master: ec.Bool(true), + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ Resource: ec.String("memory"), - Value: ec.Int32(2048), + Value: ec.Int32(1024), }, - NodeType: &models.ElasticsearchNodeType{ - Data: ec.Bool(true), - Ingest: ec.Bool(true), - Master: ec.Bool(false), + }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, + }, + { + ID: "warm", + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{ + "data": "warm", }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(0), - }, + }, + ZoneCount: 1, + InstanceConfigurationID: "aws.data.highstorage.d2", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(2048), + }, + NodeType: &models.ElasticsearchNodeType{ + Data: ec.Bool(true), + Ingest: ec.Bool(true), + Master: ec.Bool(false), + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(0), }, }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, }, }, }, - }, + }), }, { name: "parses an ES resource without a topology (HotWarm)", args: args{ - dt: hotWarmTpl(), + dt: hotWarmTpl770(), ess: []interface{}{map[string]interface{}{ "ref_id": "main-elasticsearch", "resource_id": mock.ValidClusterID, "region": "some-region", }}, }, - want: []*models.ElasticsearchPayload{ - { - Region: ec.String("some-region"), - RefID: ec.String("main-elasticsearch"), - Settings: &models.ElasticsearchClusterSettings{ - DedicatedMastersThreshold: 6, - Curation: nil, - }, - Plan: &models.ElasticsearchClusterPlan{ - Elasticsearch: &models.ElasticsearchConfiguration{ - Version: "7.7.0", - Curation: nil, - }, - DeploymentTemplate: &models.DeploymentTemplateReference{ - ID: ec.String("aws-hot-warm-v2"), - }, - ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ - { - ID: "hot_content", - Elasticsearch: &models.ElasticsearchConfiguration{ - NodeAttributes: map[string]string{ - "data": "hot", - }, - }, - ZoneCount: 2, - InstanceConfigurationID: "aws.data.highio.i3", - Size: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(4096), - }, - NodeType: &models.ElasticsearchNodeType{ - Data: ec.Bool(true), - Ingest: ec.Bool(true), - Master: ec.Bool(true), - }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(1024), - }, - }, - }, - { - ID: "warm", - Elasticsearch: &models.ElasticsearchConfiguration{ - NodeAttributes: map[string]string{ - "data": "warm", - }, - }, - ZoneCount: 2, - InstanceConfigurationID: "aws.data.highstorage.d2", - Size: &models.TopologySize{ + want: enrichWithEmptyTopologies(hotWarmTpl770(), &models.ElasticsearchPayload{ + Region: ec.String("some-region"), + RefID: ec.String("main-elasticsearch"), + Settings: &models.ElasticsearchClusterSettings{ + DedicatedMastersThreshold: 6, + Curation: nil, + }, + Plan: &models.ElasticsearchClusterPlan{ + AutoscalingEnabled: ec.Bool(false), + Elasticsearch: &models.ElasticsearchConfiguration{ + Version: "7.7.0", + Curation: nil, + }, + DeploymentTemplate: &models.DeploymentTemplateReference{ + ID: ec.String("aws-hot-warm-v2"), + }, + ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ + { + ID: "hot_content", + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{ + "data": "hot", + }, + }, + ZoneCount: 2, + InstanceConfigurationID: "aws.data.highio.i3", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(4096), + }, + NodeType: &models.ElasticsearchNodeType{ + Data: ec.Bool(true), + Ingest: ec.Bool(true), + Master: ec.Bool(true), + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ Resource: ec.String("memory"), - Value: ec.Int32(4096), + Value: ec.Int32(1024), }, - NodeType: &models.ElasticsearchNodeType{ - Data: ec.Bool(true), - Ingest: ec.Bool(true), - Master: ec.Bool(false), + }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, + }, + { + ID: "warm", + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{ + "data": "warm", }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(0), - }, + }, + ZoneCount: 2, + InstanceConfigurationID: "aws.data.highstorage.d2", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(4096), + }, + NodeType: &models.ElasticsearchNodeType{ + Data: ec.Bool(true), + Ingest: ec.Bool(true), + Master: ec.Bool(false), + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(0), }, }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, }, }, }, - }, + }), }, { name: "parses an ES resource with node type overrides (HotWarm)", args: args{ - dt: hotWarmTpl(), + dt: hotWarmTpl770(), ess: []interface{}{map[string]interface{}{ "ref_id": "main-elasticsearch", "resource_id": mock.ValidClusterID, @@ -658,78 +695,85 @@ func Test_expandEsResource(t *testing.T) { }, }}, }, - want: []*models.ElasticsearchPayload{ - { - Region: ec.String("some-region"), - RefID: ec.String("main-elasticsearch"), - Settings: &models.ElasticsearchClusterSettings{ - DedicatedMastersThreshold: 6, - Curation: nil, - }, - Plan: &models.ElasticsearchClusterPlan{ - Elasticsearch: &models.ElasticsearchConfiguration{ - Version: "7.7.0", - Curation: nil, - }, - DeploymentTemplate: &models.DeploymentTemplateReference{ - ID: ec.String("aws-hot-warm-v2"), - }, - ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ - { - ID: "hot_content", - Elasticsearch: &models.ElasticsearchConfiguration{ - NodeAttributes: map[string]string{ - "data": "hot", - }, - }, - ZoneCount: 2, - InstanceConfigurationID: "aws.data.highio.i3", - Size: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(4096), - }, - NodeType: &models.ElasticsearchNodeType{ - Data: ec.Bool(false), - Ingest: ec.Bool(false), - Master: ec.Bool(false), - Ml: ec.Bool(true), - }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(1024), - }, - }, - }, - { - ID: "warm", - Elasticsearch: &models.ElasticsearchConfiguration{ - NodeAttributes: map[string]string{ - "data": "warm", - }, - }, - ZoneCount: 2, - InstanceConfigurationID: "aws.data.highstorage.d2", - Size: &models.TopologySize{ + want: enrichWithEmptyTopologies(hotWarmTpl770(), &models.ElasticsearchPayload{ + Region: ec.String("some-region"), + RefID: ec.String("main-elasticsearch"), + Settings: &models.ElasticsearchClusterSettings{ + DedicatedMastersThreshold: 6, + Curation: nil, + }, + Plan: &models.ElasticsearchClusterPlan{ + AutoscalingEnabled: ec.Bool(false), + Elasticsearch: &models.ElasticsearchConfiguration{ + Version: "7.7.0", + Curation: nil, + }, + DeploymentTemplate: &models.DeploymentTemplateReference{ + ID: ec.String("aws-hot-warm-v2"), + }, + ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ + { + ID: "hot_content", + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{ + "data": "hot", + }, + }, + ZoneCount: 2, + InstanceConfigurationID: "aws.data.highio.i3", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(4096), + }, + NodeType: &models.ElasticsearchNodeType{ + Data: ec.Bool(false), + Ingest: ec.Bool(false), + Master: ec.Bool(false), + Ml: ec.Bool(true), + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ Resource: ec.String("memory"), - Value: ec.Int32(4096), + Value: ec.Int32(1024), }, - NodeType: &models.ElasticsearchNodeType{ - Data: ec.Bool(true), - Ingest: ec.Bool(true), - Master: ec.Bool(true), + }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, + }, + { + ID: "warm", + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{ + "data": "warm", }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(0), - }, + }, + ZoneCount: 2, + InstanceConfigurationID: "aws.data.highstorage.d2", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(4096), + }, + NodeType: &models.ElasticsearchNodeType{ + Data: ec.Bool(true), + Ingest: ec.Bool(true), + Master: ec.Bool(true), + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(0), }, }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, }, }, }, - }, + }), }, { name: "migrates old node_type state to new node_roles payload when the cold tier is set", @@ -758,103 +802,587 @@ func Test_expandEsResource(t *testing.T) { }, }}, }, - want: []*models.ElasticsearchPayload{ - { - Region: ec.String("some-region"), - RefID: ec.String("main-elasticsearch"), - Settings: &models.ElasticsearchClusterSettings{ - DedicatedMastersThreshold: 6, - Curation: nil, - }, - Plan: &models.ElasticsearchClusterPlan{ - Elasticsearch: &models.ElasticsearchConfiguration{ - Version: "7.11.1", - Curation: nil, - }, - DeploymentTemplate: &models.DeploymentTemplateReference{ - ID: ec.String("aws-hot-warm-v2"), - }, - ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ - { - ID: "hot_content", - Elasticsearch: &models.ElasticsearchConfiguration{ - NodeAttributes: map[string]string{ - "data": "hot", - }, - }, - ZoneCount: 2, - InstanceConfigurationID: "aws.data.highio.i3", - Size: &models.TopologySize{ + want: enrichWithEmptyTopologies(hotWarm7111Tpl(), &models.ElasticsearchPayload{ + Region: ec.String("some-region"), + RefID: ec.String("main-elasticsearch"), + Settings: &models.ElasticsearchClusterSettings{ + DedicatedMastersThreshold: 6, + Curation: nil, + }, + Plan: &models.ElasticsearchClusterPlan{ + AutoscalingEnabled: ec.Bool(false), + Elasticsearch: &models.ElasticsearchConfiguration{ + Version: "7.11.1", + Curation: nil, + }, + DeploymentTemplate: &models.DeploymentTemplateReference{ + ID: ec.String("aws-hot-warm-v2"), + }, + ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ + { + ID: "hot_content", + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{ + "data": "hot", + }, + }, + ZoneCount: 2, + InstanceConfigurationID: "aws.data.highio.i3", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(4096), + }, + NodeRoles: []string{ + "master", + "ingest", + "remote_cluster_client", + "data_hot", + "transform", + "data_content", + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(1024), + }, + }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, + }, + { + ID: "warm", + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{ + "data": "warm", + }, + }, + ZoneCount: 2, + InstanceConfigurationID: "aws.data.highstorage.d2", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(4096), + }, + NodeRoles: []string{ + "data_warm", + "remote_cluster_client", + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(0), + }, + }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, + }, + { + ID: "cold", + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{ + "data": "cold", + }, + }, + ZoneCount: 1, + InstanceConfigurationID: "aws.data.highstorage.d2", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(2048), + }, + NodeRoles: []string{ + "data_cold", + "remote_cluster_client", + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(0), + }, + }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(59392), + Resource: ec.String("memory"), + }, + }, + }, + }, + }), + }, + { + name: "autoscaling enabled", + args: args{ + dt: hotWarm7111Tpl(), + ess: []interface{}{map[string]interface{}{ + "autoscale": "true", + "ref_id": "main-elasticsearch", + "resource_id": mock.ValidClusterID, + "region": "some-region", + "topology": []interface{}{ + map[string]interface{}{ + "id": "hot_content", + }, + map[string]interface{}{ + "id": "warm", + }, + map[string]interface{}{ + "id": "cold", + "size": "2g", + }, + }, + }}, + }, + want: enrichWithEmptyTopologies(hotWarm7111Tpl(), &models.ElasticsearchPayload{ + Region: ec.String("some-region"), + RefID: ec.String("main-elasticsearch"), + Settings: &models.ElasticsearchClusterSettings{ + DedicatedMastersThreshold: 6, + Curation: nil, + }, + Plan: &models.ElasticsearchClusterPlan{ + AutoscalingEnabled: ec.Bool(true), + Elasticsearch: &models.ElasticsearchConfiguration{ + Version: "7.11.1", + Curation: nil, + }, + DeploymentTemplate: &models.DeploymentTemplateReference{ + ID: ec.String("aws-hot-warm-v2"), + }, + ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ + { + ID: "hot_content", + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{ + "data": "hot", + }, + }, + ZoneCount: 2, + InstanceConfigurationID: "aws.data.highio.i3", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(4096), + }, + NodeRoles: []string{ + "master", + "ingest", + "remote_cluster_client", + "data_hot", + "transform", + "data_content", + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ Resource: ec.String("memory"), - Value: ec.Int32(4096), - }, - NodeRoles: []string{ - "master", - "ingest", - "remote_cluster_client", - "data_hot", - "transform", - "data_content", - }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(1024), - }, - }, - }, - { - ID: "warm", - Elasticsearch: &models.ElasticsearchConfiguration{ - NodeAttributes: map[string]string{ - "data": "warm", - }, - }, - ZoneCount: 2, - InstanceConfigurationID: "aws.data.highstorage.d2", - Size: &models.TopologySize{ + Value: ec.Int32(1024), + }, + }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, + }, + { + ID: "warm", + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{ + "data": "warm", + }, + }, + ZoneCount: 2, + InstanceConfigurationID: "aws.data.highstorage.d2", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(4096), + }, + NodeRoles: []string{ + "data_warm", + "remote_cluster_client", + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(0), + }, + }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, + }, + { + ID: "cold", + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{ + "data": "cold", + }, + }, + ZoneCount: 1, + InstanceConfigurationID: "aws.data.highstorage.d2", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(2048), + }, + NodeRoles: []string{ + "data_cold", + "remote_cluster_client", + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ Resource: ec.String("memory"), - Value: ec.Int32(4096), + Value: ec.Int32(0), + }, + }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(59392), + Resource: ec.String("memory"), + }, + }, + }, + }, + }), + }, + { + name: "autoscaling enabled overriding the size with ml", + args: args{ + dt: hotWarm7111Tpl(), + ess: []interface{}{map[string]interface{}{ + "autoscale": "true", + "ref_id": "main-elasticsearch", + "resource_id": mock.ValidClusterID, + "region": "some-region", + "topology": []interface{}{ + map[string]interface{}{ + "id": "hot_content", + "autoscaling": []interface{}{ + map[string]interface{}{ + "max_size": "58g", + }, + }, + }, + map[string]interface{}{ + "id": "warm", + "autoscaling": []interface{}{ + map[string]interface{}{ + "max_size": "29g", + }, + }, + }, + map[string]interface{}{ + "id": "cold", + "size": "2g", + "autoscaling": []interface{}{ + map[string]interface{}{ + "max_size": "29g", }, - NodeRoles: []string{ - "data_warm", - "remote_cluster_client", + }, + }, + map[string]interface{}{ + "id": "ml", + "size": "1g", + "autoscaling": []interface{}{ + map[string]interface{}{ + "max_size": "29g", + "min_size": "1g", + }, + }, + }, + }, + }}, + }, + want: enrichWithEmptyTopologies(hotWarm7111Tpl(), &models.ElasticsearchPayload{ + Region: ec.String("some-region"), + RefID: ec.String("main-elasticsearch"), + Settings: &models.ElasticsearchClusterSettings{ + DedicatedMastersThreshold: 6, + Curation: nil, + }, + Plan: &models.ElasticsearchClusterPlan{ + AutoscalingEnabled: ec.Bool(true), + Elasticsearch: &models.ElasticsearchConfiguration{ + Version: "7.11.1", + Curation: nil, + }, + DeploymentTemplate: &models.DeploymentTemplateReference{ + ID: ec.String("aws-hot-warm-v2"), + }, + ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ + { + ID: "hot_content", + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{ + "data": "hot", }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(0), - }, + }, + ZoneCount: 2, + InstanceConfigurationID: "aws.data.highio.i3", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(4096), + }, + NodeRoles: []string{ + "master", + "ingest", + "remote_cluster_client", + "data_hot", + "transform", + "data_content", + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(1024), }, }, - { - ID: "cold", - Elasticsearch: &models.ElasticsearchConfiguration{ - NodeAttributes: map[string]string{ - "data": "cold", - }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(59392), + Resource: ec.String("memory"), + }, + }, + { + ID: "warm", + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{ + "data": "warm", }, - ZoneCount: 1, - InstanceConfigurationID: "aws.data.highstorage.d2", - Size: &models.TopologySize{ + }, + ZoneCount: 2, + InstanceConfigurationID: "aws.data.highstorage.d2", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(4096), + }, + NodeRoles: []string{ + "data_warm", + "remote_cluster_client", + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ Resource: ec.String("memory"), - Value: ec.Int32(2048), + Value: ec.Int32(0), + }, + }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(29696), + Resource: ec.String("memory"), + }, + }, + { + ID: "cold", + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{ + "data": "cold", }, - NodeRoles: []string{ - "data_cold", - "remote_cluster_client", + }, + ZoneCount: 1, + InstanceConfigurationID: "aws.data.highstorage.d2", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(2048), + }, + NodeRoles: []string{ + "data_cold", + "remote_cluster_client", + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(0), }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(0), - }, + }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(29696), + Resource: ec.String("memory"), + }, + }, + { + ID: "ml", + ZoneCount: 1, + InstanceConfigurationID: "aws.ml.m5d", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(1024), + }, + NodeRoles: []string{ + "ml", + "remote_cluster_client", + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(0), }, }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(29696), + Resource: ec.String("memory"), + }, + AutoscalingMin: &models.TopologySize{ + Value: ec.Int32(1024), + Resource: ec.String("memory"), + }, }, }, }, + }), + }, + { + name: "autoscaling enabled overriding the size and resources", + args: args{ + dt: hotWarm7111Tpl(), + ess: []interface{}{map[string]interface{}{ + "autoscale": "true", + "ref_id": "main-elasticsearch", + "resource_id": mock.ValidClusterID, + "region": "some-region", + "topology": []interface{}{ + map[string]interface{}{ + "id": "hot_content", + "autoscaling": []interface{}{ + map[string]interface{}{ + "max_size_resource": "storage", + "max_size": "450g", + }, + }, + }, + map[string]interface{}{ + "id": "warm", + "autoscaling": []interface{}{ + map[string]interface{}{ + "max_size_resource": "storage", + "max_size": "870g", + }, + }, + }, + map[string]interface{}{ + "id": "cold", + "size": "4g", + "autoscaling": []interface{}{ + map[string]interface{}{ + "max_size_resource": "storage", + "max_size": "1740g", + + "min_size_resource": "storage", + "min_size": "4g", + }, + }, + }, + }, + }}, }, + want: enrichWithEmptyTopologies(hotWarm7111Tpl(), &models.ElasticsearchPayload{ + Region: ec.String("some-region"), + RefID: ec.String("main-elasticsearch"), + Settings: &models.ElasticsearchClusterSettings{ + DedicatedMastersThreshold: 6, + Curation: nil, + }, + Plan: &models.ElasticsearchClusterPlan{ + AutoscalingEnabled: ec.Bool(true), + Elasticsearch: &models.ElasticsearchConfiguration{ + Version: "7.11.1", + Curation: nil, + }, + DeploymentTemplate: &models.DeploymentTemplateReference{ + ID: ec.String("aws-hot-warm-v2"), + }, + ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ + { + ID: "hot_content", + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{ + "data": "hot", + }, + }, + ZoneCount: 2, + InstanceConfigurationID: "aws.data.highio.i3", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(4096), + }, + NodeRoles: []string{ + "master", + "ingest", + "remote_cluster_client", + "data_hot", + "transform", + "data_content", + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(1024), + }, + }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(460800), + Resource: ec.String("storage"), + }, + }, + { + ID: "warm", + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{ + "data": "warm", + }, + }, + ZoneCount: 2, + InstanceConfigurationID: "aws.data.highstorage.d2", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(4096), + }, + NodeRoles: []string{ + "data_warm", + "remote_cluster_client", + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(0), + }, + }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(890880), + Resource: ec.String("storage"), + }, + }, + { + ID: "cold", + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{ + "data": "cold", + }, + }, + ZoneCount: 1, + InstanceConfigurationID: "aws.data.highstorage.d2", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(4096), + }, + NodeRoles: []string{ + "data_cold", + "remote_cluster_client", + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(0), + }, + }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(1781760), + Resource: ec.String("storage"), + }, + AutoscalingMin: &models.TopologySize{ + Value: ec.Int32(4096), + Resource: ec.String("storage"), + }, + }, + }, + }, + }), }, { name: "parses an ES resource with plugins", @@ -882,57 +1410,60 @@ func Test_expandEsResource(t *testing.T) { }, }, }, - want: []*models.ElasticsearchPayload{ - { - Region: ec.String("some-region"), - RefID: ec.String("main-elasticsearch"), - Settings: &models.ElasticsearchClusterSettings{ - DedicatedMastersThreshold: 6, - }, - Plan: &models.ElasticsearchClusterPlan{ - Elasticsearch: &models.ElasticsearchConfiguration{ - Version: "7.7.0", - UserSettingsYaml: `some.setting: value`, - UserSettingsOverrideYaml: `some.setting: value2`, - UserSettingsJSON: map[string]interface{}{ - "some.setting": "value", - }, - UserSettingsOverrideJSON: map[string]interface{}{ - "some.setting": "value2", - }, - EnabledBuiltInPlugins: []string{"plugin"}, - }, - DeploymentTemplate: &models.DeploymentTemplateReference{ - ID: ec.String("aws-io-optimized-v2"), - }, - ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ - { - ID: "hot_content", - ZoneCount: 1, - InstanceConfigurationID: "aws.data.highio.i3", - Size: &models.TopologySize{ + want: enrichWithEmptyTopologies(tp770(), &models.ElasticsearchPayload{ + Region: ec.String("some-region"), + RefID: ec.String("main-elasticsearch"), + Settings: &models.ElasticsearchClusterSettings{ + DedicatedMastersThreshold: 6, + }, + Plan: &models.ElasticsearchClusterPlan{ + AutoscalingEnabled: ec.Bool(false), + Elasticsearch: &models.ElasticsearchConfiguration{ + Version: "7.7.0", + UserSettingsYaml: `some.setting: value`, + UserSettingsOverrideYaml: `some.setting: value2`, + UserSettingsJSON: map[string]interface{}{ + "some.setting": "value", + }, + UserSettingsOverrideJSON: map[string]interface{}{ + "some.setting": "value2", + }, + EnabledBuiltInPlugins: []string{"plugin"}, + }, + DeploymentTemplate: &models.DeploymentTemplateReference{ + ID: ec.String("aws-io-optimized-v2"), + }, + ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ + { + ID: "hot_content", + ZoneCount: 1, + InstanceConfigurationID: "aws.data.highio.i3", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(2048), + }, + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{"data": "hot"}, + }, + NodeType: &models.ElasticsearchNodeType{ + Data: ec.Bool(true), + Ingest: ec.Bool(true), + Master: ec.Bool(true), + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ Resource: ec.String("memory"), - Value: ec.Int32(2048), - }, - Elasticsearch: &models.ElasticsearchConfiguration{ - NodeAttributes: map[string]string{"data": "hot"}, - }, - NodeType: &models.ElasticsearchNodeType{ - Data: ec.Bool(true), - Ingest: ec.Bool(true), - Master: ec.Bool(true), - }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(1024), - }, + Value: ec.Int32(1024), }, }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, }, }, }, - }, + }), }, { name: "parses an ES resource with snapshot settings", @@ -955,54 +1486,57 @@ func Test_expandEsResource(t *testing.T) { }, }, }, - want: []*models.ElasticsearchPayload{ - { - Region: ec.String("some-region"), - RefID: ec.String("main-elasticsearch"), - Settings: &models.ElasticsearchClusterSettings{ - DedicatedMastersThreshold: 6, - }, - Plan: &models.ElasticsearchClusterPlan{ - Elasticsearch: &models.ElasticsearchConfiguration{ - Version: "7.7.0", - }, - DeploymentTemplate: &models.DeploymentTemplateReference{ - ID: ec.String("aws-io-optimized-v2"), - }, - Transient: &models.TransientElasticsearchPlanConfiguration{ - RestoreSnapshot: &models.RestoreSnapshotConfiguration{ - SnapshotName: ec.String("__latest_success__"), - SourceClusterID: mock.ValidClusterID, - }, - }, - ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ - { - ID: "hot_content", - ZoneCount: 1, - InstanceConfigurationID: "aws.data.highio.i3", - Size: &models.TopologySize{ + want: enrichWithEmptyTopologies(tp770(), &models.ElasticsearchPayload{ + Region: ec.String("some-region"), + RefID: ec.String("main-elasticsearch"), + Settings: &models.ElasticsearchClusterSettings{ + DedicatedMastersThreshold: 6, + }, + Plan: &models.ElasticsearchClusterPlan{ + AutoscalingEnabled: ec.Bool(false), + Elasticsearch: &models.ElasticsearchConfiguration{ + Version: "7.7.0", + }, + DeploymentTemplate: &models.DeploymentTemplateReference{ + ID: ec.String("aws-io-optimized-v2"), + }, + Transient: &models.TransientElasticsearchPlanConfiguration{ + RestoreSnapshot: &models.RestoreSnapshotConfiguration{ + SnapshotName: ec.String("__latest_success__"), + SourceClusterID: mock.ValidClusterID, + }, + }, + ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ + { + ID: "hot_content", + ZoneCount: 1, + InstanceConfigurationID: "aws.data.highio.i3", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(2048), + }, + NodeType: &models.ElasticsearchNodeType{ + Data: ec.Bool(true), + Ingest: ec.Bool(true), + Master: ec.Bool(true), + }, + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{"data": "hot"}, + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ Resource: ec.String("memory"), - Value: ec.Int32(2048), - }, - NodeType: &models.ElasticsearchNodeType{ - Data: ec.Bool(true), - Ingest: ec.Bool(true), - Master: ec.Bool(true), - }, - Elasticsearch: &models.ElasticsearchConfiguration{ - NodeAttributes: map[string]string{"data": "hot"}, - }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(1024), - }, + Value: ec.Int32(1024), }, }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, }, }, }, - }, + }), }, } for _, tt := range tests { diff --git a/ec/ecresource/deploymentresource/expanders_test.go b/ec/ecresource/deploymentresource/expanders_test.go index 32e5749c2..7688de14f 100644 --- a/ec/ecresource/deploymentresource/expanders_test.go +++ b/ec/ecresource/deploymentresource/expanders_test.go @@ -206,81 +206,88 @@ func Test_createResourceToModel(t *testing.T) { Tags: []*models.MetadataItem{}, }, Resources: &models.DeploymentCreateResources{ - Elasticsearch: []*models.ElasticsearchPayload{ - { - Region: ec.String("us-east-1"), - RefID: ec.String("main-elasticsearch"), - Settings: &models.ElasticsearchClusterSettings{ - DedicatedMastersThreshold: 6, + Elasticsearch: enrichWithEmptyTopologies(readerToESPayload(t, hotWarmTpl(), true), &models.ElasticsearchPayload{ + Region: ec.String("us-east-1"), + RefID: ec.String("main-elasticsearch"), + Settings: &models.ElasticsearchClusterSettings{ + DedicatedMastersThreshold: 6, + }, + Plan: &models.ElasticsearchClusterPlan{ + AutoscalingEnabled: ec.Bool(false), + Elasticsearch: &models.ElasticsearchConfiguration{ + Version: "7.11.1", + UserSettingsYaml: `some.setting: value`, + UserSettingsOverrideYaml: `some.setting: value2`, + UserSettingsJSON: map[string]interface{}{ + "some.setting": "value", + }, + UserSettingsOverrideJSON: map[string]interface{}{ + "some.setting": "value2", + }, }, - Plan: &models.ElasticsearchClusterPlan{ - Elasticsearch: &models.ElasticsearchConfiguration{ - Version: "7.11.1", - UserSettingsYaml: `some.setting: value`, - UserSettingsOverrideYaml: `some.setting: value2`, - UserSettingsJSON: map[string]interface{}{ - "some.setting": "value", + DeploymentTemplate: &models.DeploymentTemplateReference{ + ID: ec.String("aws-hot-warm-v2"), + }, + ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ + { + ID: "hot_content", + ZoneCount: 1, + InstanceConfigurationID: "aws.data.highio.i3", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(2048), }, - UserSettingsOverrideJSON: map[string]interface{}{ - "some.setting": "value2", + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{"data": "hot"}, }, - }, - DeploymentTemplate: &models.DeploymentTemplateReference{ - ID: ec.String("aws-hot-warm-v2"), - }, - ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ - { - ID: "hot_content", - ZoneCount: 1, - InstanceConfigurationID: "aws.data.highio.i3", - Size: &models.TopologySize{ + NodeRoles: []string{ + "data_content", + "data_hot", + "ingest", + "master", + "remote_cluster_client", + "transform", + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ Resource: ec.String("memory"), - Value: ec.Int32(2048), - }, - Elasticsearch: &models.ElasticsearchConfiguration{ - NodeAttributes: map[string]string{"data": "hot"}, - }, - NodeRoles: []string{ - "data_content", - "data_hot", - "ingest", - "master", - "remote_cluster_client", - "transform", - }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(1024), - }, + Value: ec.Int32(1024), }, }, - { - ID: "warm", - ZoneCount: 1, - InstanceConfigurationID: "aws.data.highstorage.d2", - Size: &models.TopologySize{ + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, + }, + { + ID: "warm", + ZoneCount: 1, + InstanceConfigurationID: "aws.data.highstorage.d2", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(2048), + }, + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{"data": "warm"}, + }, + NodeRoles: []string{ + "data_warm", + "remote_cluster_client", + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ Resource: ec.String("memory"), - Value: ec.Int32(2048), - }, - Elasticsearch: &models.ElasticsearchConfiguration{ - NodeAttributes: map[string]string{"data": "warm"}, - }, - NodeRoles: []string{ - "data_warm", - "remote_cluster_client", - }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(0), - }, + Value: ec.Int32(0), }, }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, }, }, }, - }, + }), Kibana: []*models.KibanaPayload{ { ElasticsearchClusterRefID: ec.String("main-elasticsearch"), @@ -396,55 +403,58 @@ func Test_createResourceToModel(t *testing.T) { Tags: []*models.MetadataItem{}, }, Resources: &models.DeploymentCreateResources{ - Elasticsearch: []*models.ElasticsearchPayload{ - { - Region: ec.String("us-east-1"), - RefID: ec.String("main-elasticsearch"), - Settings: &models.ElasticsearchClusterSettings{ - DedicatedMastersThreshold: 6, + Elasticsearch: enrichWithEmptyTopologies(readerToESPayload(t, ioOptimizedTpl(), false), &models.ElasticsearchPayload{ + Region: ec.String("us-east-1"), + RefID: ec.String("main-elasticsearch"), + Settings: &models.ElasticsearchClusterSettings{ + DedicatedMastersThreshold: 6, + }, + Plan: &models.ElasticsearchClusterPlan{ + AutoscalingEnabled: ec.Bool(false), + Elasticsearch: &models.ElasticsearchConfiguration{ + Version: "7.7.0", + UserSettingsYaml: `some.setting: value`, + UserSettingsOverrideYaml: `some.setting: value2`, + UserSettingsJSON: map[string]interface{}{ + "some.setting": "value", + }, + UserSettingsOverrideJSON: map[string]interface{}{ + "some.setting": "value2", + }, + }, + DeploymentTemplate: &models.DeploymentTemplateReference{ + ID: ec.String("aws-io-optimized-v2"), }, - Plan: &models.ElasticsearchClusterPlan{ + ClusterTopology: []*models.ElasticsearchClusterTopologyElement{{ + ID: "hot_content", + ZoneCount: 1, + InstanceConfigurationID: "aws.data.highio.i3", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(2048), + }, Elasticsearch: &models.ElasticsearchConfiguration{ - Version: "7.7.0", - UserSettingsYaml: `some.setting: value`, - UserSettingsOverrideYaml: `some.setting: value2`, - UserSettingsJSON: map[string]interface{}{ - "some.setting": "value", - }, - UserSettingsOverrideJSON: map[string]interface{}{ - "some.setting": "value2", - }, + NodeAttributes: map[string]string{"data": "hot"}, }, - DeploymentTemplate: &models.DeploymentTemplateReference{ - ID: ec.String("aws-io-optimized-v2"), + NodeType: &models.ElasticsearchNodeType{ + Data: ec.Bool(true), + Ingest: ec.Bool(true), + Master: ec.Bool(true), + Ml: ec.Bool(false), }, - ClusterTopology: []*models.ElasticsearchClusterTopologyElement{{ - ID: "hot_content", - ZoneCount: 1, - InstanceConfigurationID: "aws.data.highio.i3", - Size: &models.TopologySize{ + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ Resource: ec.String("memory"), - Value: ec.Int32(2048), - }, - Elasticsearch: &models.ElasticsearchConfiguration{ - NodeAttributes: map[string]string{"data": "hot"}, - }, - NodeType: &models.ElasticsearchNodeType{ - Data: ec.Bool(true), - Ingest: ec.Bool(true), - Master: ec.Bool(true), - Ml: ec.Bool(false), - }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(1024), - }, + Value: ec.Int32(1024), }, - }}, - }, + }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, + }}, }, - }, + }), Kibana: []*models.KibanaPayload{ { ElasticsearchClusterRefID: ec.String("main-elasticsearch"), @@ -546,46 +556,49 @@ func Test_createResourceToModel(t *testing.T) { Tags: []*models.MetadataItem{}, }, Resources: &models.DeploymentCreateResources{ - Elasticsearch: []*models.ElasticsearchPayload{ - { - Region: ec.String("us-east-1"), - RefID: ec.String("main-elasticsearch"), - Settings: &models.ElasticsearchClusterSettings{ - DedicatedMastersThreshold: 6, + Elasticsearch: enrichWithEmptyTopologies(readerToESPayload(t, ioOptimizedTpl(), false), &models.ElasticsearchPayload{ + Region: ec.String("us-east-1"), + RefID: ec.String("main-elasticsearch"), + Settings: &models.ElasticsearchClusterSettings{ + DedicatedMastersThreshold: 6, + }, + Plan: &models.ElasticsearchClusterPlan{ + AutoscalingEnabled: ec.Bool(false), + Elasticsearch: &models.ElasticsearchConfiguration{ + Version: "7.7.0", + }, + DeploymentTemplate: &models.DeploymentTemplateReference{ + ID: ec.String("aws-io-optimized-v2"), }, - Plan: &models.ElasticsearchClusterPlan{ + ClusterTopology: []*models.ElasticsearchClusterTopologyElement{{ + ID: "hot_content", Elasticsearch: &models.ElasticsearchConfiguration{ - Version: "7.7.0", + NodeAttributes: map[string]string{"data": "hot"}, + }, + ZoneCount: 2, + InstanceConfigurationID: "aws.data.highio.i3", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(8192), }, - DeploymentTemplate: &models.DeploymentTemplateReference{ - ID: ec.String("aws-io-optimized-v2"), + NodeType: &models.ElasticsearchNodeType{ + Data: ec.Bool(true), + Ingest: ec.Bool(true), + Master: ec.Bool(true), }, - ClusterTopology: []*models.ElasticsearchClusterTopologyElement{{ - ID: "hot_content", - Elasticsearch: &models.ElasticsearchConfiguration{ - NodeAttributes: map[string]string{"data": "hot"}, - }, - ZoneCount: 2, - InstanceConfigurationID: "aws.data.highio.i3", - Size: &models.TopologySize{ + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ Resource: ec.String("memory"), - Value: ec.Int32(8192), - }, - NodeType: &models.ElasticsearchNodeType{ - Data: ec.Bool(true), - Ingest: ec.Bool(true), - Master: ec.Bool(true), - }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(1024), - }, + Value: ec.Int32(1024), }, - }}, - }, + }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, + }}, }, - }, + }), Kibana: []*models.KibanaPayload{ { ElasticsearchClusterRefID: ec.String("main-elasticsearch"), @@ -683,49 +696,52 @@ func Test_createResourceToModel(t *testing.T) { Tags: []*models.MetadataItem{}, }, Resources: &models.DeploymentCreateResources{ - Elasticsearch: []*models.ElasticsearchPayload{ - { - Region: ec.String("us-east-1"), - RefID: ec.String("main-elasticsearch"), - Settings: &models.ElasticsearchClusterSettings{ - DedicatedMastersThreshold: 6, + Elasticsearch: enrichWithEmptyTopologies(readerToESPayload(t, ioOptimizedTpl(), true), &models.ElasticsearchPayload{ + Region: ec.String("us-east-1"), + RefID: ec.String("main-elasticsearch"), + Settings: &models.ElasticsearchClusterSettings{ + DedicatedMastersThreshold: 6, + }, + Plan: &models.ElasticsearchClusterPlan{ + AutoscalingEnabled: ec.Bool(false), + Elasticsearch: &models.ElasticsearchConfiguration{ + Version: "7.11.0", + }, + DeploymentTemplate: &models.DeploymentTemplateReference{ + ID: ec.String("aws-io-optimized-v2"), }, - Plan: &models.ElasticsearchClusterPlan{ + ClusterTopology: []*models.ElasticsearchClusterTopologyElement{{ + ID: "hot_content", Elasticsearch: &models.ElasticsearchConfiguration{ - Version: "7.11.0", + NodeAttributes: map[string]string{"data": "hot"}, }, - DeploymentTemplate: &models.DeploymentTemplateReference{ - ID: ec.String("aws-io-optimized-v2"), + ZoneCount: 2, + InstanceConfigurationID: "aws.data.highio.i3", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(8192), }, - ClusterTopology: []*models.ElasticsearchClusterTopologyElement{{ - ID: "hot_content", - Elasticsearch: &models.ElasticsearchConfiguration{ - NodeAttributes: map[string]string{"data": "hot"}, - }, - ZoneCount: 2, - InstanceConfigurationID: "aws.data.highio.i3", - Size: &models.TopologySize{ + NodeRoles: []string{ + "master", + "ingest", + "remote_cluster_client", + "data_hot", + "transform", + "data_content", + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ Resource: ec.String("memory"), - Value: ec.Int32(8192), - }, - NodeRoles: []string{ - "master", - "ingest", - "remote_cluster_client", - "data_hot", - "transform", - "data_content", - }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(1024), - }, + Value: ec.Int32(1024), }, - }}, - }, + }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, + }}, }, - }, + }), Kibana: []*models.KibanaPayload{ { ElasticsearchClusterRefID: ec.String("main-elasticsearch"), @@ -809,46 +825,49 @@ func Test_createResourceToModel(t *testing.T) { Tags: []*models.MetadataItem{}, }, Resources: &models.DeploymentCreateResources{ - Elasticsearch: []*models.ElasticsearchPayload{ - { - Region: ec.String("us-east-1"), - RefID: ec.String("main-elasticsearch"), - Settings: &models.ElasticsearchClusterSettings{ - DedicatedMastersThreshold: 6, + Elasticsearch: enrichWithEmptyTopologies(readerToESPayload(t, ioOptimizedTpl(), false), &models.ElasticsearchPayload{ + Region: ec.String("us-east-1"), + RefID: ec.String("main-elasticsearch"), + Settings: &models.ElasticsearchClusterSettings{ + DedicatedMastersThreshold: 6, + }, + Plan: &models.ElasticsearchClusterPlan{ + AutoscalingEnabled: ec.Bool(false), + Elasticsearch: &models.ElasticsearchConfiguration{ + Version: "7.7.0", + }, + DeploymentTemplate: &models.DeploymentTemplateReference{ + ID: ec.String("aws-io-optimized-v2"), }, - Plan: &models.ElasticsearchClusterPlan{ + ClusterTopology: []*models.ElasticsearchClusterTopologyElement{{ + ID: "hot_content", Elasticsearch: &models.ElasticsearchConfiguration{ - Version: "7.7.0", - }, - DeploymentTemplate: &models.DeploymentTemplateReference{ - ID: ec.String("aws-io-optimized-v2"), + NodeAttributes: map[string]string{"data": "hot"}, }, - ClusterTopology: []*models.ElasticsearchClusterTopologyElement{{ - ID: "hot_content", - Elasticsearch: &models.ElasticsearchConfiguration{ - NodeAttributes: map[string]string{"data": "hot"}, - }, - ZoneCount: 2, - InstanceConfigurationID: "aws.data.highio.i3", - Size: &models.TopologySize{ + ZoneCount: 2, + InstanceConfigurationID: "aws.data.highio.i3", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(4096), + }, + NodeType: &models.ElasticsearchNodeType{ + Data: ec.Bool(true), + Ingest: ec.Bool(true), + Master: ec.Bool(true), + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ Resource: ec.String("memory"), - Value: ec.Int32(4096), - }, - NodeType: &models.ElasticsearchNodeType{ - Data: ec.Bool(true), - Ingest: ec.Bool(true), - Master: ec.Bool(true), - }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(1024), - }, + Value: ec.Int32(1024), }, - }}, - }, + }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, + }}, }, - }, + }), Kibana: []*models.KibanaPayload{ { ElasticsearchClusterRefID: ec.String("main-elasticsearch"), @@ -932,46 +951,49 @@ func Test_createResourceToModel(t *testing.T) { Tags: []*models.MetadataItem{}, }, Resources: &models.DeploymentCreateResources{ - Elasticsearch: []*models.ElasticsearchPayload{ - { - Region: ec.String("us-east-1"), - RefID: ec.String("main-elasticsearch"), - Settings: &models.ElasticsearchClusterSettings{ - DedicatedMastersThreshold: 6, + Elasticsearch: enrichWithEmptyTopologies(readerToESPayload(t, ioOptimizedTpl(), false), &models.ElasticsearchPayload{ + Region: ec.String("us-east-1"), + RefID: ec.String("main-elasticsearch"), + Settings: &models.ElasticsearchClusterSettings{ + DedicatedMastersThreshold: 6, + }, + Plan: &models.ElasticsearchClusterPlan{ + AutoscalingEnabled: ec.Bool(false), + Elasticsearch: &models.ElasticsearchConfiguration{ + Version: "7.7.0", + }, + DeploymentTemplate: &models.DeploymentTemplateReference{ + ID: ec.String("aws-io-optimized-v2"), }, - Plan: &models.ElasticsearchClusterPlan{ + ClusterTopology: []*models.ElasticsearchClusterTopologyElement{{ + ID: "hot_content", Elasticsearch: &models.ElasticsearchConfiguration{ - Version: "7.7.0", + NodeAttributes: map[string]string{"data": "hot"}, }, - DeploymentTemplate: &models.DeploymentTemplateReference{ - ID: ec.String("aws-io-optimized-v2"), + ZoneCount: 2, + InstanceConfigurationID: "aws.data.highio.i3", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(8192), }, - ClusterTopology: []*models.ElasticsearchClusterTopologyElement{{ - ID: "hot_content", - Elasticsearch: &models.ElasticsearchConfiguration{ - NodeAttributes: map[string]string{"data": "hot"}, - }, - ZoneCount: 2, - InstanceConfigurationID: "aws.data.highio.i3", - Size: &models.TopologySize{ + NodeType: &models.ElasticsearchNodeType{ + Data: ec.Bool(true), + Ingest: ec.Bool(true), + Master: ec.Bool(true), + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ Resource: ec.String("memory"), - Value: ec.Int32(8192), - }, - NodeType: &models.ElasticsearchNodeType{ - Data: ec.Bool(true), - Ingest: ec.Bool(true), - Master: ec.Bool(true), - }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(1024), - }, + Value: ec.Int32(1024), }, - }}, - }, + }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, + }}, }, - }, + }), Kibana: []*models.KibanaPayload{ { ElasticsearchClusterRefID: ec.String("main-elasticsearch"), @@ -1051,75 +1073,82 @@ func Test_createResourceToModel(t *testing.T) { Tags: []*models.MetadataItem{}, }, Resources: &models.DeploymentCreateResources{ - Elasticsearch: []*models.ElasticsearchPayload{ - { - Region: ec.String("us-east-1"), - RefID: ec.String("main-elasticsearch"), - Settings: &models.ElasticsearchClusterSettings{ - DedicatedMastersThreshold: 6, - Curation: nil, + Elasticsearch: enrichWithEmptyTopologies(readerToESPayload(t, hotWarmTpl(), false), &models.ElasticsearchPayload{ + Region: ec.String("us-east-1"), + RefID: ec.String("main-elasticsearch"), + Settings: &models.ElasticsearchClusterSettings{ + DedicatedMastersThreshold: 6, + Curation: nil, + }, + Plan: &models.ElasticsearchClusterPlan{ + AutoscalingEnabled: ec.Bool(false), + Elasticsearch: &models.ElasticsearchConfiguration{ + Curation: nil, + Version: "7.9.2", }, - Plan: &models.ElasticsearchClusterPlan{ - Elasticsearch: &models.ElasticsearchConfiguration{ - Curation: nil, - Version: "7.9.2", - }, - DeploymentTemplate: &models.DeploymentTemplateReference{ - ID: ec.String("aws-hot-warm-v2"), - }, - ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ - { - ID: "hot_content", - ZoneCount: 2, - InstanceConfigurationID: "aws.data.highio.i3", - Size: &models.TopologySize{ + DeploymentTemplate: &models.DeploymentTemplateReference{ + ID: ec.String("aws-hot-warm-v2"), + }, + ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ + { + ID: "hot_content", + ZoneCount: 2, + InstanceConfigurationID: "aws.data.highio.i3", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(4096), + }, + NodeType: &models.ElasticsearchNodeType{ + Data: ec.Bool(true), + Ingest: ec.Bool(true), + Master: ec.Bool(true), + }, + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{"data": "hot"}, + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ Resource: ec.String("memory"), - Value: ec.Int32(4096), - }, - NodeType: &models.ElasticsearchNodeType{ - Data: ec.Bool(true), - Ingest: ec.Bool(true), - Master: ec.Bool(true), - }, - Elasticsearch: &models.ElasticsearchConfiguration{ - NodeAttributes: map[string]string{"data": "hot"}, + Value: ec.Int32(1024), }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(1024), - }, + }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, + }, + { + ID: "warm", + ZoneCount: 2, + InstanceConfigurationID: "aws.data.highstorage.d2", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(4096), + }, + NodeType: &models.ElasticsearchNodeType{ + Data: ec.Bool(true), + Ingest: ec.Bool(true), + Master: ec.Bool(false), + }, + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{ + "data": "warm", }, }, - { - ID: "warm", - ZoneCount: 2, - InstanceConfigurationID: "aws.data.highstorage.d2", - Size: &models.TopologySize{ + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ Resource: ec.String("memory"), - Value: ec.Int32(4096), - }, - NodeType: &models.ElasticsearchNodeType{ - Data: ec.Bool(true), - Ingest: ec.Bool(true), - Master: ec.Bool(false), - }, - Elasticsearch: &models.ElasticsearchConfiguration{ - NodeAttributes: map[string]string{ - "data": "warm", - }, - }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(0), - }, + Value: ec.Int32(0), }, }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, }, }, }, - }, + }), Kibana: []*models.KibanaPayload{ { ElasticsearchClusterRefID: ec.String("main-elasticsearch"), @@ -1167,77 +1196,84 @@ func Test_createResourceToModel(t *testing.T) { Tags: []*models.MetadataItem{}, }, Resources: &models.DeploymentCreateResources{ - Elasticsearch: []*models.ElasticsearchPayload{ - { - Region: ec.String("us-east-1"), - RefID: ec.String("main-elasticsearch"), - Settings: &models.ElasticsearchClusterSettings{ - DedicatedMastersThreshold: 6, - Curation: nil, + Elasticsearch: enrichWithEmptyTopologies(readerToESPayload(t, hotWarmTpl(), true), &models.ElasticsearchPayload{ + Region: ec.String("us-east-1"), + RefID: ec.String("main-elasticsearch"), + Settings: &models.ElasticsearchClusterSettings{ + DedicatedMastersThreshold: 6, + Curation: nil, + }, + Plan: &models.ElasticsearchClusterPlan{ + AutoscalingEnabled: ec.Bool(false), + Elasticsearch: &models.ElasticsearchConfiguration{ + Curation: nil, + Version: "7.12.0", }, - Plan: &models.ElasticsearchClusterPlan{ - Elasticsearch: &models.ElasticsearchConfiguration{ - Curation: nil, - Version: "7.12.0", - }, - DeploymentTemplate: &models.DeploymentTemplateReference{ - ID: ec.String("aws-hot-warm-v2"), - }, - ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ - { - ID: "hot_content", - ZoneCount: 2, - InstanceConfigurationID: "aws.data.highio.i3", - Size: &models.TopologySize{ + DeploymentTemplate: &models.DeploymentTemplateReference{ + ID: ec.String("aws-hot-warm-v2"), + }, + ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ + { + ID: "hot_content", + ZoneCount: 2, + InstanceConfigurationID: "aws.data.highio.i3", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(4096), + }, + NodeRoles: []string{ + "master", + "ingest", + "remote_cluster_client", + "data_hot", + "transform", + "data_content", + }, + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{"data": "hot"}, + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ Resource: ec.String("memory"), - Value: ec.Int32(4096), - }, - NodeRoles: []string{ - "master", - "ingest", - "remote_cluster_client", - "data_hot", - "transform", - "data_content", - }, - Elasticsearch: &models.ElasticsearchConfiguration{ - NodeAttributes: map[string]string{"data": "hot"}, + Value: ec.Int32(1024), }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(1024), - }, + }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, + }, + { + ID: "warm", + ZoneCount: 2, + InstanceConfigurationID: "aws.data.highstorage.d2", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(4096), + }, + NodeRoles: []string{ + "data_warm", + "remote_cluster_client", + }, + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{ + "data": "warm", }, }, - { - ID: "warm", - ZoneCount: 2, - InstanceConfigurationID: "aws.data.highstorage.d2", - Size: &models.TopologySize{ + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ Resource: ec.String("memory"), - Value: ec.Int32(4096), - }, - NodeRoles: []string{ - "data_warm", - "remote_cluster_client", - }, - Elasticsearch: &models.ElasticsearchConfiguration{ - NodeAttributes: map[string]string{ - "data": "warm", - }, - }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(0), - }, + Value: ec.Int32(0), }, }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, }, }, }, - }, + }), Kibana: []*models.KibanaPayload{ { ElasticsearchClusterRefID: ec.String("main-elasticsearch"), @@ -1311,99 +1347,404 @@ func Test_createResourceToModel(t *testing.T) { Tags: []*models.MetadataItem{}, }, Resources: &models.DeploymentCreateResources{ - Elasticsearch: []*models.ElasticsearchPayload{ - { - Region: ec.String("us-east-1"), - RefID: ec.String("main-elasticsearch"), - Settings: &models.ElasticsearchClusterSettings{ - DedicatedMastersThreshold: 6, - }, - Plan: &models.ElasticsearchClusterPlan{ - Elasticsearch: &models.ElasticsearchConfiguration{ - Version: "7.12.0", - UserBundles: []*models.ElasticsearchUserBundle{ - { - URL: ec.String("repo://1231122"), - Name: ec.String("my-bundle"), - ElasticsearchVersion: ec.String("7.7.0"), - }, - { - URL: ec.String("repo://1231123"), - Name: ec.String("my-second-bundle"), - ElasticsearchVersion: ec.String("7.7.0"), - }, + Elasticsearch: enrichWithEmptyTopologies(readerToESPayload(t, hotWarmTpl(), true), &models.ElasticsearchPayload{ + Region: ec.String("us-east-1"), + RefID: ec.String("main-elasticsearch"), + Settings: &models.ElasticsearchClusterSettings{ + DedicatedMastersThreshold: 6, + }, + Plan: &models.ElasticsearchClusterPlan{ + AutoscalingEnabled: ec.Bool(false), + Elasticsearch: &models.ElasticsearchConfiguration{ + Version: "7.12.0", + UserBundles: []*models.ElasticsearchUserBundle{ + { + URL: ec.String("repo://1231122"), + Name: ec.String("my-bundle"), + ElasticsearchVersion: ec.String("7.7.0"), }, - UserPlugins: []*models.ElasticsearchUserPlugin{ - { - URL: ec.String("repo://12311235"), - Name: ec.String("my-second-plugin"), - ElasticsearchVersion: ec.String("7.7.0"), - }, - { - URL: ec.String("repo://12311234"), - Name: ec.String("my-plugin"), - ElasticsearchVersion: ec.String("7.7.0"), - }, + { + URL: ec.String("repo://1231123"), + Name: ec.String("my-second-bundle"), + ElasticsearchVersion: ec.String("7.7.0"), }, }, - DeploymentTemplate: &models.DeploymentTemplateReference{ - ID: ec.String("aws-hot-warm-v2"), - }, - ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ + UserPlugins: []*models.ElasticsearchUserPlugin{ { - ID: "hot_content", - ZoneCount: 2, - InstanceConfigurationID: "aws.data.highio.i3", - Size: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(4096), - }, - NodeRoles: []string{ - "master", - "ingest", - "remote_cluster_client", - "data_hot", - "transform", - "data_content", - }, - Elasticsearch: &models.ElasticsearchConfiguration{ - NodeAttributes: map[string]string{"data": "hot"}, - }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(1024), - }, - }, + URL: ec.String("repo://12311235"), + Name: ec.String("my-second-plugin"), + ElasticsearchVersion: ec.String("7.7.0"), }, { - ID: "warm", - ZoneCount: 2, - InstanceConfigurationID: "aws.data.highstorage.d2", - Size: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(4096), - }, - NodeRoles: []string{ - "data_warm", - "remote_cluster_client", + URL: ec.String("repo://12311234"), + Name: ec.String("my-plugin"), + ElasticsearchVersion: ec.String("7.7.0"), + }, + }, + }, + DeploymentTemplate: &models.DeploymentTemplateReference{ + ID: ec.String("aws-hot-warm-v2"), + }, + ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ + { + ID: "hot_content", + ZoneCount: 2, + InstanceConfigurationID: "aws.data.highio.i3", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(4096), + }, + NodeRoles: []string{ + "master", + "ingest", + "remote_cluster_client", + "data_hot", + "transform", + "data_content", + }, + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{"data": "hot"}, + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(1024), }, - Elasticsearch: &models.ElasticsearchConfiguration{ - NodeAttributes: map[string]string{ - "data": "warm", - }, + }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, + }, + { + ID: "warm", + ZoneCount: 2, + InstanceConfigurationID: "aws.data.highstorage.d2", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(4096), + }, + NodeRoles: []string{ + "data_warm", + "remote_cluster_client", + }, + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{ + "data": "warm", }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(0), - }, + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(0), }, }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, }, }, }, + }), + }, + }, + }, + { + name: "deployment with autoscaling enabled", + args: args{ + d: util.NewResourceData(t, util.ResDataParams{ + ID: mock.ValidClusterID, + Schema: newSchema(), + State: map[string]interface{}{ + "name": "my_deployment_name", + "deployment_template_id": "aws-io-optimized-v2", + "region": "us-east-1", + "version": "7.12.0", + "elasticsearch": []interface{}{map[string]interface{}{ + "autoscale": "true", + "topology": []interface{}{ + map[string]interface{}{ + "id": "cold", + "size": "2g", + }, + map[string]interface{}{ + "id": "hot_content", + "size": "8g", + }, + map[string]interface{}{ + "id": "warm", + "size": "4g", + }, + }, + }}, + }, + }), + client: api.NewMock(mock.New200Response(ioOptimizedTpl())), + }, + want: &models.DeploymentCreateRequest{ + Name: "my_deployment_name", + Settings: &models.DeploymentCreateSettings{}, + Metadata: &models.DeploymentCreateMetadata{ + Tags: []*models.MetadataItem{}, + }, + Resources: &models.DeploymentCreateResources{ + Elasticsearch: enrichWithEmptyTopologies(readerToESPayload(t, ioOptimizedTpl(), true), &models.ElasticsearchPayload{ + Region: ec.String("us-east-1"), + RefID: ec.String("main-elasticsearch"), + Settings: &models.ElasticsearchClusterSettings{ + DedicatedMastersThreshold: 6, + }, + Plan: &models.ElasticsearchClusterPlan{ + AutoscalingEnabled: ec.Bool(true), + Elasticsearch: &models.ElasticsearchConfiguration{ + Version: "7.12.0", + }, + DeploymentTemplate: &models.DeploymentTemplateReference{ + ID: ec.String("aws-io-optimized-v2"), + }, + ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ + { + ID: "cold", + ZoneCount: 1, + InstanceConfigurationID: "aws.data.highstorage.d3", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(2048), + }, + NodeRoles: []string{ + "data_cold", + "remote_cluster_client", + }, + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{ + "data": "cold", + }, + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(0), + }, + }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(59392), + Resource: ec.String("memory"), + }, + }, + { + ID: "hot_content", + ZoneCount: 2, + InstanceConfigurationID: "aws.data.highio.i3", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(8192), + }, + NodeRoles: []string{ + "master", + "ingest", + "remote_cluster_client", + "data_hot", + "transform", + "data_content", + }, + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{"data": "hot"}, + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(1024), + }, + }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, + }, + { + ID: "warm", + ZoneCount: 2, + InstanceConfigurationID: "aws.data.highstorage.d3", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(4096), + }, + NodeRoles: []string{ + "data_warm", + "remote_cluster_client", + }, + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{ + "data": "warm", + }, + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(0), + }, + }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, + }, + }, + }, + }), + }, + }, + }, + { + name: "deployment with autoscaling enabled and custom policies set", + args: args{ + d: util.NewResourceData(t, util.ResDataParams{ + ID: mock.ValidClusterID, + Schema: newSchema(), + State: map[string]interface{}{ + "name": "my_deployment_name", + "deployment_template_id": "aws-io-optimized-v2", + "region": "us-east-1", + "version": "7.12.0", + "elasticsearch": []interface{}{map[string]interface{}{ + "autoscale": "true", + "topology": []interface{}{ + map[string]interface{}{ + "id": "cold", + "size": "2g", + }, + map[string]interface{}{ + "id": "hot_content", + "size": "8g", + "autoscaling": []interface{}{map[string]interface{}{ + "max_size": "232g", + }}, + }, + map[string]interface{}{ + "id": "warm", + "size": "4g", + "autoscaling": []interface{}{map[string]interface{}{ + "max_size": "116g", + }}, + }, + }, + }}, }, + }), + client: api.NewMock(mock.New200Response(ioOptimizedTpl())), + }, + want: &models.DeploymentCreateRequest{ + Name: "my_deployment_name", + Settings: &models.DeploymentCreateSettings{}, + Metadata: &models.DeploymentCreateMetadata{ + Tags: []*models.MetadataItem{}, + }, + Resources: &models.DeploymentCreateResources{ + Elasticsearch: enrichWithEmptyTopologies(readerToESPayload(t, ioOptimizedTpl(), true), &models.ElasticsearchPayload{ + Region: ec.String("us-east-1"), + RefID: ec.String("main-elasticsearch"), + Settings: &models.ElasticsearchClusterSettings{ + DedicatedMastersThreshold: 6, + }, + Plan: &models.ElasticsearchClusterPlan{ + AutoscalingEnabled: ec.Bool(true), + Elasticsearch: &models.ElasticsearchConfiguration{ + Version: "7.12.0", + }, + DeploymentTemplate: &models.DeploymentTemplateReference{ + ID: ec.String("aws-io-optimized-v2"), + }, + ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ + { + ID: "cold", + ZoneCount: 1, + InstanceConfigurationID: "aws.data.highstorage.d3", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(2048), + }, + NodeRoles: []string{ + "data_cold", + "remote_cluster_client", + }, + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{ + "data": "cold", + }, + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(0), + }, + }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(59392), + Resource: ec.String("memory"), + }, + }, + { + ID: "hot_content", + ZoneCount: 2, + InstanceConfigurationID: "aws.data.highio.i3", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(8192), + }, + NodeRoles: []string{ + "master", + "ingest", + "remote_cluster_client", + "data_hot", + "transform", + "data_content", + }, + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{"data": "hot"}, + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(1024), + }, + }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(237568), + Resource: ec.String("memory"), + }, + }, + { + ID: "warm", + ZoneCount: 2, + InstanceConfigurationID: "aws.data.highstorage.d3", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(4096), + }, + NodeRoles: []string{ + "data_warm", + "remote_cluster_client", + }, + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{ + "data": "warm", + }, + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(0), + }, + }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, + }, + }, + }, + }), }, }, }, @@ -1450,117 +1791,128 @@ func Test_createResourceToModel(t *testing.T) { Tags: []*models.MetadataItem{}, }, Resources: &models.DeploymentCreateResources{ - Elasticsearch: []*models.ElasticsearchPayload{ - { - Region: ec.String("us-east-1"), - RefID: ec.String("main-elasticsearch"), - Settings: &models.ElasticsearchClusterSettings{ - DedicatedMastersThreshold: 6, + Elasticsearch: enrichWithEmptyTopologies(readerToESPayload(t, ioOptimizedTpl(), true), &models.ElasticsearchPayload{ + Region: ec.String("us-east-1"), + RefID: ec.String("main-elasticsearch"), + Settings: &models.ElasticsearchClusterSettings{ + DedicatedMastersThreshold: 6, + }, + Plan: &models.ElasticsearchClusterPlan{ + AutoscalingEnabled: ec.Bool(false), + Elasticsearch: &models.ElasticsearchConfiguration{ + Version: "7.12.0", }, - Plan: &models.ElasticsearchClusterPlan{ - Elasticsearch: &models.ElasticsearchConfiguration{ - Version: "7.12.0", - }, - DeploymentTemplate: &models.DeploymentTemplateReference{ - ID: ec.String("aws-io-optimized-v2"), - }, - ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ - { - ID: "cold", - ZoneCount: 1, - InstanceConfigurationID: "aws.data.highstorage.d3", - Size: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(2048), + DeploymentTemplate: &models.DeploymentTemplateReference{ + ID: ec.String("aws-io-optimized-v2"), + }, + ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ + { + ID: "cold", + ZoneCount: 1, + InstanceConfigurationID: "aws.data.highstorage.d3", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(2048), + }, + NodeRoles: []string{ + "data_cold", + "remote_cluster_client", + }, + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{ + "data": "cold", }, - NodeRoles: []string{ - "data_cold", - "remote_cluster_client", + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(0), }, - Elasticsearch: &models.ElasticsearchConfiguration{ - NodeAttributes: map[string]string{ - "data": "cold", - }, + }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(59392), + Resource: ec.String("memory"), + }, + }, + { + ID: "hot_content", + ZoneCount: 2, + InstanceConfigurationID: "aws.data.highio.i3", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(8192), + }, + NodeRoles: []string{ + "ingest", + "remote_cluster_client", + "data_hot", + "transform", + "data_content", + }, + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{"data": "hot"}, + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(1024), }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(0), - }, + }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, + }, + { + ID: "master", + ZoneCount: 3, + InstanceConfigurationID: "aws.master.r5d", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(1024), + }, + NodeRoles: []string{ + "master", + "remote_cluster_client", + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(0), }, }, - { - ID: "hot_content", - ZoneCount: 2, - InstanceConfigurationID: "aws.data.highio.i3", - Size: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(8192), - }, - NodeRoles: []string{ - "ingest", - "remote_cluster_client", - "data_hot", - "transform", - "data_content", - }, - Elasticsearch: &models.ElasticsearchConfiguration{ - NodeAttributes: map[string]string{"data": "hot"}, - }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(1024), - }, + }, + { + ID: "warm", + ZoneCount: 2, + InstanceConfigurationID: "aws.data.highstorage.d3", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(4096), + }, + NodeRoles: []string{ + "data_warm", + "remote_cluster_client", + }, + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{ + "data": "warm", }, }, - { - ID: "master", - ZoneCount: 3, - InstanceConfigurationID: "aws.master.r5d", - Size: &models.TopologySize{ + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ Resource: ec.String("memory"), - Value: ec.Int32(1024), - }, - NodeRoles: []string{ - "master", - "remote_cluster_client", - }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(0), - }, + Value: ec.Int32(0), }, }, - { - ID: "warm", - ZoneCount: 2, - InstanceConfigurationID: "aws.data.highstorage.d3", - Size: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(4096), - }, - NodeRoles: []string{ - "data_warm", - "remote_cluster_client", - }, - Elasticsearch: &models.ElasticsearchConfiguration{ - NodeAttributes: map[string]string{ - "data": "warm", - }, - }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(0), - }, - }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), }, }, }, }, - }, + }), }, }, }, @@ -1607,117 +1959,128 @@ func Test_createResourceToModel(t *testing.T) { Tags: []*models.MetadataItem{}, }, Resources: &models.DeploymentCreateResources{ - Elasticsearch: []*models.ElasticsearchPayload{ - { - Region: ec.String("us-east-1"), - RefID: ec.String("main-elasticsearch"), - Settings: &models.ElasticsearchClusterSettings{ - DedicatedMastersThreshold: 6, + Elasticsearch: enrichWithEmptyTopologies(readerToESPayload(t, ioOptimizedTpl(), true), &models.ElasticsearchPayload{ + Region: ec.String("us-east-1"), + RefID: ec.String("main-elasticsearch"), + Settings: &models.ElasticsearchClusterSettings{ + DedicatedMastersThreshold: 6, + }, + Plan: &models.ElasticsearchClusterPlan{ + AutoscalingEnabled: ec.Bool(false), + Elasticsearch: &models.ElasticsearchConfiguration{ + Version: "7.12.0", }, - Plan: &models.ElasticsearchClusterPlan{ - Elasticsearch: &models.ElasticsearchConfiguration{ - Version: "7.12.0", - }, - DeploymentTemplate: &models.DeploymentTemplateReference{ - ID: ec.String("aws-io-optimized-v2"), - }, - ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ - { - ID: "cold", - ZoneCount: 1, - InstanceConfigurationID: "aws.data.highstorage.d3", - Size: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(2048), - }, - NodeRoles: []string{ - "data_cold", - "remote_cluster_client", - }, - Elasticsearch: &models.ElasticsearchConfiguration{ - NodeAttributes: map[string]string{ - "data": "cold", - }, - }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(0), - }, + DeploymentTemplate: &models.DeploymentTemplateReference{ + ID: ec.String("aws-io-optimized-v2"), + }, + ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ + { + ID: "cold", + ZoneCount: 1, + InstanceConfigurationID: "aws.data.highstorage.d3", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(2048), + }, + NodeRoles: []string{ + "data_cold", + "remote_cluster_client", + }, + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{ + "data": "cold", }, }, - { - ID: "coordinating", - ZoneCount: 2, - InstanceConfigurationID: "aws.coordinating.m5d", - Size: &models.TopologySize{ + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ Resource: ec.String("memory"), - Value: ec.Int32(2048), - }, - NodeRoles: []string{ - "ingest", - "remote_cluster_client", - }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(0), - }, + Value: ec.Int32(0), }, }, - { - ID: "hot_content", - ZoneCount: 2, - InstanceConfigurationID: "aws.data.highio.i3", - Size: &models.TopologySize{ + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(59392), + Resource: ec.String("memory"), + }, + }, + { + ID: "coordinating", + ZoneCount: 2, + InstanceConfigurationID: "aws.coordinating.m5d", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(2048), + }, + NodeRoles: []string{ + "ingest", + "remote_cluster_client", + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ Resource: ec.String("memory"), - Value: ec.Int32(8192), - }, - NodeRoles: []string{ - "master", - "remote_cluster_client", - "data_hot", - "transform", - "data_content", - }, - Elasticsearch: &models.ElasticsearchConfiguration{ - NodeAttributes: map[string]string{"data": "hot"}, - }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(1024), - }, + Value: ec.Int32(0), }, }, - { - ID: "warm", - ZoneCount: 2, - InstanceConfigurationID: "aws.data.highstorage.d3", - Size: &models.TopologySize{ + }, + { + ID: "hot_content", + ZoneCount: 2, + InstanceConfigurationID: "aws.data.highio.i3", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(8192), + }, + NodeRoles: []string{ + "master", + "remote_cluster_client", + "data_hot", + "transform", + "data_content", + }, + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{"data": "hot"}, + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ Resource: ec.String("memory"), - Value: ec.Int32(4096), - }, - NodeRoles: []string{ - "data_warm", - "remote_cluster_client", + Value: ec.Int32(1024), }, - Elasticsearch: &models.ElasticsearchConfiguration{ - NodeAttributes: map[string]string{ - "data": "warm", - }, + }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, + }, + { + ID: "warm", + ZoneCount: 2, + InstanceConfigurationID: "aws.data.highstorage.d3", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(4096), + }, + NodeRoles: []string{ + "data_warm", + "remote_cluster_client", + }, + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{ + "data": "warm", }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(0), - }, + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(0), }, }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, }, }, }, - }, + }), }, }, }, @@ -1769,135 +2132,146 @@ func Test_createResourceToModel(t *testing.T) { Tags: []*models.MetadataItem{}, }, Resources: &models.DeploymentCreateResources{ - Elasticsearch: []*models.ElasticsearchPayload{ - { - Region: ec.String("us-east-1"), - RefID: ec.String("main-elasticsearch"), - Settings: &models.ElasticsearchClusterSettings{ - DedicatedMastersThreshold: 6, + Elasticsearch: enrichWithEmptyTopologies(readerToESPayload(t, ioOptimizedTpl(), true), &models.ElasticsearchPayload{ + Region: ec.String("us-east-1"), + RefID: ec.String("main-elasticsearch"), + Settings: &models.ElasticsearchClusterSettings{ + DedicatedMastersThreshold: 6, + }, + Plan: &models.ElasticsearchClusterPlan{ + AutoscalingEnabled: ec.Bool(false), + Elasticsearch: &models.ElasticsearchConfiguration{ + Version: "7.12.0", }, - Plan: &models.ElasticsearchClusterPlan{ - Elasticsearch: &models.ElasticsearchConfiguration{ - Version: "7.12.0", - }, - DeploymentTemplate: &models.DeploymentTemplateReference{ - ID: ec.String("aws-io-optimized-v2"), - }, - ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ - { - ID: "cold", - ZoneCount: 1, - InstanceConfigurationID: "aws.data.highstorage.d3", - Size: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(2048), - }, - NodeRoles: []string{ - "data_cold", - "remote_cluster_client", - }, - Elasticsearch: &models.ElasticsearchConfiguration{ - NodeAttributes: map[string]string{ - "data": "cold", - }, - }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(0), - }, + DeploymentTemplate: &models.DeploymentTemplateReference{ + ID: ec.String("aws-io-optimized-v2"), + }, + ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ + { + ID: "cold", + ZoneCount: 1, + InstanceConfigurationID: "aws.data.highstorage.d3", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(2048), + }, + NodeRoles: []string{ + "data_cold", + "remote_cluster_client", + }, + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{ + "data": "cold", }, }, - { - ID: "coordinating", - ZoneCount: 2, - InstanceConfigurationID: "aws.coordinating.m5d", - Size: &models.TopologySize{ + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ Resource: ec.String("memory"), - Value: ec.Int32(2048), - }, - NodeRoles: []string{ - "ingest", - "remote_cluster_client", - }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(0), - }, + Value: ec.Int32(0), }, }, - { - ID: "hot_content", - ZoneCount: 2, - InstanceConfigurationID: "aws.data.highio.i3", - Size: &models.TopologySize{ + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(59392), + Resource: ec.String("memory"), + }, + }, + { + ID: "coordinating", + ZoneCount: 2, + InstanceConfigurationID: "aws.coordinating.m5d", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(2048), + }, + NodeRoles: []string{ + "ingest", + "remote_cluster_client", + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ Resource: ec.String("memory"), - Value: ec.Int32(8192), - }, - NodeRoles: []string{ - "remote_cluster_client", - "data_hot", - "transform", - "data_content", + Value: ec.Int32(0), }, - Elasticsearch: &models.ElasticsearchConfiguration{ - NodeAttributes: map[string]string{"data": "hot"}, + }, + }, + { + ID: "hot_content", + ZoneCount: 2, + InstanceConfigurationID: "aws.data.highio.i3", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(8192), + }, + NodeRoles: []string{ + "remote_cluster_client", + "data_hot", + "transform", + "data_content", + }, + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{"data": "hot"}, + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(1024), }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(1024), - }, + }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, + }, + { + ID: "master", + ZoneCount: 3, + InstanceConfigurationID: "aws.master.r5d", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(1024), + }, + NodeRoles: []string{ + "master", + "remote_cluster_client", + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(0), }, }, - { - ID: "master", - ZoneCount: 3, - InstanceConfigurationID: "aws.master.r5d", - Size: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(1024), - }, - NodeRoles: []string{ - "master", - "remote_cluster_client", - }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(0), - }, + }, + { + ID: "warm", + ZoneCount: 2, + InstanceConfigurationID: "aws.data.highstorage.d3", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(4096), + }, + NodeRoles: []string{ + "data_warm", + "remote_cluster_client", + }, + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{ + "data": "warm", }, }, - { - ID: "warm", - ZoneCount: 2, - InstanceConfigurationID: "aws.data.highstorage.d3", - Size: &models.TopologySize{ + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ Resource: ec.String("memory"), - Value: ec.Int32(4096), - }, - NodeRoles: []string{ - "data_warm", - "remote_cluster_client", - }, - Elasticsearch: &models.ElasticsearchConfiguration{ - NodeAttributes: map[string]string{ - "data": "warm", - }, - }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(0), - }, + Value: ec.Int32(0), }, }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, }, }, }, - }, + }), }, }, }, @@ -1914,43 +2288,41 @@ func Test_createResourceToModel(t *testing.T) { Tags: []*models.MetadataItem{}, }, Resources: &models.DeploymentCreateResources{ - Elasticsearch: []*models.ElasticsearchPayload{ - { - Region: ec.String("us-east-1"), - RefID: ec.String("main-elasticsearch"), - Settings: &models.ElasticsearchClusterSettings{}, - Plan: &models.ElasticsearchClusterPlan{ - Elasticsearch: &models.ElasticsearchConfiguration{ - Version: "7.9.2", - }, - DeploymentTemplate: &models.DeploymentTemplateReference{ - ID: ec.String("aws-cross-cluster-search-v2"), - }, - ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ - { - ID: "hot_content", - ZoneCount: 1, - InstanceConfigurationID: "aws.ccs.r5d", - Size: &models.TopologySize{ + Elasticsearch: enrichWithEmptyTopologies(readerToESPayload(t, ccsTpl(), false), &models.ElasticsearchPayload{ + Region: ec.String("us-east-1"), + RefID: ec.String("main-elasticsearch"), + Settings: &models.ElasticsearchClusterSettings{}, + Plan: &models.ElasticsearchClusterPlan{ + Elasticsearch: &models.ElasticsearchConfiguration{ + Version: "7.9.2", + }, + DeploymentTemplate: &models.DeploymentTemplateReference{ + ID: ec.String("aws-cross-cluster-search-v2"), + }, + ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ + { + ID: "hot_content", + ZoneCount: 1, + InstanceConfigurationID: "aws.ccs.r5d", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(1024), + }, + NodeType: &models.ElasticsearchNodeType{ + Data: ec.Bool(true), + Ingest: ec.Bool(true), + Master: ec.Bool(true), + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ Resource: ec.String("memory"), Value: ec.Int32(1024), }, - NodeType: &models.ElasticsearchNodeType{ - Data: ec.Bool(true), - Ingest: ec.Bool(true), - Master: ec.Bool(true), - }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(1024), - }, - }, }, }, }, }, - }, + }), Kibana: []*models.KibanaPayload{ { ElasticsearchClusterRefID: ec.String("main-elasticsearch"), @@ -1989,49 +2361,52 @@ func Test_createResourceToModel(t *testing.T) { {Key: ec.String("owner"), Value: ec.String("elastic")}, }}, Resources: &models.DeploymentCreateResources{ - Elasticsearch: []*models.ElasticsearchPayload{ - { - Region: ec.String("us-east-1"), - RefID: ec.String("main-elasticsearch"), - Settings: &models.ElasticsearchClusterSettings{ - DedicatedMastersThreshold: 6, + Elasticsearch: enrichWithEmptyTopologies(readerToESPayload(t, ioOptimizedTpl(), true), &models.ElasticsearchPayload{ + Region: ec.String("us-east-1"), + RefID: ec.String("main-elasticsearch"), + Settings: &models.ElasticsearchClusterSettings{ + DedicatedMastersThreshold: 6, + }, + Plan: &models.ElasticsearchClusterPlan{ + AutoscalingEnabled: ec.Bool(false), + Elasticsearch: &models.ElasticsearchConfiguration{ + Version: "7.10.1", + }, + DeploymentTemplate: &models.DeploymentTemplateReference{ + ID: ec.String("aws-io-optimized-v2"), }, - Plan: &models.ElasticsearchClusterPlan{ + ClusterTopology: []*models.ElasticsearchClusterTopologyElement{{ + ID: "hot_content", Elasticsearch: &models.ElasticsearchConfiguration{ - Version: "7.10.1", + NodeAttributes: map[string]string{"data": "hot"}, }, - DeploymentTemplate: &models.DeploymentTemplateReference{ - ID: ec.String("aws-io-optimized-v2"), + ZoneCount: 2, + InstanceConfigurationID: "aws.data.highio.i3", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(8192), }, - ClusterTopology: []*models.ElasticsearchClusterTopologyElement{{ - ID: "hot_content", - Elasticsearch: &models.ElasticsearchConfiguration{ - NodeAttributes: map[string]string{"data": "hot"}, - }, - ZoneCount: 2, - InstanceConfigurationID: "aws.data.highio.i3", - Size: &models.TopologySize{ + NodeRoles: []string{ + "master", + "ingest", + "remote_cluster_client", + "data_hot", + "transform", + "data_content", + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ Resource: ec.String("memory"), - Value: ec.Int32(8192), - }, - NodeRoles: []string{ - "master", - "ingest", - "remote_cluster_client", - "data_hot", - "transform", - "data_content", - }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(1024), - }, + Value: ec.Int32(1024), }, - }}, - }, + }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, + }}, }, - }, + }), }, }, }, @@ -2361,55 +2736,58 @@ func Test_updateResourceToModel(t *testing.T) { Tags: []*models.MetadataItem{}, }, Resources: &models.DeploymentUpdateResources{ - Elasticsearch: []*models.ElasticsearchPayload{ - { - Region: ec.String("us-east-1"), - RefID: ec.String("main-elasticsearch"), - Settings: &models.ElasticsearchClusterSettings{ - DedicatedMastersThreshold: 6, + Elasticsearch: enrichWithEmptyTopologies(readerToESPayload(t, ioOptimizedTpl(), false), &models.ElasticsearchPayload{ + Region: ec.String("us-east-1"), + RefID: ec.String("main-elasticsearch"), + Settings: &models.ElasticsearchClusterSettings{ + DedicatedMastersThreshold: 6, + }, + Plan: &models.ElasticsearchClusterPlan{ + AutoscalingEnabled: ec.Bool(false), + Elasticsearch: &models.ElasticsearchConfiguration{ + Version: "7.7.0", + UserSettingsYaml: `some.setting: value`, + UserSettingsOverrideYaml: `some.setting: value2`, + UserSettingsJSON: map[string]interface{}{ + "some.setting": "value", + }, + UserSettingsOverrideJSON: map[string]interface{}{ + "some.setting": "value2", + }, + }, + DeploymentTemplate: &models.DeploymentTemplateReference{ + ID: ec.String("aws-io-optimized-v2"), }, - Plan: &models.ElasticsearchClusterPlan{ + ClusterTopology: []*models.ElasticsearchClusterTopologyElement{{ + ID: "hot_content", Elasticsearch: &models.ElasticsearchConfiguration{ - Version: "7.7.0", - UserSettingsYaml: `some.setting: value`, - UserSettingsOverrideYaml: `some.setting: value2`, - UserSettingsJSON: map[string]interface{}{ - "some.setting": "value", - }, - UserSettingsOverrideJSON: map[string]interface{}{ - "some.setting": "value2", - }, + NodeAttributes: map[string]string{"data": "hot"}, }, - DeploymentTemplate: &models.DeploymentTemplateReference{ - ID: ec.String("aws-io-optimized-v2"), + ZoneCount: 1, + InstanceConfigurationID: "aws.data.highio.i3", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(2048), }, - ClusterTopology: []*models.ElasticsearchClusterTopologyElement{{ - ID: "hot_content", - Elasticsearch: &models.ElasticsearchConfiguration{ - NodeAttributes: map[string]string{"data": "hot"}, - }, - ZoneCount: 1, - InstanceConfigurationID: "aws.data.highio.i3", - Size: &models.TopologySize{ + NodeType: &models.ElasticsearchNodeType{ + Data: ec.Bool(true), + Ingest: ec.Bool(true), + Master: ec.Bool(true), + Ml: ec.Bool(false), + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ Resource: ec.String("memory"), - Value: ec.Int32(2048), - }, - NodeType: &models.ElasticsearchNodeType{ - Data: ec.Bool(true), - Ingest: ec.Bool(true), - Master: ec.Bool(true), - Ml: ec.Bool(false), - }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(1024), - }, + Value: ec.Int32(1024), }, - }}, - }, + }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, + }}, }, - }, + }), Kibana: []*models.KibanaPayload{ { ElasticsearchClusterRefID: ec.String("main-elasticsearch"), @@ -2494,46 +2872,49 @@ func Test_updateResourceToModel(t *testing.T) { Tags: []*models.MetadataItem{}, }, Resources: &models.DeploymentUpdateResources{ - Elasticsearch: []*models.ElasticsearchPayload{ - { - Region: ec.String("us-east-1"), - RefID: ec.String("main-elasticsearch"), - Settings: &models.ElasticsearchClusterSettings{ - DedicatedMastersThreshold: 6, + Elasticsearch: enrichWithEmptyTopologies(readerToESPayload(t, ioOptimizedTpl(), false), &models.ElasticsearchPayload{ + Region: ec.String("us-east-1"), + RefID: ec.String("main-elasticsearch"), + Settings: &models.ElasticsearchClusterSettings{ + DedicatedMastersThreshold: 6, + }, + Plan: &models.ElasticsearchClusterPlan{ + AutoscalingEnabled: ec.Bool(false), + Elasticsearch: &models.ElasticsearchConfiguration{ + Version: "7.7.0", + }, + DeploymentTemplate: &models.DeploymentTemplateReference{ + ID: ec.String("aws-io-optimized-v2"), }, - Plan: &models.ElasticsearchClusterPlan{ + ClusterTopology: []*models.ElasticsearchClusterTopologyElement{{ + ID: "hot_content", Elasticsearch: &models.ElasticsearchConfiguration{ - Version: "7.7.0", + NodeAttributes: map[string]string{"data": "hot"}, + }, + ZoneCount: 2, + InstanceConfigurationID: "aws.data.highio.i3", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(8192), }, - DeploymentTemplate: &models.DeploymentTemplateReference{ - ID: ec.String("aws-io-optimized-v2"), + NodeType: &models.ElasticsearchNodeType{ + Data: ec.Bool(true), + Ingest: ec.Bool(true), + Master: ec.Bool(true), }, - ClusterTopology: []*models.ElasticsearchClusterTopologyElement{{ - ID: "hot_content", - Elasticsearch: &models.ElasticsearchConfiguration{ - NodeAttributes: map[string]string{"data": "hot"}, - }, - ZoneCount: 2, - InstanceConfigurationID: "aws.data.highio.i3", - Size: &models.TopologySize{ + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ Resource: ec.String("memory"), - Value: ec.Int32(8192), - }, - NodeType: &models.ElasticsearchNodeType{ - Data: ec.Bool(true), - Ingest: ec.Bool(true), - Master: ec.Bool(true), - }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(1024), - }, + Value: ec.Int32(1024), }, - }}, - }, + }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, + }}, }, - }, + }), Kibana: []*models.KibanaPayload{ { ElasticsearchClusterRefID: ec.String("main-elasticsearch"), @@ -2614,46 +2995,49 @@ func Test_updateResourceToModel(t *testing.T) { Tags: []*models.MetadataItem{}, }, Resources: &models.DeploymentUpdateResources{ - Elasticsearch: []*models.ElasticsearchPayload{ - { - Region: ec.String("us-east-1"), - RefID: ec.String("main-elasticsearch"), - Settings: &models.ElasticsearchClusterSettings{ - DedicatedMastersThreshold: 6, + Elasticsearch: enrichWithEmptyTopologies(readerToESPayload(t, ioOptimizedTpl(), false), &models.ElasticsearchPayload{ + Region: ec.String("us-east-1"), + RefID: ec.String("main-elasticsearch"), + Settings: &models.ElasticsearchClusterSettings{ + DedicatedMastersThreshold: 6, + }, + Plan: &models.ElasticsearchClusterPlan{ + AutoscalingEnabled: ec.Bool(false), + Elasticsearch: &models.ElasticsearchConfiguration{ + Version: "7.7.0", + }, + DeploymentTemplate: &models.DeploymentTemplateReference{ + ID: ec.String("aws-io-optimized-v2"), }, - Plan: &models.ElasticsearchClusterPlan{ + ClusterTopology: []*models.ElasticsearchClusterTopologyElement{{ + ID: "hot_content", Elasticsearch: &models.ElasticsearchConfiguration{ - Version: "7.7.0", + NodeAttributes: map[string]string{"data": "hot"}, }, - DeploymentTemplate: &models.DeploymentTemplateReference{ - ID: ec.String("aws-io-optimized-v2"), + ZoneCount: 2, + InstanceConfigurationID: "aws.data.highio.i3", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(4096), }, - ClusterTopology: []*models.ElasticsearchClusterTopologyElement{{ - ID: "hot_content", - Elasticsearch: &models.ElasticsearchConfiguration{ - NodeAttributes: map[string]string{"data": "hot"}, - }, - ZoneCount: 2, - InstanceConfigurationID: "aws.data.highio.i3", - Size: &models.TopologySize{ + NodeType: &models.ElasticsearchNodeType{ + Data: ec.Bool(true), + Ingest: ec.Bool(true), + Master: ec.Bool(true), + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ Resource: ec.String("memory"), - Value: ec.Int32(4096), - }, - NodeType: &models.ElasticsearchNodeType{ - Data: ec.Bool(true), - Ingest: ec.Bool(true), - Master: ec.Bool(true), - }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(1024), - }, + Value: ec.Int32(1024), }, - }}, - }, + }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, + }}, }, - }, + }), Kibana: []*models.KibanaPayload{ { ElasticsearchClusterRefID: ec.String("main-elasticsearch"), @@ -2734,73 +3118,80 @@ func Test_updateResourceToModel(t *testing.T) { Tags: []*models.MetadataItem{}, }, Resources: &models.DeploymentUpdateResources{ - Elasticsearch: []*models.ElasticsearchPayload{ - { - Region: ec.String("us-east-1"), - RefID: ec.String("main-elasticsearch"), - Settings: &models.ElasticsearchClusterSettings{ - DedicatedMastersThreshold: 6, - Curation: nil, + Elasticsearch: enrichWithEmptyTopologies(readerToESPayload(t, hotWarmTpl(), false), &models.ElasticsearchPayload{ + Region: ec.String("us-east-1"), + RefID: ec.String("main-elasticsearch"), + Settings: &models.ElasticsearchClusterSettings{ + DedicatedMastersThreshold: 6, + Curation: nil, + }, + Plan: &models.ElasticsearchClusterPlan{ + AutoscalingEnabled: ec.Bool(false), + Elasticsearch: &models.ElasticsearchConfiguration{ + Version: "7.9.2", + Curation: nil, }, - Plan: &models.ElasticsearchClusterPlan{ - Elasticsearch: &models.ElasticsearchConfiguration{ - Version: "7.9.2", - Curation: nil, - }, - DeploymentTemplate: &models.DeploymentTemplateReference{ - ID: ec.String("aws-hot-warm-v2"), - }, - ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ - { - ID: "hot_content", - ZoneCount: 2, - InstanceConfigurationID: "aws.data.highio.i3", - Size: &models.TopologySize{ + DeploymentTemplate: &models.DeploymentTemplateReference{ + ID: ec.String("aws-hot-warm-v2"), + }, + ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ + { + ID: "hot_content", + ZoneCount: 2, + InstanceConfigurationID: "aws.data.highio.i3", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(4096), + }, + NodeType: &models.ElasticsearchNodeType{ + Data: ec.Bool(true), + Ingest: ec.Bool(true), + Master: ec.Bool(true), + }, + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{"data": "hot"}, + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ Resource: ec.String("memory"), - Value: ec.Int32(4096), - }, - NodeType: &models.ElasticsearchNodeType{ - Data: ec.Bool(true), - Ingest: ec.Bool(true), - Master: ec.Bool(true), - }, - Elasticsearch: &models.ElasticsearchConfiguration{ - NodeAttributes: map[string]string{"data": "hot"}, - }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(1024), - }, + Value: ec.Int32(1024), }, }, - { - ID: "warm", - ZoneCount: 2, - InstanceConfigurationID: "aws.data.highstorage.d2", - Size: &models.TopologySize{ + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, + }, + { + ID: "warm", + ZoneCount: 2, + InstanceConfigurationID: "aws.data.highstorage.d2", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(4096), + }, + NodeType: &models.ElasticsearchNodeType{ + Data: ec.Bool(true), + Ingest: ec.Bool(true), + Master: ec.Bool(false), + }, + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{"data": "warm"}, + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ Resource: ec.String("memory"), - Value: ec.Int32(4096), - }, - NodeType: &models.ElasticsearchNodeType{ - Data: ec.Bool(true), - Ingest: ec.Bool(true), - Master: ec.Bool(false), - }, - Elasticsearch: &models.ElasticsearchConfiguration{ - NodeAttributes: map[string]string{"data": "warm"}, - }, - TopologyElementControl: &models.TopologyElementControl{ - Min: &models.TopologySize{ - Resource: ec.String("memory"), - Value: ec.Int32(0), - }, + Value: ec.Int32(0), }, }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, }, }, }, - }, + }), Kibana: []*models.KibanaPayload{ { ElasticsearchClusterRefID: ec.String("main-elasticsearch"), @@ -2840,7 +3231,7 @@ func Test_updateResourceToModel(t *testing.T) { Tags: []*models.MetadataItem{}, }, Resources: &models.DeploymentUpdateResources{ - Elasticsearch: []*models.ElasticsearchPayload{{ + Elasticsearch: enrichWithEmptyTopologies(readerToESPayload(t, ccsTpl(), false), &models.ElasticsearchPayload{ Region: ec.String("us-east-1"), RefID: ec.String("main-elasticsearch"), Settings: &models.ElasticsearchClusterSettings{}, @@ -2872,7 +3263,7 @@ func Test_updateResourceToModel(t *testing.T) { }, }}, }, - }}, + }), Kibana: []*models.KibanaPayload{{ ElasticsearchClusterRefID: ec.String("main-elasticsearch"), Region: ec.String("us-east-1"), @@ -2912,7 +3303,7 @@ func Test_updateResourceToModel(t *testing.T) { Tags: []*models.MetadataItem{}, }, Resources: &models.DeploymentUpdateResources{ - Elasticsearch: []*models.ElasticsearchPayload{{ + Elasticsearch: enrichWithEmptyTopologies(readerToESPayload(t, ccsTpl(), false), &models.ElasticsearchPayload{ Region: ec.String("us-east-1"), RefID: ec.String("main-elasticsearch"), Settings: &models.ElasticsearchClusterSettings{}, @@ -2945,7 +3336,7 @@ func Test_updateResourceToModel(t *testing.T) { }, }}, }, - }}, + }), Kibana: []*models.KibanaPayload{{ ElasticsearchClusterRefID: ec.String("main-elasticsearch"), Region: ec.String("us-east-1"), @@ -2983,13 +3374,14 @@ func Test_updateResourceToModel(t *testing.T) { Tags: []*models.MetadataItem{}, }, Resources: &models.DeploymentUpdateResources{ - Elasticsearch: []*models.ElasticsearchPayload{{ + Elasticsearch: enrichWithEmptyTopologies(readerToESPayload(t, ioOptimizedTpl(), false), &models.ElasticsearchPayload{ Region: ec.String("us-east-1"), RefID: ec.String("main-elasticsearch"), Settings: &models.ElasticsearchClusterSettings{ DedicatedMastersThreshold: 6, }, Plan: &models.ElasticsearchClusterPlan{ + AutoscalingEnabled: ec.Bool(false), Elasticsearch: &models.ElasticsearchConfiguration{ Version: "7.9.2", }, @@ -3018,9 +3410,14 @@ func Test_updateResourceToModel(t *testing.T) { Value: ec.Int32(1024), }, }, - }}, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, + }, + }, }, - }}, + }), Kibana: []*models.KibanaPayload{{ ElasticsearchClusterRefID: ec.String("main-elasticsearch"), Region: ec.String("us-east-1"), @@ -3130,13 +3527,14 @@ func Test_updateResourceToModel(t *testing.T) { Tags: []*models.MetadataItem{}, }, Resources: &models.DeploymentUpdateResources{ - Elasticsearch: []*models.ElasticsearchPayload{{ + Elasticsearch: enrichWithEmptyTopologies(readerToESPayload(t, ioOptimizedTpl(), false), &models.ElasticsearchPayload{ Region: ec.String("us-east-1"), RefID: ec.String("main-elasticsearch"), Settings: &models.ElasticsearchClusterSettings{ DedicatedMastersThreshold: 6, }, Plan: &models.ElasticsearchClusterPlan{ + AutoscalingEnabled: ec.Bool(false), Elasticsearch: &models.ElasticsearchConfiguration{ Version: "7.11.1", }, @@ -3166,9 +3564,13 @@ func Test_updateResourceToModel(t *testing.T) { Value: ec.Int32(1024), }, }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, }}, }, - }}, + }), }, }, }, @@ -3221,13 +3623,14 @@ func Test_updateResourceToModel(t *testing.T) { Tags: []*models.MetadataItem{}, }, Resources: &models.DeploymentUpdateResources{ - Elasticsearch: []*models.ElasticsearchPayload{{ + Elasticsearch: enrichWithEmptyTopologies(readerToESPayload(t, ioOptimizedTpl(), true), &models.ElasticsearchPayload{ Region: ec.String("us-east-1"), RefID: ec.String("main-elasticsearch"), Settings: &models.ElasticsearchClusterSettings{ DedicatedMastersThreshold: 6, }, Plan: &models.ElasticsearchClusterPlan{ + AutoscalingEnabled: ec.Bool(false), Elasticsearch: &models.ElasticsearchConfiguration{ Version: "7.10.1", }, @@ -3259,14 +3662,18 @@ func Test_updateResourceToModel(t *testing.T) { Value: ec.Int32(1024), }, }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, }}, }, - }}, + }), }, }, }, { - name: "migrates node_type to node_role when the existing topology element size is updated", + name: "migrates node_type to node_role when the existing topology element size is updated and adds warm tier", args: args{ d: util.NewResourceData(t, util.ResDataParams{ ID: mock.ValidClusterID, @@ -3320,13 +3727,14 @@ func Test_updateResourceToModel(t *testing.T) { Tags: []*models.MetadataItem{}, }, Resources: &models.DeploymentUpdateResources{ - Elasticsearch: []*models.ElasticsearchPayload{{ + Elasticsearch: enrichWithEmptyTopologies(readerToESPayload(t, ioOptimizedTpl(), true), &models.ElasticsearchPayload{ Region: ec.String("us-east-1"), RefID: ec.String("main-elasticsearch"), Settings: &models.ElasticsearchClusterSettings{ DedicatedMastersThreshold: 6, }, Plan: &models.ElasticsearchClusterPlan{ + AutoscalingEnabled: ec.Bool(false), Elasticsearch: &models.ElasticsearchConfiguration{ Version: "7.10.1", }, @@ -3359,6 +3767,10 @@ func Test_updateResourceToModel(t *testing.T) { Value: ec.Int32(1024), }, }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, }, { ID: "warm", @@ -3381,44 +3793,81 @@ func Test_updateResourceToModel(t *testing.T) { Value: ec.Int32(0), }, }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, }, }, }, - }}, + }), }, }, }, { - name: "parses the resources with tags", + name: "enables autoscaling with the default policies", args: args{ - d: deploymentWithTags, + d: util.NewResourceData(t, util.ResDataParams{ + ID: mock.ValidClusterID, + State: map[string]interface{}{ + "name": "my_deployment_name", + "deployment_template_id": "aws-io-optimized-v2", + "region": "us-east-1", + "version": "7.12.1", + "elasticsearch": []interface{}{map[string]interface{}{ + "topology": []interface{}{map[string]interface{}{ + "id": "hot_content", + "size": "16g", + }}, + }}, + }, + Change: map[string]interface{}{ + "name": "my_deployment_name", + "deployment_template_id": "aws-io-optimized-v2", + "region": "us-east-1", + "version": "7.12.1", + "elasticsearch": []interface{}{map[string]interface{}{ + "autoscale": "true", + "topology": []interface{}{ + map[string]interface{}{ + "id": "hot_content", + "size": "16g", + }, + map[string]interface{}{ + "id": "warm", + "size": "8g", + }, + }, + }}, + }, + Schema: newSchema(), + }), client: api.NewMock(mock.New200Response(ioOptimizedTpl())), }, want: &models.DeploymentUpdateRequest{ Name: "my_deployment_name", PruneOrphans: ec.Bool(true), Settings: &models.DeploymentUpdateSettings{}, - Metadata: &models.DeploymentUpdateMetadata{Tags: []*models.MetadataItem{ - {Key: ec.String("aaa"), Value: ec.String("bbb")}, - {Key: ec.String("cost-center"), Value: ec.String("rnd")}, - {Key: ec.String("owner"), Value: ec.String("elastic")}, - }}, + Metadata: &models.DeploymentUpdateMetadata{ + Tags: []*models.MetadataItem{}, + }, Resources: &models.DeploymentUpdateResources{ - Elasticsearch: []*models.ElasticsearchPayload{ - { - Region: ec.String("us-east-1"), - RefID: ec.String("main-elasticsearch"), - Settings: &models.ElasticsearchClusterSettings{ - DedicatedMastersThreshold: 6, + Elasticsearch: enrichWithEmptyTopologies(readerToESPayload(t, ioOptimizedTpl(), true), &models.ElasticsearchPayload{ + Region: ec.String("us-east-1"), + RefID: ec.String("main-elasticsearch"), + Settings: &models.ElasticsearchClusterSettings{ + DedicatedMastersThreshold: 6, + }, + Plan: &models.ElasticsearchClusterPlan{ + AutoscalingEnabled: ec.Bool(true), + Elasticsearch: &models.ElasticsearchConfiguration{ + Version: "7.12.1", }, - Plan: &models.ElasticsearchClusterPlan{ - Elasticsearch: &models.ElasticsearchConfiguration{ - Version: "7.10.1", - }, - DeploymentTemplate: &models.DeploymentTemplateReference{ - ID: ec.String("aws-io-optimized-v2"), - }, - ClusterTopology: []*models.ElasticsearchClusterTopologyElement{{ + DeploymentTemplate: &models.DeploymentTemplateReference{ + ID: ec.String("aws-io-optimized-v2"), + }, + ClusterTopology: []*models.ElasticsearchClusterTopologyElement{ + { ID: "hot_content", Elasticsearch: &models.ElasticsearchConfiguration{ NodeAttributes: map[string]string{"data": "hot"}, @@ -3427,7 +3876,7 @@ func Test_updateResourceToModel(t *testing.T) { InstanceConfigurationID: "aws.data.highio.i3", Size: &models.TopologySize{ Resource: ec.String("memory"), - Value: ec.Int32(8192), + Value: ec.Int32(16384), }, NodeRoles: []string{ "master", @@ -3443,10 +3892,105 @@ func Test_updateResourceToModel(t *testing.T) { Value: ec.Int32(1024), }, }, - }}, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, + }, + { + ID: "warm", + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{"data": "warm"}, + }, + ZoneCount: 2, + InstanceConfigurationID: "aws.data.highstorage.d3", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(8192), + }, + NodeRoles: []string{ + "data_warm", + "remote_cluster_client", + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(0), + }, + }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, + }, }, }, - }, + }), + }, + }, + }, + { + name: "parses the resources with tags", + args: args{ + d: deploymentWithTags, + client: api.NewMock(mock.New200Response(ioOptimizedTpl())), + }, + want: &models.DeploymentUpdateRequest{ + Name: "my_deployment_name", + PruneOrphans: ec.Bool(true), + Settings: &models.DeploymentUpdateSettings{}, + Metadata: &models.DeploymentUpdateMetadata{Tags: []*models.MetadataItem{ + {Key: ec.String("aaa"), Value: ec.String("bbb")}, + {Key: ec.String("cost-center"), Value: ec.String("rnd")}, + {Key: ec.String("owner"), Value: ec.String("elastic")}, + }}, + Resources: &models.DeploymentUpdateResources{ + Elasticsearch: enrichWithEmptyTopologies(readerToESPayload(t, ioOptimizedTpl(), true), &models.ElasticsearchPayload{ + Region: ec.String("us-east-1"), + RefID: ec.String("main-elasticsearch"), + Settings: &models.ElasticsearchClusterSettings{ + DedicatedMastersThreshold: 6, + }, + Plan: &models.ElasticsearchClusterPlan{ + AutoscalingEnabled: ec.Bool(false), + Elasticsearch: &models.ElasticsearchConfiguration{ + Version: "7.10.1", + }, + DeploymentTemplate: &models.DeploymentTemplateReference{ + ID: ec.String("aws-io-optimized-v2"), + }, + ClusterTopology: []*models.ElasticsearchClusterTopologyElement{{ + ID: "hot_content", + Elasticsearch: &models.ElasticsearchConfiguration{ + NodeAttributes: map[string]string{"data": "hot"}, + }, + ZoneCount: 2, + InstanceConfigurationID: "aws.data.highio.i3", + Size: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(8192), + }, + NodeRoles: []string{ + "master", + "ingest", + "remote_cluster_client", + "data_hot", + "transform", + "data_content", + }, + TopologyElementControl: &models.TopologyElementControl{ + Min: &models.TopologySize{ + Resource: ec.String("memory"), + Value: ec.Int32(1024), + }, + }, + AutoscalingMax: &models.TopologySize{ + Value: ec.Int32(118784), + Resource: ec.String("memory"), + }, + }}, + }, + }), }, }, }, diff --git a/ec/ecresource/deploymentresource/testutil_func.go b/ec/ecresource/deploymentresource/testutil_func.go index fc9368f1c..801ae70cb 100644 --- a/ec/ecresource/deploymentresource/testutil_func.go +++ b/ec/ecresource/deploymentresource/testutil_func.go @@ -19,6 +19,7 @@ package deploymentresource import ( "encoding/json" + "io" "os" "testing" @@ -64,3 +65,40 @@ func openDeploymentGet(t *testing.T, name string) *models.DeploymentGetResponse } return &res } + +func enrichWithEmptyTopologies(tpl, want *models.ElasticsearchPayload) []*models.ElasticsearchPayload { + tpl.DisplayName = want.DisplayName + tpl.RefID = want.RefID + tpl.Region = want.Region + tpl.Settings = want.Settings + tpl.Plan.AutoscalingEnabled = want.Plan.AutoscalingEnabled + tpl.Plan.Elasticsearch = want.Plan.Elasticsearch + tpl.Plan.ZoneCount = want.Plan.ZoneCount + tpl.Plan.Transient = want.Plan.Transient + + for i, t := range tpl.Plan.ClusterTopology { + for _, w := range want.Plan.ClusterTopology { + if t.ID == w.ID { + tpl.Plan.ClusterTopology[i] = w + } + } + } + + return []*models.ElasticsearchPayload{tpl} +} + +func readerToESPayload(t *testing.T, rc io.Reader, nr bool) *models.ElasticsearchPayload { + t.Helper() + + var tpl models.DeploymentTemplateInfoV2 + if err := json.NewDecoder(rc).Decode(&tpl); err != nil { + t.Fatal(err) + } + + return enrichElasticsearchTemplate( + tpl.DeploymentTemplate.Resources.Elasticsearch[0], + *tpl.ID, + "", + nr, + ) +} From 5b458547c9523d7c916633b731f39799c3947aa8 Mon Sep 17 00:00:00 2001 From: Marc Lopez Date: Wed, 14 Apr 2021 15:50:45 +0500 Subject: [PATCH 11/12] Add acceptance tests Signed-off-by: Marc Lopez --- ec/acc/deployment_autoscaling_test.go | 112 ++++++++++++++++++ ...deployment_pre_node_role_migration_test.go | 8 +- ec/acc/testdata/deployment_autoscaling_1.tf | 41 +++++++ ec/acc/testdata/deployment_autoscaling_2.tf | 41 +++++++ .../deployment_pre_node_roles_migration_3.tf | 1 + .../deployment_pre_node_roles_migration_4.tf | 24 ---- .../schema_elasticsearch.go | 1 - 7 files changed, 196 insertions(+), 32 deletions(-) create mode 100644 ec/acc/deployment_autoscaling_test.go create mode 100644 ec/acc/testdata/deployment_autoscaling_1.tf create mode 100644 ec/acc/testdata/deployment_autoscaling_2.tf delete mode 100644 ec/acc/testdata/deployment_pre_node_roles_migration_4.tf diff --git a/ec/acc/deployment_autoscaling_test.go b/ec/acc/deployment_autoscaling_test.go new file mode 100644 index 000000000..9f4016597 --- /dev/null +++ b/ec/acc/deployment_autoscaling_test.go @@ -0,0 +1,112 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package acc + +import ( + "testing" + + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/acctest" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" +) + +func TestAccDeployment_autoscaling(t *testing.T) { + resName := "ec_deployment.autoscaling" + randomName := prefix + acctest.RandStringFromCharSet(10, acctest.CharSetAlphaNum) + startCfg := "testdata/deployment_autoscaling_1.tf" + disableAutoscale := "testdata/deployment_autoscaling_2.tf" + + cfgF := func(cfg string) string { + return fixtureAccDeploymentResourceBasic( + t, cfg, randomName, getRegion(), defaultTemplate, + ) + } + + resource.ParallelTest(t, resource.TestCase{ + PreCheck: func() { testAccPreCheck(t) }, + ProviderFactories: testAccProviderFactory, + CheckDestroy: testAccDeploymentDestroy, + Steps: []resource.TestStep{ + { + Config: cfgF(startCfg), + Check: resource.ComposeAggregateTestCheckFunc( + resource.TestCheckResourceAttr(resName, "elasticsearch.#", "1"), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.autoscale", "true"), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.#", "3"), + resource.TestCheckResourceAttrSet(resName, "elasticsearch.0.topology.0.instance_configuration_id"), + + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.0.id", "hot_content"), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.0.size", "1g"), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.0.size_resource", "memory"), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.0.zone_count", "1"), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.0.autoscaling.#", "1"), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.0.autoscaling.0.max_size", "8g"), + + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.1.id", "ml"), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.1.size", "1g"), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.1.size_resource", "memory"), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.1.zone_count", "1"), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.1.autoscaling.#", "1"), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.1.autoscaling.0.max_size", "4g"), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.1.autoscaling.0.min_size", "1g"), + + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.2.id", "warm"), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.2.size", "2g"), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.2.size_resource", "memory"), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.2.zone_count", "1"), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.2.autoscaling.#", "1"), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.2.autoscaling.0.max_size", "15g"), + + resource.TestCheckResourceAttr(resName, "kibana.#", "0"), + resource.TestCheckResourceAttr(resName, "apm.#", "0"), + resource.TestCheckResourceAttr(resName, "enterprise_search.#", "0"), + ), + }, + // also disables ML + { + Config: cfgF(disableAutoscale), + // When disabling a tier the plan will be non empty on refresh + // since the topology block is present with size = "0g". + ExpectNonEmptyPlan: true, + Check: resource.ComposeAggregateTestCheckFunc( + resource.TestCheckResourceAttr(resName, "elasticsearch.#", "1"), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.autoscale", "false"), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.#", "2"), + resource.TestCheckResourceAttrSet(resName, "elasticsearch.0.topology.0.instance_configuration_id"), + + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.0.id", "hot_content"), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.0.size", "1g"), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.0.size_resource", "memory"), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.0.zone_count", "1"), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.0.autoscaling.#", "1"), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.0.autoscaling.0.max_size", "8g"), + + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.1.id", "warm"), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.1.size", "2g"), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.1.size_resource", "memory"), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.1.zone_count", "1"), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.1.autoscaling.#", "1"), + resource.TestCheckResourceAttr(resName, "elasticsearch.0.topology.1.autoscaling.0.max_size", "15g"), + + resource.TestCheckResourceAttr(resName, "kibana.#", "0"), + resource.TestCheckResourceAttr(resName, "apm.#", "0"), + resource.TestCheckResourceAttr(resName, "enterprise_search.#", "0"), + ), + }, + }, + }) +} diff --git a/ec/acc/deployment_pre_node_role_migration_test.go b/ec/acc/deployment_pre_node_role_migration_test.go index 270e88de7..f20cd68ee 100644 --- a/ec/acc/deployment_pre_node_role_migration_test.go +++ b/ec/acc/deployment_pre_node_role_migration_test.go @@ -18,7 +18,6 @@ package acc import ( - "regexp" "testing" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/acctest" @@ -30,8 +29,7 @@ func TestAccDeployment_pre_node_roles(t *testing.T) { randomName := prefix + acctest.RandStringFromCharSet(10, acctest.CharSetAlphaNum) startCfg := "testdata/deployment_pre_node_roles_migration_1.tf" upgradeVersionCfg := "testdata/deployment_pre_node_roles_migration_2.tf" - invalidWarmTopologyCfg := "testdata/deployment_pre_node_roles_migration_3.tf" - addWarmTopologyCfg := "testdata/deployment_pre_node_roles_migration_4.tf" + addWarmTopologyCfg := "testdata/deployment_pre_node_roles_migration_3.tf" cfgF := func(cfg string) string { return fixtureAccDeploymentResourceBasic( @@ -84,10 +82,6 @@ func TestAccDeployment_pre_node_roles(t *testing.T) { resource.TestCheckResourceAttr(resName, "enterprise_search.#", "0"), ), }, - { - Config: cfgF(invalidWarmTopologyCfg), - ExpectError: regexp.MustCompile(`(?m)invalid configuration: 1 error occurred:\n\t\* elasticsearch topology warm: size cannot be zero\n\n`), - }, { Config: cfgF(addWarmTopologyCfg), Check: resource.ComposeAggregateTestCheckFunc( diff --git a/ec/acc/testdata/deployment_autoscaling_1.tf b/ec/acc/testdata/deployment_autoscaling_1.tf new file mode 100644 index 000000000..d0d1bb5c2 --- /dev/null +++ b/ec/acc/testdata/deployment_autoscaling_1.tf @@ -0,0 +1,41 @@ +data "ec_stack" "autoscaling" { + version_regex = "latest" + region = "%s" +} + +resource "ec_deployment" "autoscaling" { + name = "%s" + region = "%s" + version = data.ec_stack.autoscaling.version + deployment_template_id = "%s" + + elasticsearch { + autoscale = "true" + + topology { + id = "hot_content" + size = "1g" + zone_count = 1 + autoscaling { + max_size = "8g" + } + } + topology { + id = "ml" + size = "1g" + zone_count = 1 + autoscaling { + min_size = "1g" + max_size = "4g" + } + } + topology { + id = "warm" + size = "2g" + zone_count = 1 + autoscaling { + max_size = "15g" + } + } + } +} \ No newline at end of file diff --git a/ec/acc/testdata/deployment_autoscaling_2.tf b/ec/acc/testdata/deployment_autoscaling_2.tf new file mode 100644 index 000000000..dbd7c00b9 --- /dev/null +++ b/ec/acc/testdata/deployment_autoscaling_2.tf @@ -0,0 +1,41 @@ +data "ec_stack" "autoscaling" { + version_regex = "latest" + region = "%s" +} + +resource "ec_deployment" "autoscaling" { + name = "%s" + region = "%s" + version = data.ec_stack.autoscaling.version + deployment_template_id = "%s" + + elasticsearch { + autoscale = "false" + + topology { + id = "hot_content" + size = "1g" + zone_count = 1 + autoscaling { + max_size = "8g" + } + } + topology { + id = "ml" + size = "0g" + zone_count = 1 + autoscaling { + min_size = "0g" + max_size = "4g" + } + } + topology { + id = "warm" + size = "2g" + zone_count = 1 + autoscaling { + max_size = "15g" + } + } + } +} \ No newline at end of file diff --git a/ec/acc/testdata/deployment_pre_node_roles_migration_3.tf b/ec/acc/testdata/deployment_pre_node_roles_migration_3.tf index 3397367bb..6110edbd1 100644 --- a/ec/acc/testdata/deployment_pre_node_roles_migration_3.tf +++ b/ec/acc/testdata/deployment_pre_node_roles_migration_3.tf @@ -17,6 +17,7 @@ resource "ec_deployment" "pre_nr" { } topology { id = "warm" + size = "2g" zone_count = 1 } } diff --git a/ec/acc/testdata/deployment_pre_node_roles_migration_4.tf b/ec/acc/testdata/deployment_pre_node_roles_migration_4.tf deleted file mode 100644 index 6110edbd1..000000000 --- a/ec/acc/testdata/deployment_pre_node_roles_migration_4.tf +++ /dev/null @@ -1,24 +0,0 @@ -data "ec_stack" "pre_node_roles" { - version_regex = "7.11.?" - region = "%s" -} - -resource "ec_deployment" "pre_nr" { - name = "%s" - region = "%s" - version = data.ec_stack.pre_node_roles.version - deployment_template_id = "%s" - - elasticsearch { - topology { - id = "hot_content" - size = "1g" - zone_count = 1 - } - topology { - id = "warm" - size = "2g" - zone_count = 1 - } - } -} \ No newline at end of file diff --git a/ec/ecdatasource/deploymentdatasource/schema_elasticsearch.go b/ec/ecdatasource/deploymentdatasource/schema_elasticsearch.go index b5aa53ee1..d742c18ce 100644 --- a/ec/ecdatasource/deploymentdatasource/schema_elasticsearch.go +++ b/ec/ecdatasource/deploymentdatasource/schema_elasticsearch.go @@ -116,7 +116,6 @@ func elasticsearchTopologySchema() *schema.Schema { Type: schema.TypeList, Description: "Optional Elasticsearch autoscaling settings, such a maximum and minimum size and resources.", Computed: true, - MaxItems: 1, Elem: &schema.Resource{ Schema: map[string]*schema.Schema{ "max_size_resource": { From f58f3a6200314ec5c9f400380c40884e13488e1a Mon Sep 17 00:00:00 2001 From: Marc Lopez Date: Wed, 14 Apr 2021 15:56:14 +0500 Subject: [PATCH 12/12] Terraform fmt files Signed-off-by: Marc Lopez --- ec/acc/testdata/deployment_autoscaling_1.tf | 2 +- ec/acc/testdata/deployment_autoscaling_2.tf | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ec/acc/testdata/deployment_autoscaling_1.tf b/ec/acc/testdata/deployment_autoscaling_1.tf index d0d1bb5c2..48baebc41 100644 --- a/ec/acc/testdata/deployment_autoscaling_1.tf +++ b/ec/acc/testdata/deployment_autoscaling_1.tf @@ -11,7 +11,7 @@ resource "ec_deployment" "autoscaling" { elasticsearch { autoscale = "true" - + topology { id = "hot_content" size = "1g" diff --git a/ec/acc/testdata/deployment_autoscaling_2.tf b/ec/acc/testdata/deployment_autoscaling_2.tf index dbd7c00b9..7dc2c000b 100644 --- a/ec/acc/testdata/deployment_autoscaling_2.tf +++ b/ec/acc/testdata/deployment_autoscaling_2.tf @@ -11,7 +11,7 @@ resource "ec_deployment" "autoscaling" { elasticsearch { autoscale = "false" - + topology { id = "hot_content" size = "1g"