From 388299df1025fede7c3a99767e8b255b1d18d630 Mon Sep 17 00:00:00 2001 From: VenelinMartinov Date: Tue, 11 Jun 2024 10:58:56 +0100 Subject: [PATCH] Test collections refresh (#2072) Related to https://github.com/pulumi/pulumi-terraform-bridge/issues/2047 Pulling out the tests from https://github.com/pulumi/pulumi-terraform-bridge/pull/2065 so that the changes in behaviour in the other PR are more visible and easy to review. This adds tests for refreshes on collections for: - list/set/map - RPC/non-PRC - null/empty/non-empty collections - nil/non-nil cloud values for the nil collections - top-level collections/nested collections - Properties with a value override in Create/ ones without - this is a behaviour observed in AWS and GCP labels and some other providers, mainly seen around maps. The goal of the tests is to: - Ensure we do not regress on behaviour in PRC vs non-PRC - Ensure we do not regress on behaviour in PRC as we change the implementation in https://github.com/pulumi/pulumi-terraform-bridge/pull/2073 / https://github.com/pulumi/pulumi-terraform-bridge/pull/2065 I've flattened the matrix in order to allow us to annotate/skip individual test cases. --- pkg/tests/internal/pulcheck/pulcheck.go | 23 +- pkg/tests/schema_pulumi_test.go | 680 ++++++++++++++++++++++++ 2 files changed, 701 insertions(+), 2 deletions(-) diff --git a/pkg/tests/internal/pulcheck/pulcheck.go b/pkg/tests/internal/pulcheck/pulcheck.go index 542cbe1f2..3ddf97454 100644 --- a/pkg/tests/internal/pulcheck/pulcheck.go +++ b/pkg/tests/internal/pulcheck/pulcheck.go @@ -104,13 +104,32 @@ type T interface { pulumitest.PT } +type bridgedProviderOpts struct { + DisablePlanResourceChange bool +} + +// BridgedProviderOpts +type BridgedProviderOpt func(*bridgedProviderOpts) + +// WithPlanResourceChange +func DisablePlanResourceChange() BridgedProviderOpt { + return func(o *bridgedProviderOpts) { + o.DisablePlanResourceChange = true + } +} + // This is an experimental API. -func BridgedProvider(t T, providerName string, resMap map[string]*schema.Resource) info.Provider { +func BridgedProvider(t T, providerName string, resMap map[string]*schema.Resource, opts ...BridgedProviderOpt) info.Provider { + options := &bridgedProviderOpts{} + for _, opt := range opts { + opt(options) + } + tfp := &schema.Provider{ResourcesMap: resMap} EnsureProviderValid(t, tfp) shimProvider := shimv2.NewProvider(tfp, shimv2.WithPlanResourceChange( - func(tfResourceType string) bool { return true }, + func(tfResourceType string) bool { return !options.DisablePlanResourceChange }, )) provider := tfbridge.ProviderInfo{ diff --git a/pkg/tests/schema_pulumi_test.go b/pkg/tests/schema_pulumi_test.go index 955a372ba..89c77ecf8 100644 --- a/pkg/tests/schema_pulumi_test.go +++ b/pkg/tests/schema_pulumi_test.go @@ -2,12 +2,14 @@ package tests import ( "context" + "fmt" "testing" "github.com/hashicorp/terraform-plugin-sdk/v2/diag" "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" "github.com/pulumi/pulumi-terraform-bridge/v3/pkg/tests/internal/pulcheck" "github.com/pulumi/pulumi/sdk/v3/go/auto/optpreview" + "github.com/pulumi/pulumi/sdk/v3/go/auto/optrefresh" "github.com/stretchr/testify/require" ) @@ -59,3 +61,681 @@ outputs: // assert that the property gets resolved require.Equal(t, "aux", resUp.Outputs["testOut"].Value) } + +func TestCollectionsNullEmptyRefreshClean(t *testing.T) { + for _, tc := range []struct { + name string + planResourceChange bool + schemaType schema.ValueType + cloudVal interface{} + programVal string + // If true, the cloud value will be set in the CreateContext + // This is behaviour observed in both AWS and GCP providers, as well as a few others + // where the provider returns an empty collections when a nil one was specified in inputs. + // See [pulumi/pulumi-terraform-bridge#2047] for more details around this behavior + createCloudValOverride bool + expectedOutputTopLevel interface{} + expectedOutputNested interface{} + expectFailTopLevel bool + expectFailNested bool + }{ + { + name: "map null with planResourceChange", + planResourceChange: true, + schemaType: schema.TypeMap, + cloudVal: map[string]interface{}{}, + programVal: "null", + expectedOutputTopLevel: nil, + expectedOutputNested: nil, + expectFailTopLevel: true, + expectFailNested: true, + }, + { + name: "map null without planResourceChange", + planResourceChange: false, + schemaType: schema.TypeMap, + cloudVal: map[string]interface{}{}, + programVal: "null", + // Note the difference in expected output between top level and nested properties + expectedOutputTopLevel: nil, + expectedOutputNested: map[string]interface{}{}, + // Note only fails at the top level + expectFailTopLevel: true, + }, + { + name: "map null with planResourceChange with nil cloud value", + planResourceChange: true, + schemaType: schema.TypeMap, + cloudVal: nil, + programVal: "null", + expectedOutputTopLevel: nil, + expectedOutputNested: nil, + expectFailTopLevel: true, + expectFailNested: true, + }, + { + name: "map null without planResourceChange with nil cloud value", + planResourceChange: false, + schemaType: schema.TypeMap, + cloudVal: nil, + programVal: "null", + expectedOutputTopLevel: nil, + expectedOutputNested: map[string]interface{}{}, + // Note only fails at the top level + expectFailTopLevel: true, + }, + { + name: "map null with planResourceChange with cloud override", + planResourceChange: true, + schemaType: schema.TypeMap, + cloudVal: map[string]interface{}{}, + programVal: "null", + createCloudValOverride: true, + expectedOutputTopLevel: nil, + expectedOutputNested: nil, + expectFailTopLevel: true, + expectFailNested: true, + }, + { + name: "map null without planResourceChange with cloud override", + planResourceChange: false, + schemaType: schema.TypeMap, + cloudVal: map[string]interface{}{}, + programVal: "null", + createCloudValOverride: true, + expectedOutputTopLevel: map[string]interface{}{}, + expectedOutputNested: map[string]interface{}{}, + }, + { + name: "map null with planResourceChange with nil cloud value and cloud override", + planResourceChange: true, + schemaType: schema.TypeMap, + cloudVal: nil, + programVal: "null", + createCloudValOverride: true, + expectedOutputTopLevel: nil, + expectedOutputNested: nil, + expectFailTopLevel: true, + expectFailNested: true, + }, + { + name: "map null without planResourceChange with nil cloud value and cloud override", + planResourceChange: false, + schemaType: schema.TypeMap, + cloudVal: nil, + programVal: "null", + createCloudValOverride: true, + expectedOutputTopLevel: map[string]interface{}{}, + expectedOutputNested: map[string]interface{}{}, + }, + { + name: "map empty with planResourceChange", + planResourceChange: true, + schemaType: schema.TypeMap, + cloudVal: map[string]interface{}{}, + programVal: "{}", + expectedOutputTopLevel: nil, + expectedOutputNested: nil, + expectFailTopLevel: true, + expectFailNested: true, + }, + { + name: "map empty without planResourceChange", + planResourceChange: false, + schemaType: schema.TypeMap, + cloudVal: map[string]interface{}{}, + programVal: "{}", + // Note the difference in expected output between top level and nested properties + expectedOutputTopLevel: nil, + expectedOutputNested: map[string]interface{}{}, + // Note only fails at the top level + expectFailTopLevel: true, + }, + { + name: "map empty with planResourceChange with cloud override", + planResourceChange: true, + schemaType: schema.TypeMap, + cloudVal: map[string]interface{}{}, + programVal: "{}", + createCloudValOverride: true, + expectedOutputTopLevel: nil, + expectedOutputNested: nil, + expectFailTopLevel: true, + expectFailNested: true, + }, + { + name: "map empty without planResourceChange with cloud override", + planResourceChange: false, + schemaType: schema.TypeMap, + cloudVal: map[string]interface{}{}, + programVal: "{}", + createCloudValOverride: true, + expectedOutputTopLevel: map[string]interface{}{}, + expectedOutputNested: map[string]interface{}{}, + }, + { + name: "map nonempty with planResourceChange", + planResourceChange: true, + schemaType: schema.TypeMap, + cloudVal: map[string]interface{}{"val": "test"}, + programVal: `{"val": "test"}`, + expectedOutputTopLevel: map[string]interface{}{"val": "test"}, + expectedOutputNested: map[string]interface{}{"val": "test"}, + }, + { + name: "map nonempty without planResourceChange", + planResourceChange: false, + schemaType: schema.TypeMap, + cloudVal: map[string]interface{}{"val": "test"}, + programVal: `{"val": "test"}`, + expectedOutputTopLevel: map[string]interface{}{"val": "test"}, + expectedOutputNested: map[string]interface{}{"val": "test"}, + }, + { + name: "map nonempty with planResourceChange with cloud override", + planResourceChange: true, + schemaType: schema.TypeMap, + cloudVal: map[string]interface{}{"val": "test"}, + programVal: `{"val": "test"}`, + createCloudValOverride: true, + expectedOutputTopLevel: map[string]interface{}{"val": "test"}, + expectedOutputNested: map[string]interface{}{"val": "test"}, + }, + { + name: "map nonempty without planResourceChange with cloud override", + planResourceChange: false, + schemaType: schema.TypeMap, + cloudVal: map[string]interface{}{"val": "test"}, + programVal: `{"val": "test"}`, + createCloudValOverride: true, + expectedOutputTopLevel: map[string]interface{}{"val": "test"}, + expectedOutputNested: map[string]interface{}{"val": "test"}, + }, + { + name: "list null with planResourceChange", + planResourceChange: true, + schemaType: schema.TypeList, + cloudVal: []interface{}{}, + programVal: "null", + expectedOutputTopLevel: nil, + expectedOutputNested: nil, + expectFailTopLevel: true, + expectFailNested: true, + }, + { + name: "list null without planResourceChange", + planResourceChange: false, + schemaType: schema.TypeList, + cloudVal: []interface{}{}, + programVal: "null", + // Note the difference in expected output between top level and nested properties + expectedOutputTopLevel: nil, + expectedOutputNested: []interface{}{}, + // Note only fails at the top level + expectFailTopLevel: true, + }, + { + name: "list null with planResourceChange with nil cloud value", + planResourceChange: true, + schemaType: schema.TypeList, + cloudVal: nil, + programVal: "null", + expectedOutputTopLevel: nil, + expectedOutputNested: nil, + expectFailTopLevel: true, + expectFailNested: true, + }, + { + name: "list null without planResourceChange with nil cloud value", + planResourceChange: false, + schemaType: schema.TypeList, + cloudVal: nil, + programVal: "null", + // Note the difference in expected output between top level and nested properties + expectedOutputTopLevel: nil, + expectedOutputNested: []interface{}{}, + // Note only fails at the top level + expectFailTopLevel: true, + }, + { + name: "list null with planResourceChange with cloud override", + planResourceChange: true, + schemaType: schema.TypeList, + cloudVal: []interface{}{}, + programVal: "null", + createCloudValOverride: true, + expectedOutputTopLevel: nil, + expectedOutputNested: nil, + expectFailTopLevel: true, + expectFailNested: true, + }, + { + name: "list null without planResourceChange with cloud override", + planResourceChange: false, + schemaType: schema.TypeList, + cloudVal: []interface{}{}, + programVal: "null", + createCloudValOverride: true, + expectedOutputTopLevel: []interface{}{}, + expectedOutputNested: []interface{}{}, + }, + { + name: "list null with planResourceChange with nil cloud value and cloud override", + planResourceChange: true, + schemaType: schema.TypeList, + cloudVal: nil, + programVal: "null", + createCloudValOverride: true, + expectedOutputTopLevel: nil, + expectedOutputNested: nil, + expectFailTopLevel: true, + expectFailNested: true, + }, + { + name: "list null without planResourceChange with nil cloud value and cloud override", + planResourceChange: false, + schemaType: schema.TypeList, + cloudVal: nil, + programVal: "null", + createCloudValOverride: true, + expectedOutputTopLevel: []interface{}{}, + expectedOutputNested: []interface{}{}, + }, + { + name: "list empty with planResourceChange", + planResourceChange: true, + schemaType: schema.TypeList, + cloudVal: []string{}, + programVal: "[]", + expectedOutputTopLevel: []interface{}{}, + expectedOutputNested: []interface{}{}, + }, + { + name: "list empty without planResourceChange", + planResourceChange: false, + schemaType: schema.TypeList, + cloudVal: []string{}, + programVal: "[]", + // Note the difference in expected output between top level and nested properties + expectedOutputTopLevel: nil, + expectedOutputNested: []interface{}{}, + // Note only fails at the top level + expectFailTopLevel: true, + }, + { + name: "list empty with planResourceChange with cloud override", + planResourceChange: true, + schemaType: schema.TypeList, + cloudVal: []string{}, + programVal: "[]", + createCloudValOverride: true, + expectedOutputTopLevel: []interface{}{}, + expectedOutputNested: []interface{}{}, + }, + { + name: "list empty without planResourceChange with cloud override", + planResourceChange: false, + schemaType: schema.TypeList, + cloudVal: []string{}, + programVal: "[]", + createCloudValOverride: true, + expectedOutputTopLevel: []interface{}{}, + expectedOutputNested: []interface{}{}, + }, + { + name: "list nonempty with planResourceChange", + planResourceChange: true, + schemaType: schema.TypeList, + cloudVal: []interface{}{"val"}, + programVal: `["val"]`, + expectedOutputTopLevel: []interface{}{"val"}, + expectedOutputNested: []interface{}{"val"}, + }, + { + name: "list nonempty without planResourceChange", + planResourceChange: false, + schemaType: schema.TypeList, + cloudVal: []interface{}{"val"}, + programVal: `["val"]`, + expectedOutputTopLevel: []interface{}{"val"}, + expectedOutputNested: []interface{}{"val"}, + }, + { + name: "list nonempty with planResourceChange with cloud override", + planResourceChange: true, + schemaType: schema.TypeList, + cloudVal: []interface{}{"val"}, + programVal: `["val"]`, + createCloudValOverride: true, + expectedOutputTopLevel: []interface{}{"val"}, + expectedOutputNested: []interface{}{"val"}, + }, + { + name: "list nonempty without planResourceChange with cloud override", + planResourceChange: false, + schemaType: schema.TypeList, + cloudVal: []interface{}{"val"}, + programVal: `["val"]`, + createCloudValOverride: true, + expectedOutputTopLevel: []interface{}{"val"}, + expectedOutputNested: []interface{}{"val"}, + }, + { + name: "set null with planResourceChange", + planResourceChange: true, + schemaType: schema.TypeSet, + cloudVal: []interface{}{}, + programVal: "null", + expectedOutputTopLevel: nil, + expectedOutputNested: nil, + expectFailTopLevel: true, + expectFailNested: true, + }, + { + name: "set null without planResourceChange", + planResourceChange: false, + schemaType: schema.TypeSet, + cloudVal: []interface{}{}, + programVal: "null", + // Note the difference in expected output between top level and nested properties + expectedOutputTopLevel: nil, + expectedOutputNested: []interface{}{}, + // Note only fails at the top level + expectFailTopLevel: true, + }, + { + name: "set null with planResourceChange with nil cloud value", + planResourceChange: true, + schemaType: schema.TypeSet, + cloudVal: nil, + programVal: "null", + expectedOutputTopLevel: nil, + expectedOutputNested: nil, + expectFailTopLevel: true, + expectFailNested: true, + }, + { + name: "set null without planResourceChange with nil cloud value", + planResourceChange: false, + schemaType: schema.TypeSet, + cloudVal: nil, + programVal: "null", + expectedOutputTopLevel: nil, + expectedOutputNested: []interface{}{}, + // Note only fails at the top level + expectFailTopLevel: true, + }, + { + name: "set null with planResourceChange with cloud override", + planResourceChange: true, + schemaType: schema.TypeSet, + cloudVal: []interface{}{}, + programVal: "null", + createCloudValOverride: true, + expectedOutputTopLevel: nil, + expectedOutputNested: nil, + expectFailTopLevel: true, + expectFailNested: true, + }, + { + name: "set null without planResourceChange with cloud override", + planResourceChange: false, + schemaType: schema.TypeSet, + cloudVal: []interface{}{}, + programVal: "null", + createCloudValOverride: true, + expectedOutputTopLevel: []interface{}{}, + expectedOutputNested: []interface{}{}, + }, + { + name: "set null with planResourceChange with nil cloud value and cloud override", + planResourceChange: true, + schemaType: schema.TypeSet, + cloudVal: nil, + programVal: "null", + createCloudValOverride: true, + expectedOutputTopLevel: nil, + expectedOutputNested: nil, + expectFailTopLevel: true, + expectFailNested: true, + }, + { + name: "set null without planResourceChange with nil cloud value and cloud override", + planResourceChange: false, + schemaType: schema.TypeSet, + cloudVal: nil, + programVal: "null", + createCloudValOverride: true, + expectedOutputTopLevel: []interface{}{}, + expectedOutputNested: []interface{}{}, + }, + { + name: "set empty with planResourceChange", + planResourceChange: true, + schemaType: schema.TypeSet, + cloudVal: []interface{}{}, + programVal: "[]", + expectedOutputTopLevel: nil, + expectedOutputNested: nil, + expectFailTopLevel: true, + expectFailNested: true, + }, + { + name: "set empty without planResourceChange", + planResourceChange: false, + schemaType: schema.TypeSet, + cloudVal: []interface{}{}, + programVal: "[]", + // Note the difference in expected output between top level and nested properties + expectedOutputTopLevel: nil, + expectedOutputNested: []interface{}{}, + // Note only fails at the top level + expectFailTopLevel: true, + }, + { + name: "set empty with planResourceChange with cloud override", + planResourceChange: true, + schemaType: schema.TypeSet, + cloudVal: []interface{}{}, + programVal: "[]", + createCloudValOverride: true, + expectedOutputTopLevel: nil, + expectedOutputNested: nil, + expectFailTopLevel: true, + expectFailNested: true, + }, + { + name: "set empty without planResourceChange with cloud override", + planResourceChange: false, + schemaType: schema.TypeSet, + cloudVal: []interface{}{}, + programVal: "[]", + createCloudValOverride: true, + expectedOutputTopLevel: []interface{}{}, + expectedOutputNested: []interface{}{}, + }, + { + name: "set nonempty with planResourceChange", + schemaType: schema.TypeSet, + cloudVal: []interface{}{"val"}, + programVal: `["val"]`, + expectedOutputTopLevel: []interface{}{"val"}, + expectedOutputNested: []interface{}{"val"}, + }, + { + name: "set nonempty without planResourceChange", + planResourceChange: false, + schemaType: schema.TypeSet, + cloudVal: []interface{}{"val"}, + programVal: `["val"]`, + expectedOutputTopLevel: []interface{}{"val"}, + expectedOutputNested: []interface{}{"val"}, + }, + { + name: "set nonempty with planResourceChange with cloud override", + schemaType: schema.TypeSet, + cloudVal: []interface{}{"val"}, + programVal: `["val"]`, + createCloudValOverride: true, + expectedOutputTopLevel: []interface{}{"val"}, + expectedOutputNested: []interface{}{"val"}, + }, + { + name: "set nonempty without planResourceChange with cloud override", + planResourceChange: false, + schemaType: schema.TypeSet, + cloudVal: []interface{}{"val"}, + programVal: `["val"]`, + createCloudValOverride: true, + expectedOutputTopLevel: []interface{}{"val"}, + expectedOutputNested: []interface{}{"val"}, + }, + } { + collectionPropPlural := "" + pluralized := tc.schemaType == schema.TypeList || tc.schemaType == schema.TypeSet + if pluralized { + collectionPropPlural += "s" + } + + opts := []pulcheck.BridgedProviderOpt{} + if !tc.planResourceChange { + opts = append(opts, pulcheck.DisablePlanResourceChange()) + } + + t.Run(tc.name, func(t *testing.T) { + t.Run("top level", func(t *testing.T) { + t.Parallel() + resMap := map[string]*schema.Resource{ + "prov_test": { + Schema: map[string]*schema.Schema{ + "collection_prop": { + Type: tc.schemaType, + Optional: true, + Elem: &schema.Schema{Type: schema.TypeString}, + }, + "other_prop": { + Type: schema.TypeString, + Optional: true, + }, + }, + ReadContext: func(_ context.Context, rd *schema.ResourceData, _ interface{}) diag.Diagnostics { + err := rd.Set("collection_prop", tc.cloudVal) + require.NoError(t, err) + err = rd.Set("other_prop", "test") + require.NoError(t, err) + return nil + }, + CreateContext: func(_ context.Context, rd *schema.ResourceData, _ interface{}) diag.Diagnostics { + if tc.createCloudValOverride { + err := rd.Set("collection_prop", tc.cloudVal) + require.NoError(t, err) + } + + rd.SetId("id0") + return nil + }, + }, + } + + bridgedProvider := pulcheck.BridgedProvider(t, "prov", resMap, opts...) + program := fmt.Sprintf(` +name: test +runtime: yaml +resources: + mainRes: + type: prov:index:Test + properties: + otherProp: "test" + collectionProp%s: %s +outputs: + collectionOutput: ${mainRes.collectionProp%s} +`, collectionPropPlural, tc.programVal, collectionPropPlural) + pt := pulcheck.PulCheck(t, bridgedProvider, program) + upRes := pt.Up() + require.Equal(t, tc.expectedOutputTopLevel, upRes.Outputs["collectionOutput"].Value) + res, err := pt.CurrentStack().Refresh(pt.Context(), optrefresh.ExpectNoChanges()) + if tc.expectFailTopLevel { + require.Error(t, err) + } else { + require.NoError(t, err) + } + t.Logf(res.StdOut) + }) + + t.Run("nested", func(t *testing.T) { + t.Parallel() + resMap := map[string]*schema.Resource{ + "prov_test": { + Schema: map[string]*schema.Schema{ + "prop": { + Type: schema.TypeList, + Optional: true, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "collection_prop": { + Type: tc.schemaType, + Optional: true, + Elem: &schema.Schema{Type: schema.TypeString}, + }, + "other_nested_prop": { + Type: schema.TypeString, + Optional: true, + }, + }, + }, + }, + "other_prop": { + Type: schema.TypeString, + Optional: true, + }, + }, + ReadContext: func(_ context.Context, rd *schema.ResourceData, _ interface{}) diag.Diagnostics { + err := rd.Set("prop", []map[string]interface{}{{"collection_prop": tc.cloudVal, "other_nested_prop": "test"}}) + require.NoError(t, err) + err = rd.Set("other_prop", "test") + require.NoError(t, err) + + return nil + }, + CreateContext: func(_ context.Context, rd *schema.ResourceData, _ interface{}) diag.Diagnostics { + if tc.createCloudValOverride { + err := rd.Set("prop", []map[string]interface{}{{"collection_prop": tc.cloudVal, "other_nested_prop": "test"}}) + require.NoError(t, err) + } + rd.SetId("id0") + return nil + }, + }, + } + + bridgedProvider := pulcheck.BridgedProvider(t, "prov", resMap, opts...) + program := fmt.Sprintf(` +name: test +runtime: yaml +resources: + mainRes: + type: prov:index:Test + properties: + otherProp: "test" + props: + - collectionProp%s: %s + otherNestedProp: "test" +outputs: + collectionOutput: ${mainRes.props[0].collectionProp%s} +`, collectionPropPlural, tc.programVal, collectionPropPlural) + pt := pulcheck.PulCheck(t, bridgedProvider, program) + upRes := pt.Up() + require.Equal(t, tc.expectedOutputNested, upRes.Outputs["collectionOutput"].Value) + + res, err := pt.CurrentStack().Refresh(pt.Context(), optrefresh.ExpectNoChanges()) + if tc.expectFailNested { + require.Error(t, err) + } else { + require.NoError(t, err) + } + + t.Logf(res.StdOut) + }) + }) + } +}