Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

sort array of schema for bigquery_table #8321

Merged
merged 1 commit into from
Jan 28, 2021
Merged

sort array of schema for bigquery_table #8321

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .changelog/4440.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
```release-note:bug
bigquery: fixed bug where you could not reorder columns on `schema` for resource `google_bigquery_table`
```
33 changes: 33 additions & 0 deletions google/resource_bigquery_table.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"log"
"sort"

"github.com/hashicorp/terraform-plugin-sdk/v2/helper/customdiff"
"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
Expand All @@ -14,6 +15,22 @@ import (
"google.golang.org/api/bigquery/v2"
)

func bigQueryTableSortArrayByName(array []interface{}) {
sort.Slice(array, func(i, k int) bool {
return array[i].(map[string]interface{})["name"].(string) < array[k].(map[string]interface{})["name"].(string)
})
}

func bigQueryTablecheckNameExists(jsonList []interface{}) error {
for _, m := range jsonList {
if _, ok := m.(map[string]interface{})["name"]; !ok {
return fmt.Errorf("No name in schema %+v", m)
}
}

return nil
}

// Compares two json's while optionally taking in a compareMapKeyVal function.
// This function will override any comparison of a given map[string]interface{}
// on a specific key value allowing for a separate equality in specific scenarios
Expand All @@ -27,6 +44,14 @@ func jsonCompareWithMapKeyOverride(a, b interface{}, compareMapKeyVal func(key s
} else if len(arrayA) != len(arrayB) {
return false, nil
}
if err := bigQueryTablecheckNameExists(arrayA); err != nil {
return false, err
}
bigQueryTableSortArrayByName(arrayA)
if err := bigQueryTablecheckNameExists(arrayB); err != nil {
return false, err
}
bigQueryTableSortArrayByName(arrayB)
for i := range arrayA {
eq, err := jsonCompareWithMapKeyOverride(arrayA[i], arrayB[i], compareMapKeyVal)
if err != nil {
Expand Down Expand Up @@ -169,6 +194,14 @@ func resourceBigQueryTableSchemaIsChangeable(old, new interface{}) (bool, error)
// if not growing not changeable
return false, nil
}
if err := bigQueryTablecheckNameExists(arrayOld); err != nil {
return false, err
}
bigQueryTableSortArrayByName(arrayOld)
if err := bigQueryTablecheckNameExists(arrayNew); err != nil {
return false, err
}
bigQueryTableSortArrayByName(arrayNew)
for i := range arrayOld {
if isChangable, err :=
resourceBigQueryTableSchemaIsChangeable(arrayOld[i], arrayNew[i]); err != nil || !isChangable {
Expand Down
172 changes: 142 additions & 30 deletions google/resource_bigquery_table_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -438,7 +438,7 @@ func TestAccBigQueryDataTable_jsonEquivalency(t *testing.T) {
ImportStateVerifyIgnore: []string{"etag", "last_modified_time"},
},
{
Config: testAccBigQueryTable_jsonEqUpdate(datasetID, tableID),
Config: testAccBigQueryTable_jsonEqModeRemoved(datasetID, tableID),
},
{
ResourceName: "google_bigquery_table.test",
Expand All @@ -450,6 +450,32 @@ func TestAccBigQueryDataTable_jsonEquivalency(t *testing.T) {
})
}

func TestAccBigQueryDataTable_canReorderParameters(t *testing.T) {
t.Parallel()

datasetID := fmt.Sprintf("tf_test_%s", randString(t, 10))
tableID := fmt.Sprintf("tf_test_%s", randString(t, 10))

vcrTest(t, resource.TestCase{
PreCheck: func() { testAccPreCheck(t) },
Providers: testAccProviders,
CheckDestroy: testAccCheckBigQueryTableDestroyProducer(t),
Steps: []resource.TestStep{
{
// we don't run any checks because the resource will error out if
// it attempts to destroy/tear down.
Config: testAccBigQueryTable_jsonPreventDestroy(datasetID, tableID),
},
{
Config: testAccBigQueryTable_jsonPreventDestroyOrderChanged(datasetID, tableID),
},
{
Config: testAccBigQueryTable_jsonEq(datasetID, tableID),
},
},
})
}

func TestUnitBigQueryDataTable_jsonEquivalency(t *testing.T) {
t.Parallel()

Expand Down Expand Up @@ -598,77 +624,91 @@ var testUnitBigQueryDataTableIsChangableTestCases = []testUnitBigQueryDataTableJ
jsonNew: "[{\"name\": \"someValue\", \"type\" : \"BOOLEAN\", \"description\" : \"some new value\" }]",
changeable: false,
},
{
name: "orderOfArrayChangesAndDescriptionChanges",
jsonOld: "[{\"name\": \"value1\", \"type\" : \"INTEGER\", \"mode\" : \"NULLABLE\", \"description\" : \"someVal\" }, {\"name\": \"value2\", \"type\" : \"BOOLEAN\", \"mode\" : \"NULLABLE\", \"description\" : \"someVal\" }]",
jsonNew: "[{\"name\": \"value2\", \"type\" : \"BOOLEAN\", \"mode\" : \"NULLABLE\", \"description\" : \"newVal\" }, {\"name\": \"value1\", \"type\" : \"INTEGER\", \"mode\" : \"NULLABLE\", \"description\" : \"someVal\" }]",
changeable: true,
},
{
name: "orderOfArrayChangesAndNameChanges",
jsonOld: "[{\"name\": \"value1\", \"type\" : \"INTEGER\", \"mode\" : \"NULLABLE\", \"description\" : \"someVal\" }, {\"name\": \"value2\", \"type\" : \"BOOLEAN\", \"mode\" : \"NULLABLE\", \"description\" : \"someVal\" }]",
jsonNew: "[{\"name\": \"value3\", \"type\" : \"BOOLEAN\", \"mode\" : \"NULLABLE\", \"description\" : \"newVal\" }, {\"name\": \"value1\", \"type\" : \"INTEGER\", \"mode\" : \"NULLABLE\", \"description\" : \"someVal\" }]",
changeable: false,
},
}

var testUnitBigQueryDataTableJSONEquivalencyTestCases = []testUnitBigQueryDataTableJSONEquivalencyTestCase{
{
"[{\"someKey\": \"someValue\", \"anotherKey\" : \"anotherValue\", \"finalKey\" : {} }]",
"[{\"someKey\": \"someValue\", \"anotherKey\" : \"anotherValue\", \"finalKey\" : {} }]",
"[{\"name\": \"someValue\", \"anotherKey\" : \"anotherValue\", \"finalKey\" : {} }]",
"[{\"name\": \"someValue\", \"anotherKey\" : \"anotherValue\", \"finalKey\" : {} }]",
true,
},
{
"[{\"someKey\": \"someValue\", \"anotherKey\" : \"anotherValue\", \"finalKey\" : {} }]",
"[{\"someKey\": \"someValue\", \"finalKey\" : {} }]",
"[{\"name\": \"someValue\", \"anotherKey\" : \"anotherValue\", \"finalKey\" : {} }]",
"[{\"name\": \"someValue\", \"finalKey\" : {} }]",
false,
},
{
"[{\"someKey\": \"someValue\", \"anotherKey\" : \"anotherValue\", \"mode\": \"NULLABLE\" }]",
"[{\"someKey\": \"someValue\", \"anotherKey\" : \"anotherValue\" }]",
"[{\"name\": \"someValue\", \"anotherKey\" : \"anotherValue\", \"mode\": \"NULLABLE\" }]",
"[{\"name\": \"someValue\", \"anotherKey\" : \"anotherValue\" }]",
true,
},
{
"[{\"someKey\": \"someValue\", \"anotherKey\" : \"anotherValue\", \"mode\": \"NULLABLE\" }]",
"[{\"someKey\": \"someValue\", \"anotherKey\" : \"anotherValue\", \"mode\": \"somethingRandom\" }]",
"[{\"name\": \"someValue\", \"anotherKey\" : \"anotherValue\", \"mode\": \"NULLABLE\" }]",
"[{\"name\": \"someValue\", \"anotherKey\" : \"anotherValue\", \"mode\": \"somethingRandom\" }]",
false,
},
{
"[{\"someKey\": \"someValue\", \"anotherKey\" : \"anotherValue\", \"description\": \"\" }]",
"[{\"someKey\": \"someValue\", \"anotherKey\" : \"anotherValue\" }]",
"[{\"name\": \"someValue\", \"anotherKey\" : \"anotherValue\", \"description\": \"\" }]",
"[{\"name\": \"someValue\", \"anotherKey\" : \"anotherValue\" }]",
true,
},
{
"[{\"someKey\": \"someValue\", \"anotherKey\" : \"anotherValue\", \"description\": \"\" }]",
"[{\"someKey\": \"someValue\", \"anotherKey\" : \"anotherValue\", \"description\": \"somethingRandom\" }]",
"[{\"name\": \"someValue\", \"anotherKey\" : \"anotherValue\", \"description\": \"\" }]",
"[{\"name\": \"someValue\", \"anotherKey\" : \"anotherValue\", \"description\": \"somethingRandom\" }]",
false,
},
{
"[{\"someKey\": \"someValue\", \"anotherKey\" : \"anotherValue\", \"type\": \"INTEGER\" }]",
"[{\"someKey\": \"someValue\", \"anotherKey\" : \"anotherValue\", \"type\": \"INT64\" }]",
"[{\"name\": \"someValue\", \"anotherKey\" : \"anotherValue\", \"type\": \"INTEGER\" }]",
"[{\"name\": \"someValue\", \"anotherKey\" : \"anotherValue\", \"type\": \"INT64\" }]",
true,
},
{
"[{\"someKey\": \"someValue\", \"anotherKey\" : \"anotherValue\", \"type\": \"INTEGER\" }]",
"[{\"someKey\": \"someValue\", \"anotherKey\" : \"anotherValue\", \"type\": \"somethingRandom\" }]",
"[{\"name\": \"someValue\", \"anotherKey\" : \"anotherValue\", \"type\": \"INTEGER\" }]",
"[{\"name\": \"someValue\", \"anotherKey\" : \"anotherValue\", \"type\": \"somethingRandom\" }]",
false,
},
{
"[{\"someKey\": \"someValue\", \"anotherKey\" : \"anotherValue\", \"type\": \"FLOAT\" }]",
"[{\"someKey\": \"someValue\", \"anotherKey\" : \"anotherValue\", \"type\": \"FLOAT64\" }]",
"[{\"name\": \"someValue\", \"anotherKey\" : \"anotherValue\", \"type\": \"FLOAT\" }]",
"[{\"name\": \"someValue\", \"anotherKey\" : \"anotherValue\", \"type\": \"FLOAT64\" }]",
true,
},
{
"[{\"someKey\": \"someValue\", \"anotherKey\" : \"anotherValue\", \"type\": \"FLOAT\" }]",
"[{\"someKey\": \"someValue\", \"anotherKey\" : \"anotherValue\" }]",
"[{\"name\": \"someValue\", \"anotherKey\" : \"anotherValue\", \"type\": \"FLOAT\" }]",
"[{\"name\": \"someValue\", \"anotherKey\" : \"anotherValue\" }]",
false,
},
{
"[{\"someKey\": \"someValue\", \"anotherKey\" : \"anotherValue\", \"type\": \"BOOLEAN\" }]",
"[{\"someKey\": \"someValue\", \"anotherKey\" : \"anotherValue\", \"type\": \"BOOL\" }]",
"[{\"name\": \"someValue\", \"anotherKey\" : \"anotherValue\", \"type\": \"BOOLEAN\" }]",
"[{\"name\": \"someValue\", \"anotherKey\" : \"anotherValue\", \"type\": \"BOOL\" }]",
true,
},
{
"[{\"someKey\": \"someValue\", \"anotherKey\" : \"anotherValue\", \"type\": \"BOOLEAN\" }]",
"[{\"someKey\": \"someValue\", \"anotherKey\" : \"anotherValue\" }]",
"[{\"name\": \"someValue\", \"anotherKey\" : \"anotherValue\", \"type\": \"BOOLEAN\" }]",
"[{\"name\": \"someValue\", \"anotherKey\" : \"anotherValue\" }]",
false,
},
{
"[1,2,3]",
"[1,2,3]",
// order changes, name same
"[{\"name\": \"value1\", \"1\" : \"1\"},{\"name\": \"value2\", \"2\" : \"2\" }]",
"[{\"name\": \"value2\", \"2\" : \"2\" },{\"name\": \"value1\", \"1\" : \"1\" }]",
true,
},
{
"[1,2,3]",
"[1,2,\"banana\"]",
// order changes, value different
"[{\"name\": \"value1\", \"1\" : \"1\"},{\"name\": \"value2\", \"2\" : \"2\" }]",
"[{\"name\": \"value2\", \"2\" : \"random\" },{\"name\": \"value1\", \"1\" : \"1\" }]",
false,
},
}
Expand Down Expand Up @@ -1338,7 +1378,7 @@ resource "google_bigquery_table" "test" {
`, datasetID, tableID)
}

func testAccBigQueryTable_jsonEqUpdate(datasetID, tableID string) string {
func testAccBigQueryTable_jsonEqModeRemoved(datasetID, tableID string) string {
return fmt.Sprintf(`
resource "google_bigquery_dataset" "test" {
dataset_id = "%s"
Expand Down Expand Up @@ -1370,6 +1410,78 @@ resource "google_bigquery_table" "test" {
`, datasetID, tableID)
}

func testAccBigQueryTable_jsonPreventDestroy(datasetID, tableID string) string {
return fmt.Sprintf(`
resource "google_bigquery_dataset" "test" {
dataset_id = "%s"
}

resource "google_bigquery_table" "test" {
table_id = "%s"
dataset_id = google_bigquery_dataset.test.dataset_id
lifecycle {
prevent_destroy = true
}

friendly_name = "bigquerytest"
labels = {
"terrafrom_managed" = "true"
}

schema = jsonencode(
[
{
description = "Time snapshot was taken, in Epoch milliseconds. Same across all rows and all tables in the snapshot, and uniquely defines a particular snapshot."
name = "snapshot_timestamp"
mode = "NULLABLE"
type = "INTEGER"
},
{
description = "Timestamp of dataset creation"
name = "creation_time"
type = "TIMESTAMP"
},
])
}
`, datasetID, tableID)
}

func testAccBigQueryTable_jsonPreventDestroyOrderChanged(datasetID, tableID string) string {
return fmt.Sprintf(`
resource "google_bigquery_dataset" "test" {
dataset_id = "%s"
}

resource "google_bigquery_table" "test" {
table_id = "%s"
dataset_id = google_bigquery_dataset.test.dataset_id
lifecycle {
prevent_destroy = true
}

friendly_name = "bigquerytest"
labels = {
"terrafrom_managed" = "true"
}

schema = jsonencode(
[
{
description = "Timestamp of dataset creation"
name = "creation_time"
type = "TIMESTAMP"
},
{
description = "Time snapshot was taken, in Epoch milliseconds. Same across all rows and all tables in the snapshot, and uniquely defines a particular snapshot."
name = "snapshot_timestamp"
mode = "NULLABLE"
type = "INTEGER"
},
])
}
`, datasetID, tableID)
}

var TEST_CSV = `lifelock,LifeLock,,web,Tempe,AZ,1-May-07,6850000,USD,b
lifelock,LifeLock,,web,Tempe,AZ,1-Oct-06,6000000,USD,a
lifelock,LifeLock,,web,Tempe,AZ,1-Jan-08,25000000,USD,c
Expand Down