From d3ad8a8019ffff65e644e347e21b8b1512be65c4 Mon Sep 17 00:00:00 2001 From: Harsh Chouraria Date: Sat, 17 Sep 2022 00:35:30 +0530 Subject: [PATCH] fix: Pass file_format values as-is in external table configuration (#1183) Common use of FILE_FORMAT under CREATE EXTERNAL TABLE includes specifying a previously created format name (FORMAT_NAME = 'NAME') or the various type options (TYPE=CSV FIELD_DELIMITER='|'). Both of these require defining string literals with the single quote character (') that should not be escaped (\\'). This change removes the escaping of single quotes performed for the file_format values to allow them to be specified without failing the query compilation. Some examples have also been added to the documentation to make it easier to understand how values for file_format need to be passed for external tables. Testing: - Modified unit tests to exercise passing of typical file format values - Ran 'make test' to confirm existing tests continue to pass - Setup a trial account, exported env-vars and ran 'make test-acceptance' and verified all tests passed - Attempted an external table creation on a personal account with the changes included through a local buildand observed it to execute a successful SQL. Tried with both FORMAT_NAME and FIELD_DELIMITER options with string literal values. Fixes #1046 --- docs/resources/external_table.md | 9 +++++---- examples/resources/snowflake_external_table/resource.tf | 9 +++++---- pkg/resources/external_table_test.go | 4 ++-- pkg/snowflake/external_table.go | 2 +- pkg/snowflake/external_table_test.go | 6 +++--- 5 files changed, 16 insertions(+), 14 deletions(-) diff --git a/docs/resources/external_table.md b/docs/resources/external_table.md index 8fac34ca66..3e91e2f131 100644 --- a/docs/resources/external_table.md +++ b/docs/resources/external_table.md @@ -14,10 +14,11 @@ description: |- ```terraform resource snowflake_external_table external_table { - database = "db" - schema = "schema" - name = "external_table" - comment = "External table" + database = "db" + schema = "schema" + name = "external_table" + comment = "External table" + file_format = "TYPE = CSV FIELD_DELIMITER = '|'" column { name = "id" diff --git a/examples/resources/snowflake_external_table/resource.tf b/examples/resources/snowflake_external_table/resource.tf index 5868c2c5ca..68b9f47d04 100644 --- a/examples/resources/snowflake_external_table/resource.tf +++ b/examples/resources/snowflake_external_table/resource.tf @@ -1,8 +1,9 @@ resource snowflake_external_table external_table { - database = "db" - schema = "schema" - name = "external_table" - comment = "External table" + database = "db" + schema = "schema" + name = "external_table" + comment = "External table" + file_format = "TYPE = CSV FIELD_DELIMITER = '|'" column { name = "id" diff --git a/pkg/resources/external_table_test.go b/pkg/resources/external_table_test.go index 087a6a9030..0cad818fc6 100644 --- a/pkg/resources/external_table_test.go +++ b/pkg/resources/external_table_test.go @@ -27,13 +27,13 @@ func TestExternalTableCreate(t *testing.T) { "comment": "great comment", "column": []interface{}{map[string]interface{}{"name": "column1", "type": "OBJECT", "as": "a"}, map[string]interface{}{"name": "column2", "type": "VARCHAR", "as": "b"}}, "location": "location", - "file_format": "format", + "file_format": "FORMAT_NAME = 'format'", "pattern": "pattern", } d := externalTable(t, "database_name|schema_name|good_name", in) WithMockDb(t, func(db *sql.DB, mock sqlmock.Sqlmock) { - mock.ExpectExec(`CREATE EXTERNAL TABLE "database_name"."schema_name"."good_name" \("column1" OBJECT AS a, "column2" VARCHAR AS b\) WITH LOCATION = location REFRESH_ON_CREATE = true AUTO_REFRESH = true PATTERN = 'pattern' FILE_FORMAT = \( format \) COMMENT = 'great comment'`).WillReturnResult(sqlmock.NewResult(1, 1)) + mock.ExpectExec(`CREATE EXTERNAL TABLE "database_name"."schema_name"."good_name" \("column1" OBJECT AS a, "column2" VARCHAR AS b\) WITH LOCATION = location REFRESH_ON_CREATE = true AUTO_REFRESH = true PATTERN = 'pattern' FILE_FORMAT = \( FORMAT_NAME = 'format' \) COMMENT = 'great comment'`).WillReturnResult(sqlmock.NewResult(1, 1)) expectExternalTableRead(mock) err := resources.CreateExternalTable(d, db) diff --git a/pkg/snowflake/external_table.go b/pkg/snowflake/external_table.go index 73f0b9abb6..c3f5394e0a 100644 --- a/pkg/snowflake/external_table.go +++ b/pkg/snowflake/external_table.go @@ -141,7 +141,7 @@ func (tb *ExternalTableBuilder) Create() string { q.WriteString(fmt.Sprintf(` PATTERN = '%v'`, EscapeString(tb.pattern))) } - q.WriteString(fmt.Sprintf(` FILE_FORMAT = ( %v )`, EscapeString(tb.fileFormat))) + q.WriteString(fmt.Sprintf(` FILE_FORMAT = ( %v )`, tb.fileFormat)) if tb.awsSNSTopic != "" { q.WriteString(fmt.Sprintf(` AWS_SNS_TOPIC = '%v'`, EscapeString(tb.awsSNSTopic))) diff --git a/pkg/snowflake/external_table_test.go b/pkg/snowflake/external_table_test.go index 8eaa9a59fd..3e2ea06fd6 100644 --- a/pkg/snowflake/external_table_test.go +++ b/pkg/snowflake/external_table_test.go @@ -12,13 +12,13 @@ func TestExternalTableCreate(t *testing.T) { s.WithColumns([]map[string]string{{"name": "column1", "type": "OBJECT", "as": "expression1"}, {"name": "column2", "type": "VARCHAR", "as": "expression2"}}) s.WithLocation("location") s.WithPattern("pattern") - s.WithFileFormat("file format") + s.WithFileFormat("TYPE = CSV FIELD_DELIMITER = '|'") r.Equal(s.QualifiedName(), `"test_db"."test_schema"."test_table"`) - r.Equal(s.Create(), `CREATE EXTERNAL TABLE "test_db"."test_schema"."test_table" ("column1" OBJECT AS expression1, "column2" VARCHAR AS expression2) WITH LOCATION = location REFRESH_ON_CREATE = false AUTO_REFRESH = false PATTERN = 'pattern' FILE_FORMAT = ( file format )`) + r.Equal(s.Create(), `CREATE EXTERNAL TABLE "test_db"."test_schema"."test_table" ("column1" OBJECT AS expression1, "column2" VARCHAR AS expression2) WITH LOCATION = location REFRESH_ON_CREATE = false AUTO_REFRESH = false PATTERN = 'pattern' FILE_FORMAT = ( TYPE = CSV FIELD_DELIMITER = '|' )`) s.WithComment("Test Comment") - r.Equal(s.Create(), `CREATE EXTERNAL TABLE "test_db"."test_schema"."test_table" ("column1" OBJECT AS expression1, "column2" VARCHAR AS expression2) WITH LOCATION = location REFRESH_ON_CREATE = false AUTO_REFRESH = false PATTERN = 'pattern' FILE_FORMAT = ( file format ) COMMENT = 'Test Comment'`) + r.Equal(s.Create(), `CREATE EXTERNAL TABLE "test_db"."test_schema"."test_table" ("column1" OBJECT AS expression1, "column2" VARCHAR AS expression2) WITH LOCATION = location REFRESH_ON_CREATE = false AUTO_REFRESH = false PATTERN = 'pattern' FILE_FORMAT = ( TYPE = CSV FIELD_DELIMITER = '|' ) COMMENT = 'Test Comment'`) } func TestExternalTableUpdate(t *testing.T) {