Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: add support for JSON data type #39

Merged
merged 3 commits into from
Sep 5, 2021
Merged

feat: add support for JSON data type #39

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
828ebee
feat: add support for JSON data type
olavloite Sep 1, 2021
98332fa
fix: remove parameter exceeding the length of sql string
olavloite Sep 1, 2021
dd18ed5
Merge branch 'main' into json-data-type
olavloite Sep 5, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
58 changes: 43 additions & 15 deletions checksum_row_iterator_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,15 +35,24 @@ func TestUpdateChecksum(t *testing.T) {

row1, err := spanner.NewRow(
[]string{
"ColBool", "ColInt64", "ColFloat64", "ColNumeric", "ColString", "ColBytes", "ColDate", "ColTimestamp",
"ArrBool", "ArrInt64", "ArrFloat64", "ArrNumeric", "ArrString", "ArrBytes", "ArrDate", "ArrTimestamp",
"ColBool", "ColInt64", "ColFloat64", "ColNumeric", "ColString", "ColBytes", "ColDate", "ColTimestamp", "ColJson",
"ArrBool", "ArrInt64", "ArrFloat64", "ArrNumeric", "ArrString", "ArrBytes", "ArrDate", "ArrTimestamp", "ArrJson",
},
[]interface{}{
true, int64(1), 3.14, numeric("6.626"), "test", []byte("testbytes"), civil.Date{Year: 2021, Month: 8, Day: 5}, time.Date(2021, 8, 5, 13, 19, 23, 123456789, time.UTC),
true, int64(1), 3.14, numeric("6.626"), "test", []byte("testbytes"), civil.Date{Year: 2021, Month: 8, Day: 5},
time.Date(2021, 8, 5, 13, 19, 23, 123456789, time.UTC),
nullJson(true, `"key": "value", "other-key": ["value1", "value2"]}`),
[]bool{true, false}, []int64{1, 2}, []float64{3.14, 6.626}, []big.Rat{numeric("3.14"), numeric("6.626")},
[]string{"test1", "test2"}, [][]byte{[]byte("testbytes1"), []byte("testbytes1")},
[]civil.Date{{Year: 2021, Month: 8, Day: 5}, {Year: 2021, Month: 8, Day: 6}},
[]time.Time{time.Date(2021, 8, 5, 13, 19, 23, 123456789, time.UTC), time.Date(2021, 8, 6, 13, 19, 23, 123456789, time.UTC)},
[]time.Time{
time.Date(2021, 8, 5, 13, 19, 23, 123456789, time.UTC),
time.Date(2021, 8, 6, 13, 19, 23, 123456789, time.UTC),
},
[]spanner.NullJSON{
nullJson(true, `"key1": "value1", "other-key1": ["value1", "value2"]}`),
nullJson(true, `"key2": "value2", "other-key2": ["value1", "value2"]}`),
},
},
)
if err != nil {
Expand All @@ -52,15 +61,24 @@ func TestUpdateChecksum(t *testing.T) {
// row2 is different from row1
row2, err := spanner.NewRow(
[]string{
"ColBool", "ColInt64", "ColFloat64", "ColNumeric", "ColString", "ColBytes", "ColDate", "ColTimestamp",
"ArrBool", "ArrInt64", "ArrFloat64", "ArrNumeric", "ArrString", "ArrBytes", "ArrDate", "ArrTimestamp",
"ColBool", "ColInt64", "ColFloat64", "ColNumeric", "ColString", "ColBytes", "ColDate", "ColTimestamp", "ColJson",
"ArrBool", "ArrInt64", "ArrFloat64", "ArrNumeric", "ArrString", "ArrBytes", "ArrDate", "ArrTimestamp", "ArrJson",
},
[]interface{}{
true, int64(2), 6.626, numeric("3.14"), "test2", []byte("testbytes2"), civil.Date{Year: 2020, Month: 8, Day: 5}, time.Date(2020, 8, 5, 13, 19, 23, 123456789, time.UTC),
true, int64(2), 6.626, numeric("3.14"), "test2", []byte("testbytes2"), civil.Date{Year: 2020, Month: 8, Day: 5},
time.Date(2020, 8, 5, 13, 19, 23, 123456789, time.UTC),
nullJson(true, `"key": "other-value", "other-key": ["other-value1", "other-value2"]}`),
[]bool{true, false}, []int64{1, 2}, []float64{3.14, 6.626}, []big.Rat{numeric("3.14"), numeric("6.626")},
[]string{"test1_", "test2_"}, [][]byte{[]byte("testbytes1_"), []byte("testbytes1_")},
[]civil.Date{{Year: 2020, Month: 8, Day: 5}, {Year: 2020, Month: 8, Day: 6}},
[]time.Time{time.Date(2020, 8, 5, 13, 19, 23, 123456789, time.UTC), time.Date(2020, 8, 6, 13, 19, 23, 123456789, time.UTC)},
[]time.Time{
time.Date(2020, 8, 5, 13, 19, 23, 123456789, time.UTC),
time.Date(2020, 8, 6, 13, 19, 23, 123456789, time.UTC),
},
[]spanner.NullJSON{
nullJson(true, `"key1": "other-value1", "other-key1": ["other-value1", "other-value2"]}`),
nullJson(true, `"key2": "other-value2", "other-key2": ["other-value1", "other-value2"]}`),
},
},
)
if err != nil {
Expand All @@ -69,15 +87,24 @@ func TestUpdateChecksum(t *testing.T) {
// row3 is equal to row1.
row3, err := spanner.NewRow(
[]string{
"ColBool", "ColInt64", "ColFloat64", "ColNumeric", "ColString", "ColBytes", "ColDate", "ColTimestamp",
"ArrBool", "ArrInt64", "ArrFloat64", "ArrNumeric", "ArrString", "ArrBytes", "ArrDate", "ArrTimestamp",
"ColBool", "ColInt64", "ColFloat64", "ColNumeric", "ColString", "ColBytes", "ColDate", "ColTimestamp", "ColJson",
"ArrBool", "ArrInt64", "ArrFloat64", "ArrNumeric", "ArrString", "ArrBytes", "ArrDate", "ArrTimestamp", "ArrJson",
},
[]interface{}{
true, int64(1), 3.14, numeric("6.626"), "test", []byte("testbytes"), civil.Date{Year: 2021, Month: 8, Day: 5}, time.Date(2021, 8, 5, 13, 19, 23, 123456789, time.UTC),
true, int64(1), 3.14, numeric("6.626"), "test", []byte("testbytes"), civil.Date{Year: 2021, Month: 8, Day: 5},
time.Date(2021, 8, 5, 13, 19, 23, 123456789, time.UTC),
nullJson(true, `"key": "value", "other-key": ["value1", "value2"]}`),
[]bool{true, false}, []int64{1, 2}, []float64{3.14, 6.626}, []big.Rat{numeric("3.14"), numeric("6.626")},
[]string{"test1", "test2"}, [][]byte{[]byte("testbytes1"), []byte("testbytes1")},
[]civil.Date{{Year: 2021, Month: 8, Day: 5}, {Year: 2021, Month: 8, Day: 6}},
[]time.Time{time.Date(2021, 8, 5, 13, 19, 23, 123456789, time.UTC), time.Date(2021, 8, 6, 13, 19, 23, 123456789, time.UTC)},
[]time.Time{
time.Date(2021, 8, 5, 13, 19, 23, 123456789, time.UTC),
time.Date(2021, 8, 6, 13, 19, 23, 123456789, time.UTC),
},
[]spanner.NullJSON{
nullJson(true, `"key1": "value1", "other-key1": ["value1", "value2"]}`),
nullJson(true, `"key2": "value2", "other-key2": ["value1", "value2"]}`),
},
},
)
if err != nil {
Expand Down Expand Up @@ -138,15 +165,16 @@ func TestUpdateChecksumForNullValues(t *testing.T) {

row, err := spanner.NewRow(
[]string{
"ColBool", "ColInt64", "ColFloat64", "ColNumeric", "ColString", "ColBytes", "ColDate", "ColTimestamp",
"ArrBool", "ArrInt64", "ArrFloat64", "ArrNumeric", "ArrString", "ArrBytes", "ArrDate", "ArrTimestamp",
"ColBool", "ColInt64", "ColFloat64", "ColNumeric", "ColString", "ColBytes", "ColDate", "ColTimestamp", "ColJson",
"ArrBool", "ArrInt64", "ArrFloat64", "ArrNumeric", "ArrString", "ArrBytes", "ArrDate", "ArrTimestamp", "ArrJson",
},
[]interface{}{
spanner.NullBool{}, spanner.NullInt64{}, spanner.NullFloat64{}, spanner.NullNumeric{}, spanner.NullString{},
[]byte(nil), spanner.NullDate{}, spanner.NullTime{},
[]byte(nil), spanner.NullDate{}, spanner.NullTime{}, spanner.NullJSON{},
// Note: The following arrays all contain one NULL value.
[]spanner.NullBool{{}}, []spanner.NullInt64{{}}, []spanner.NullFloat64{{}}, []spanner.NullNumeric{{}},
[]spanner.NullString{{}}, [][]byte{[]byte(nil)}, []spanner.NullDate{{}}, []spanner.NullTime{{}},
[]spanner.NullJSON{{}},
},
)
if err != nil {
Expand Down
101 changes: 93 additions & 8 deletions driver_with_mockserver_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import (
"database/sql"
"database/sql/driver"
"encoding/base64"
"encoding/json"
"fmt"
"math/big"
"reflect"
Expand Down Expand Up @@ -323,14 +324,16 @@ func TestQueryWithAllTypes(t *testing.T) {
AND ColNumeric=@numeric
AND ColDate=@date
AND ColTimestamp=@timestamp
AND ColJson=@json
AND ColBoolArray=@boolArray
AND ColStringArray=@stringArray
AND ColBytesArray=@bytesArray
AND ColIntArray=@int64Array
AND ColFloatArray=@float64Array
AND ColNumericArray=@numericArray
AND ColDateArray=@dateArray
AND ColTimestampArray=@timestampArray`
AND ColTimestampArray=@timestampArray
AND ColJsonArray=@jsonArray`
_ = server.TestSpanner.PutStatementResult(
query,
&testutil.StatementResult{
Expand All @@ -357,6 +360,7 @@ func TestQueryWithAllTypes(t *testing.T) {
numeric("6.626"),
civil.Date{Year: 2021, Month: 7, Day: 21},
ts,
nullJson(true, `{"key":"value","other-key":["value1","value2"]}`),
[]spanner.NullBool{{Valid: true, Bool: true}, {}, {Valid: true, Bool: false}},
[]spanner.NullString{{Valid: true, StringVal: "test1"}, {}, {Valid: true, StringVal: "test2"}},
[][]byte{[]byte("testbytes1"), nil, []byte("testbytes2")},
Expand All @@ -365,6 +369,11 @@ func TestQueryWithAllTypes(t *testing.T) {
[]spanner.NullNumeric{nullNumeric(true, "3.14"), {}, nullNumeric(true, "10.01")},
[]spanner.NullDate{{Valid: true, Date: civil.Date{Year: 2000, Month: 2, Day: 29}}, {}, {Valid: true, Date: civil.Date{Year: 2021, Month: 7, Day: 27}}},
[]spanner.NullTime{{Valid: true, Time: ts1}, {}, {Valid: true, Time: ts2}},
[]spanner.NullJSON{
nullJson(true, `{"key1": "value1", "other-key1": ["value1", "value2"]}`),
nullJson(false, ""),
nullJson(true, `{"key2": "value2", "other-key2": ["value1", "value2"]}`),
},
)
if err != nil {
t.Fatal(err)
Expand All @@ -380,6 +389,7 @@ func TestQueryWithAllTypes(t *testing.T) {
var r spanner.NullNumeric
var d spanner.NullDate
var ts time.Time
var j spanner.NullJSON
var bArray []spanner.NullBool
var sArray []spanner.NullString
var btArray [][]byte
Expand All @@ -388,7 +398,8 @@ func TestQueryWithAllTypes(t *testing.T) {
var rArray []spanner.NullNumeric
var dArray []spanner.NullDate
var tsArray []spanner.NullTime
err = rows.Scan(&b, &s, &bt, &i, &f, &r, &d, &ts, &bArray, &sArray, &btArray, &iArray, &fArray, &rArray, &dArray, &tsArray)
var jArray []spanner.NullJSON
err = rows.Scan(&b, &s, &bt, &i, &f, &r, &d, &ts, &j, &bArray, &sArray, &btArray, &iArray, &fArray, &rArray, &dArray, &tsArray, &jArray)
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -416,6 +427,11 @@ func TestQueryWithAllTypes(t *testing.T) {
if g, w := ts, time.Date(2021, 7, 21, 21, 7, 59, 339911800, time.UTC); g != w {
t.Errorf("row value mismatch for timestamp\nGot: %v\nWant: %v", g, w)
}
if !runsOnEmulator() {
if g, w := j, nullJson(true, `{"key":"value","other-key":["value1","value2"]}`); !cmp.Equal(g, w) {
t.Errorf("row value mismatch for json\nGot: %v\nWant: %v", g, w)
}
}
if g, w := bArray, []spanner.NullBool{{Valid: true, Bool: true}, {}, {Valid: true, Bool: false}}; !cmp.Equal(g, w) {
t.Errorf("row value mismatch for bool array\nGot: %v\nWant: %v", g, w)
}
Expand All @@ -440,6 +456,15 @@ func TestQueryWithAllTypes(t *testing.T) {
if g, w := tsArray, []spanner.NullTime{{Valid: true, Time: ts1}, {}, {Valid: true, Time: ts2}}; !cmp.Equal(g, w) {
t.Errorf("row value mismatch for timestamp array\nGot: %v\nWant: %v", g, w)
}
if !runsOnEmulator() {
if g, w := jArray, []spanner.NullJSON{
nullJson(true, `{"key1": "value1", "other-key1": ["value1", "value2"]}`),
nullJson(false, ""),
nullJson(true, `{"key2": "value2", "other-key2": ["value1", "value2"]}`),
}; !cmp.Equal(g, w) {
t.Errorf("row value mismatch for json array\nGot: %v\nWant: %v", g, w)
}
}
}
if rows.Err() != nil {
t.Fatal(rows.Err())
Expand All @@ -450,10 +475,10 @@ func TestQueryWithAllTypes(t *testing.T) {
t.Fatalf("sql requests count mismatch\nGot: %v\nWant: %v", g, w)
}
req := sqlRequests[0].(*sppb.ExecuteSqlRequest)
if g, w := len(req.ParamTypes), 16; g != w {
if g, w := len(req.ParamTypes), 18; g != w {
t.Fatalf("param types length mismatch\nGot: %v\nWant: %v", g, w)
}
if g, w := len(req.Params.Fields), 16; g != w {
if g, w := len(req.Params.Fields), 18; g != w {
t.Fatalf("params length mismatch\nGot: %v\nWant: %v", g, w)
}
wantParams := []struct {
Expand Down Expand Up @@ -502,6 +527,11 @@ func TestQueryWithAllTypes(t *testing.T) {
code: sppb.TypeCode_TIMESTAMP,
value: "2021-07-22T10:26:17.123Z",
},
{
name: "json",
code: sppb.TypeCode_JSON,
value: `{"key":"value","other-key":["value1","value2"]}`,
},
{
name: "boolArray",
code: sppb.TypeCode_BOOL,
Expand Down Expand Up @@ -582,6 +612,16 @@ func TestQueryWithAllTypes(t *testing.T) {
{Kind: &structpb.Value_StringValue{StringValue: "2021-07-27T21:07:59.3399118Z"}},
}},
},
{
name: "jsonArray",
code: sppb.TypeCode_JSON,
array: true,
value: &structpb.ListValue{Values: []*structpb.Value{
{Kind: &structpb.Value_StringValue{StringValue: `{"key1":"value1","other-key1":["value1","value2"]}`}},
{Kind: &structpb.Value_NullValue{}},
{Kind: &structpb.Value_StringValue{StringValue: `{"key2":"value2","other-key2":["value1","value2"]}`}},
}},
},
}
for _, wantParam := range wantParams {
if pt, ok := req.ParamTypes[wantParam.name]; ok {
Expand Down Expand Up @@ -616,7 +656,7 @@ func TestQueryWithAllTypes(t *testing.T) {
}
if wantParam.array {
if !cmp.Equal(g, wantParam.value, cmpopts.IgnoreUnexported(structpb.ListValue{}, structpb.Value{})) {
t.Errorf("array param value mismatch\nGot: %v\nWant: %v", g, wantParam.value)
t.Errorf("array param value mismatch\nGot: %v\nWant: %v", g, wantParam.value)
}
} else {
if g != wantParam.value {
Expand Down Expand Up @@ -644,14 +684,16 @@ func TestQueryWithNullParameters(t *testing.T) {
AND ColNumeric=@numeric
AND ColDate=@date
AND ColTimestamp=@timestamp
AND ColJson=@json
AND ColBoolArray=@boolArray
AND ColStringArray=@stringArray
AND ColBytesArray=@bytesArray
AND ColIntArray=@int64Array
AND ColFloatArray=@float64Array
AND ColNumericArray=@numericArray
AND ColDateArray=@dateArray
AND ColTimestampArray=@timestampArray`
AND ColTimestampArray=@timestampArray
AND ColJsonArray=@jsonArray`
_ = server.TestSpanner.PutStatementResult(
query,
&testutil.StatementResult{
Expand All @@ -675,6 +717,7 @@ func TestQueryWithNullParameters(t *testing.T) {
nil, // numeric
nil, // date
nil, // timestamp
nil, // json
nil, // bool array
nil, // string array
nil, // bytes array
Expand All @@ -683,6 +726,7 @@ func TestQueryWithNullParameters(t *testing.T) {
nil, // numeric array
nil, // date array
nil, // timestamp array
nil, // json array
)
if err != nil {
t.Fatal(err)
Expand All @@ -698,6 +742,7 @@ func TestQueryWithNullParameters(t *testing.T) {
var r spanner.NullNumeric // There's no equivalent sql type.
var d spanner.NullDate // There's no equivalent sql type.
var ts sql.NullTime
var j spanner.NullJSON // There's no equivalent sql type.
var bArray []spanner.NullBool
var sArray []spanner.NullString
var btArray [][]byte
Expand All @@ -706,7 +751,8 @@ func TestQueryWithNullParameters(t *testing.T) {
var rArray []spanner.NullNumeric
var dArray []spanner.NullDate
var tsArray []spanner.NullTime
err = rows.Scan(&b, &s, &bt, &i, &f, &r, &d, &ts, &bArray, &sArray, &btArray, &iArray, &fArray, &rArray, &dArray, &tsArray)
var jArray []spanner.NullJSON
err = rows.Scan(&b, &s, &bt, &i, &f, &r, &d, &ts, &j, &bArray, &sArray, &btArray, &iArray, &fArray, &rArray, &dArray, &tsArray, &jArray)
if err != nil {
t.Fatal(err)
}
Expand Down Expand Up @@ -734,6 +780,36 @@ func TestQueryWithNullParameters(t *testing.T) {
if ts.Valid {
t.Errorf("row value mismatch for timestamp\nGot: %v\nWant: %v", ts, spanner.NullTime{})
}
if j.Valid {
t.Errorf("row value mismatch for json\nGot: %v\nWant: %v", j, spanner.NullJSON{})
}
if bArray != nil {
t.Errorf("row value mismatch for bool array\nGot: %v\nWant: %v", bArray, nil)
}
if sArray != nil {
t.Errorf("row value mismatch for string array\nGot: %v\nWant: %v", sArray, nil)
}
if btArray != nil {
t.Errorf("row value mismatch for bytes array array\nGot: %v\nWant: %v", btArray, nil)
}
if iArray != nil {
t.Errorf("row value mismatch for int64 array\nGot: %v\nWant: %v", iArray, nil)
}
if fArray != nil {
t.Errorf("row value mismatch for float64 array\nGot: %v\nWant: %v", fArray, nil)
}
if rArray != nil {
t.Errorf("row value mismatch for numeric array\nGot: %v\nWant: %v", rArray, nil)
}
if dArray != nil {
t.Errorf("row value mismatch for date array\nGot: %v\nWant: %v", dArray, nil)
}
if tsArray != nil {
t.Errorf("row value mismatch for timestamp array\nGot: %v\nWant: %v", tsArray, nil)
}
if jArray != nil {
t.Errorf("row value mismatch for json array\nGot: %v\nWant: %v", jArray, nil)
}
}
if rows.Err() != nil {
t.Fatal(rows.Err())
Expand All @@ -748,7 +824,7 @@ func TestQueryWithNullParameters(t *testing.T) {
if g, w := len(req.ParamTypes), 0; g != w {
t.Fatalf("param types length mismatch\nGot: %v\nWant: %v", g, w)
}
if g, w := len(req.Params.Fields), 16; g != w {
if g, w := len(req.Params.Fields), 18; g != w {
t.Fatalf("params length mismatch\nGot: %v\nWant: %v", g, w)
}
for _, param := range req.Params.Fields {
Expand Down Expand Up @@ -978,6 +1054,15 @@ func nullDate(valid bool, v string) spanner.NullDate {
return spanner.NullDate{Valid: true, Date: date(v)}
}

func nullJson(valid bool, v string) spanner.NullJSON {
if !valid {
return spanner.NullJSON{}
}
var m map[string]interface{}
_ = json.Unmarshal([]byte(v), &m)
return spanner.NullJSON{Valid: true, Value: m}
}

func setupTestDBConnection(t *testing.T) (db *sql.DB, server *testutil.MockedSpannerInMemTestServer, teardown func()) {
return setupTestDBConnectionWithParams(t, "")
}
Expand Down
12 changes: 6 additions & 6 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,12 @@ module github.com/cloudspannerecosystem/go-sql-spanner
go 1.14

require (
cloud.google.com/go v0.88.0
cloud.google.com/go/spanner v1.23.1-0.20210727075241-3d6c6c7873e1
cloud.google.com/go v0.93.3
cloud.google.com/go/spanner v1.25.0
github.com/golang/protobuf v1.5.2
github.com/google/go-cmp v0.5.6
google.golang.org/api v0.51.0
google.golang.org/genproto v0.0.0-20210726143408-b02e89920bf0
google.golang.org/grpc v1.39.0
google.golang.org/protobuf v1.27.1 // indirect
google.golang.org/api v0.54.0
google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8
google.golang.org/grpc v1.40.0
google.golang.org/protobuf v1.27.1
)
Loading