Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(bigquery): add support for explicit query parameter type #6596

Merged
merged 15 commits into from
Sep 16, 2022
Merged

feat(bigquery): add support for explicit query parameter type #6596

Show file tree
Hide file tree
Changes from 13 commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
b00a940
feat(bigquery): add support for explicit query parameter type
alvarowolfx Aug 31, 2022
5608b3e
fix(bigquery): separated explicit scalar tests
alvarowolfx Aug 31, 2022
550902c
feat(bigquery): add explicit query parameter types
alvarowolfx Sep 2, 2022
2393332
fix(bigquery): add missing struct comments
alvarowolfx Sep 2, 2022
b183f87
fix(bigquery): make struct query parameter schema be an array
alvarowolfx Sep 6, 2022
bb56b23
fix(bigquery): remove extra log on integration test
alvarowolfx Sep 6, 2022
37b80c2
fix(bigquery): use explicit parameter name if applies
alvarowolfx Sep 7, 2022
642fb38
feat(bigquery): remove QueryParameter method from explicit types
alvarowolfx Sep 7, 2022
b117528
feat(bigquery): add QueryParameterValue for explicit query param
alvarowolfx Sep 7, 2022
c0260bb
fix(bigquery): doc on explicit query param type
alvarowolfx Sep 7, 2022
459aab2
fix(bigquery): revert param test struct
alvarowolfx Sep 8, 2022
6ab0448
Merge branch 'main' into bq-explicit-query-param
alvarowolfx Sep 12, 2022
92bfba1
minor copy fixes
alvarowolfx Sep 13, 2022
5474411
docs(bigquery): add QueryParameterValue field docs
alvarowolfx Sep 15, 2022
a919b90
Merge branch 'main' into bq-explicit-query-param
alvarowolfx Sep 16, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
168 changes: 155 additions & 13 deletions bigquery/integration_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1807,6 +1807,7 @@ func TestIntegration_QueryParameters(t *testing.T) {
dtm := civil.DateTime{Date: d, Time: tm}
ts := time.Date(2016, 3, 20, 15, 04, 05, 0, time.UTC)
rat := big.NewRat(13, 10)
bigRat := big.NewRat(12345, 10e10)

type ss struct {
String string
Expand All @@ -1827,73 +1828,73 @@ func TestIntegration_QueryParameters(t *testing.T) {
}{
{
"SELECT @val",
[]QueryParameter{{"val", 1}},
[]QueryParameter{{Name: "val", Value: 1}},
[]Value{int64(1)},
int64(1),
},
{
"SELECT @val",
[]QueryParameter{{"val", 1.3}},
[]QueryParameter{{Name: "val", Value: 1.3}},
[]Value{1.3},
1.3,
},
{
"SELECT @val",
[]QueryParameter{{"val", rat}},
[]QueryParameter{{Name: "val", Value: rat}},
[]Value{rat},
rat,
},
{
"SELECT @val",
[]QueryParameter{{"val", true}},
[]QueryParameter{{Name: "val", Value: true}},
[]Value{true},
true,
},
{
"SELECT @val",
[]QueryParameter{{"val", "ABC"}},
[]QueryParameter{{Name: "val", Value: "ABC"}},
[]Value{"ABC"},
"ABC",
},
{
"SELECT @val",
[]QueryParameter{{"val", []byte("foo")}},
[]QueryParameter{{Name: "val", Value: []byte("foo")}},
[]Value{[]byte("foo")},
[]byte("foo"),
},
{
"SELECT @val",
[]QueryParameter{{"val", ts}},
[]QueryParameter{{Name: "val", Value: ts}},
[]Value{ts},
ts,
},
{
"SELECT @val",
[]QueryParameter{{"val", []time.Time{ts, ts}}},
[]QueryParameter{{Name: "val", Value: []time.Time{ts, ts}}},
[]Value{[]Value{ts, ts}},
[]interface{}{ts, ts},
},
{
"SELECT @val",
[]QueryParameter{{"val", dtm}},
[]QueryParameter{{Name: "val", Value: dtm}},
[]Value{civil.DateTime{Date: d, Time: rtm}},
civil.DateTime{Date: d, Time: rtm},
},
{
"SELECT @val",
[]QueryParameter{{"val", d}},
[]QueryParameter{{Name: "val", Value: d}},
[]Value{d},
d,
},
{
"SELECT @val",
[]QueryParameter{{"val", tm}},
[]QueryParameter{{Name: "val", Value: tm}},
[]Value{rtm},
rtm,
},
{
"SELECT @val",
[]QueryParameter{{"val", s{ts, []string{"a", "b"}, ss{"c"}, []ss{{"d"}, {"e"}}}}},
[]QueryParameter{{Name: "val", Value: s{ts, []string{"a", "b"}, ss{"c"}, []ss{{"d"}, {"e"}}}}},
[]Value{[]Value{ts, []Value{"a", "b"}, []Value{"c"}, []Value{[]Value{"d"}, []Value{"e"}}}},
map[string]interface{}{
"Timestamp": ts,
Expand All @@ -1907,7 +1908,7 @@ func TestIntegration_QueryParameters(t *testing.T) {
},
{
"SELECT @val.Timestamp, @val.SubStruct.String",
[]QueryParameter{{"val", s{Timestamp: ts, SubStruct: ss{"a"}}}},
[]QueryParameter{{Name: "val", Value: s{Timestamp: ts, SubStruct: ss{"a"}}}},
[]Value{ts, "a"},
map[string]interface{}{
"Timestamp": ts,
Expand All @@ -1916,6 +1917,147 @@ func TestIntegration_QueryParameters(t *testing.T) {
"SubStructArray": nil,
},
},
{
"SELECT @val",
[]QueryParameter{
{
Name: "val",
Value: &QueryParameterValue{
Type: StandardSQLDataType{
TypeKind: "BIGNUMERIC",
},
Value: BigNumericString(bigRat),
},
},
},
[]Value{bigRat},
bigRat,
},
{
"SELECT @val",
[]QueryParameter{
{
Name: "val",
Value: &QueryParameterValue{
ArrayValue: []QueryParameterValue{
{Value: "a"},
{Value: "b"},
},
Type: StandardSQLDataType{
ArrayElementType: &StandardSQLDataType{
TypeKind: "STRING",
},
},
},
},
},
[]Value{[]Value{"a", "b"}},
[]interface{}{"a", "b"},
},
{
"SELECT @val",
alvarowolfx marked this conversation as resolved.
Show resolved Hide resolved
[]QueryParameter{
{
Name: "val",
Value: &QueryParameterValue{
StructValue: map[string]QueryParameterValue{
"Timestamp": {
Value: ts,
},
"BigNumericArray": {
ArrayValue: []QueryParameterValue{
{Value: BigNumericString(bigRat)},
{Value: BigNumericString(rat)},
},
},
"ArraySingleValueStruct": {
ArrayValue: []QueryParameterValue{
{StructValue: map[string]QueryParameterValue{
"Number": {
Value: int64(42),
},
}},
{StructValue: map[string]QueryParameterValue{
"Number": {
Value: int64(43),
},
}},
},
},
"SubStruct": {
StructValue: map[string]QueryParameterValue{
"String": {
Value: "c",
},
},
},
},
Type: StandardSQLDataType{
StructType: &StandardSQLStructType{
Fields: []*StandardSQLField{
{
Name: "Timestamp",
Type: &StandardSQLDataType{
TypeKind: "TIMESTAMP",
},
},
{
Name: "BigNumericArray",
Type: &StandardSQLDataType{
ArrayElementType: &StandardSQLDataType{
TypeKind: "BIGNUMERIC",
},
},
},
{
Name: "ArraySingleValueStruct",
Type: &StandardSQLDataType{
ArrayElementType: &StandardSQLDataType{
StructType: &StandardSQLStructType{
Fields: []*StandardSQLField{
{
Name: "Number",
Type: &StandardSQLDataType{
TypeKind: "INT64",
},
},
},
},
},
},
},
{
Name: "SubStruct",
Type: &StandardSQLDataType{
StructType: &StandardSQLStructType{
Fields: []*StandardSQLField{
{
Name: "String",
Type: &StandardSQLDataType{
TypeKind: "STRING",
},
},
},
},
},
},
},
},
},
},
},
},
[]Value{[]Value{ts, []Value{bigRat, rat}, []Value{[]Value{int64(42)}, []Value{int64(43)}}, []Value{"c"}}},
map[string]interface{}{
"Timestamp": ts,
"BigNumericArray": []interface{}{bigRat, rat},
"ArraySingleValueStruct": []interface{}{
map[string]interface{}{"Number": int64(42)},
map[string]interface{}{"Number": int64(43)},
},
"SubStruct": map[string]interface{}{"String": "c"},
},
},
}
for _, c := range testCases {
q := client.Query(c.query)
Expand Down
81 changes: 70 additions & 11 deletions bigquery/params.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,12 +82,13 @@ var (
)

var (
typeOfDate = reflect.TypeOf(civil.Date{})
typeOfTime = reflect.TypeOf(civil.Time{})
typeOfDateTime = reflect.TypeOf(civil.DateTime{})
typeOfGoTime = reflect.TypeOf(time.Time{})
typeOfRat = reflect.TypeOf(&big.Rat{})
typeOfIntervalValue = reflect.TypeOf(&IntervalValue{})
typeOfDate = reflect.TypeOf(civil.Date{})
typeOfTime = reflect.TypeOf(civil.Time{})
typeOfDateTime = reflect.TypeOf(civil.DateTime{})
typeOfGoTime = reflect.TypeOf(time.Time{})
typeOfRat = reflect.TypeOf(&big.Rat{})
typeOfIntervalValue = reflect.TypeOf(&IntervalValue{})
typeOfQueryParameterValue = reflect.TypeOf(&QueryParameterValue{})
)

// A QueryParameter is a parameter to a query.
Expand Down Expand Up @@ -116,6 +117,15 @@ type QueryParameter struct {
// For scalar values, you can supply the Null types within this library
// to send the appropriate NULL values (e.g. NullInt64, NullString, etc).
//
// To specify query parameters explicitly rather by inference, *QueryParameterValue can be used.
// For example, a BIGNUMERIC can be specified like this:
// &QueryParameterValue{
// Type: StandardSQLDataType{
// TypeKind: "BIGNUMERIC",
// },
// Value: BigNumericString(*big.Rat),
// }
//
// When a QueryParameter is returned inside a QueryConfig from a call to
// Job.Config:
// Integers are of type int64.
Expand All @@ -129,12 +139,57 @@ type QueryParameter struct {
Value interface{}
}

func (p QueryParameter) toBQ() (*bq.QueryParameter, error) {
// QueryParameterValue is a go type for representing a explicit typed BigQuery Query Parameter.
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm still on the fence about the name due to the collision with the API, but it's a reasonable name otherwise.

Should we document the behavior for nested types as part of the type description? Specifically that you need to fully specify types and values for complex params?

type QueryParameterValue struct {
Type StandardSQLDataType
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Document these fields as appropriate.

ArrayValue []QueryParameterValue
StructValue map[string]QueryParameterValue
Value interface{}
}

func (p QueryParameterValue) toBQParamType() *bq.QueryParameterType {
return p.Type.toBQParamType()
}

func (p QueryParameterValue) toBQParamValue() (*bq.QueryParameterValue, error) {
if len(p.ArrayValue) > 0 {
pv := &bq.QueryParameterValue{}
pv.ArrayValues = []*bq.QueryParameterValue{}
for _, v := range p.ArrayValue {
val, err := v.toBQParamValue()
if err != nil {
return nil, err
}
pv.ArrayValues = append(pv.ArrayValues, val)
}
return pv, nil
}
if len(p.StructValue) > 0 {
pv := &bq.QueryParameterValue{}
pv.StructValues = map[string]bq.QueryParameterValue{}
for name, param := range p.StructValue {
v, err := param.toBQParamValue()
if err != nil {
return nil, err
}
pv.StructValues[name] = *v
}
return pv, nil
}
pv, err := paramValue(reflect.ValueOf(p.Value))
if err != nil {
return nil, err
}
pt, err := paramType(reflect.TypeOf(p.Value))
return pv, nil
}

func (p QueryParameter) toBQ() (*bq.QueryParameter, error) {
v := reflect.ValueOf(p.Value)
pv, err := paramValue(v)
if err != nil {
return nil, err
}
pt, err := paramType(reflect.TypeOf(p.Value), v)
if err != nil {
return nil, err
}
Expand All @@ -145,7 +200,7 @@ func (p QueryParameter) toBQ() (*bq.QueryParameter, error) {
}, nil
}

func paramType(t reflect.Type) (*bq.QueryParameterType, error) {
func paramType(t reflect.Type, v reflect.Value) (*bq.QueryParameterType, error) {
if t == nil {
return nil, errors.New("bigquery: nil parameter")
}
Expand Down Expand Up @@ -174,6 +229,8 @@ func paramType(t reflect.Type) (*bq.QueryParameterType, error) {
return geographyParamType, nil
case typeOfNullJSON:
return jsonParamType, nil
case typeOfQueryParameterValue:
return v.Interface().(*QueryParameterValue).toBQParamType(), nil
}
switch t.Kind() {
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint8, reflect.Uint16, reflect.Uint32:
Expand All @@ -195,7 +252,7 @@ func paramType(t reflect.Type) (*bq.QueryParameterType, error) {
fallthrough

case reflect.Array:
et, err := paramType(t.Elem())
et, err := paramType(t.Elem(), v)
if err != nil {
return nil, err
}
Expand All @@ -215,7 +272,7 @@ func paramType(t reflect.Type) (*bq.QueryParameterType, error) {
return nil, err
}
for _, f := range fields {
pt, err := paramType(f.Type)
pt, err := paramType(f.Type, v)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -314,6 +371,8 @@ func paramValue(v reflect.Value) (*bq.QueryParameterValue, error) {
case typeOfIntervalValue:
res.Value = IntervalString(v.Interface().(*IntervalValue))
return res, nil
case typeOfQueryParameterValue:
return v.Interface().(*QueryParameterValue).toBQParamValue()
}
switch t.Kind() {
case reflect.Slice:
Expand Down
Loading