Update proto.ColumnDefinition (part of GetSchemaResponse) to include …

…column default and hydrate func. Closes #711

grpc/proto/column.go

@@ -0,0 +1,33 @@
+package proto
+
+import (
+	"encoding/json"
+	"github.com/golang/protobuf/ptypes"
+	"google.golang.org/protobuf/reflect/protoreflect"
+)
+
+func (x *Column) ValueToInterface() (any, error) {
+	// extract x value as interface from protobuf message
+	var val any
+	if bytes := x.GetJsonValue(); bytes != nil {
+		if err := json.Unmarshal(bytes, &val); err != nil {
+			return nil, err
+		}
+	} else if timestamp := x.GetTimestampValue(); timestamp != nil {
+		// convert from protobuf timestamp to a RFC 3339 time string
+		val = ptypes.TimestampString(timestamp)
+	} else {
+		// get the first field descriptor and value (we only expect x message to contain a single field
+		x.ProtoReflect().Range(func(descriptor protoreflect.FieldDescriptor, v protoreflect.Value) bool {
+			// is this value null?
+			if descriptor.JSONName() == "nullValue" {
+				val = nil
+			} else {
+				val = v.Interface()
+			}
+			return false
+		})
+	}
+
+	return val, nil
+}

grpc/proto/plugin.proto

@@ -262,6 +262,8 @@ message ColumnDefinition {
   string name = 1;
   ColumnType type = 2;
   string description = 3;
+  string hydrate = 4;
+  Column default = 5;
 }
 
 enum ColumnType {

grpc/proto/table_schema.go

@@ -11,6 +11,22 @@ func (x *TableSchema) GetColumnMap() map[string]*ColumnDefinition {
 	return nil
 }
 
+func (x *TableSchema) GetKeyColumnMap() map[string]*KeyColumn {
+	var res = make(map[string]*KeyColumn)
+	for _, k := range x.GetCallKeyColumnList {
+		res[k.Name] = k
+	}
+	return res
+}
+
+func (x *TableSchema) ListKeyColumnMap() map[string]*KeyColumn {
+	var res = make(map[string]*KeyColumn)
+	for _, k := range x.ListCallKeyColumnList {
+		res[k.Name] = k
+	}
+	return res
+}
+
 func (x *TableSchema) Equals(other *TableSchema) bool {
 	return !x.Diff(other).HasDiffs()
 }

plugin/column.go

@@ -1,8 +1,15 @@
 package plugin
 
 import (
+	"encoding/json"
+	"fmt"
+	"github.com/golang/protobuf/ptypes"
+	"github.com/turbot/go-kit/helpers"
+	"github.com/turbot/go-kit/types"
 	"github.com/turbot/steampipe-plugin-sdk/v5/grpc/proto"
 	"github.com/turbot/steampipe-plugin-sdk/v5/plugin/transform"
+	"log"
+	"net"
 )
 
 /*
@@ -84,6 +91,131 @@ type QueryColumn struct {
 	hydrateName string
 }
 
+// ToColumnValue converts a value of unknown type to a valid protobuf column value.type
+func (c Column) ToColumnValue(val any) (*proto.Column, error) {
+	defer func() {
+		if r := recover(); r != nil {
+			panic(fmt.Errorf("%s: %v", c.Name, r))
+		}
+	}()
+
+	// if the value is a pointer, get its value and use that
+	val = helpers.DereferencePointer(val)
+	if val == nil {
+		if c.Default != nil {
+			val = c.Default
+		} else {
+			// return nil
+			return &proto.Column{Value: &proto.Column_NullValue{}}, nil
+		}
+	}
+
+	var columnValue *proto.Column
+
+	switch c.Type {
+	case proto.ColumnType_STRING:
+		columnValue = &proto.Column{Value: &proto.Column_StringValue{StringValue: types.ToString(val)}}
+		break
+	case proto.ColumnType_BOOL:
+		b, err := types.ToBool(val)
+		if err != nil {
+			return nil, fmt.Errorf("interfaceToColumnValue failed for column '%s': %v", c.Name, err)
+		}
+		columnValue = &proto.Column{Value: &proto.Column_BoolValue{BoolValue: b}}
+		break
+	case proto.ColumnType_INT:
+		i, err := types.ToInt64(val)
+		if err != nil {
+			return nil, fmt.Errorf("interfaceToColumnValue failed for column '%s': %v", c.Name, err)
+		}
+
+		columnValue = &proto.Column{Value: &proto.Column_IntValue{IntValue: i}}
+		break
+	case proto.ColumnType_DOUBLE:
+		d, err := types.ToFloat64(val)
+		if err != nil {
+			return nil, fmt.Errorf("interfaceToColumnValue failed for column '%s': %v", c.Name, err)
+		}
+		columnValue = &proto.Column{Value: &proto.Column_DoubleValue{DoubleValue: d}}
+		break
+	case proto.ColumnType_JSON:
+		strValue, ok := val.(string)
+		if ok {
+			// NOTE: Strings are assumed to be raw JSON, so are passed through directly.
+			// This is the most common case, but means it's currently impossible to
+			// pass through a string and have it marshalled to be a JSON representation
+			// of a string.
+			columnValue = &proto.Column{Value: &proto.Column_JsonValue{JsonValue: []byte(strValue)}}
+		} else {
+			res, err := json.Marshal(val)
+			if err != nil {
+				log.Printf("[ERROR] failed to marshal value to json: %v\n", err)
+				return nil, fmt.Errorf("%s: %v ", c.Name, err)
+			}
+			columnValue = &proto.Column{Value: &proto.Column_JsonValue{JsonValue: res}}
+		}
+	case proto.ColumnType_DATETIME, proto.ColumnType_TIMESTAMP:
+		// cast val to time
+		var timeVal, err = types.ToTime(val)
+		if err != nil {
+			return nil, fmt.Errorf("interfaceToColumnValue failed for column '%s': %v", c.Name, err)
+		}
+		// now convert time to protobuf timestamp
+		timestamp, err := ptypes.TimestampProto(timeVal)
+		if err != nil {
+			return nil, fmt.Errorf("interfaceToColumnValue failed for column '%s': %v", c.Name, err)
+		}
+		columnValue = &proto.Column{Value: &proto.Column_TimestampValue{TimestampValue: timestamp}}
+		break
+	case proto.ColumnType_IPADDR:
+		ipString := types.SafeString(val)
+		// treat an empty string as a null ip address
+		if ipString == "" {
+			columnValue = &proto.Column{Value: &proto.Column_NullValue{}}
+		} else {
+			if ip := net.ParseIP(ipString); ip == nil {
+				return nil, fmt.Errorf("%s: invalid ip address %s", c.Name, ipString)
+			}
+			columnValue = &proto.Column{Value: &proto.Column_IpAddrValue{IpAddrValue: ipString}}
+		}
+		break
+	case proto.ColumnType_CIDR:
+		cidrRangeString := types.SafeString(val)
+		// treat an empty string as a null ip address
+		if cidrRangeString == "" {
+			columnValue = &proto.Column{Value: &proto.Column_NullValue{}}
+		} else {
+			if _, _, err := net.ParseCIDR(cidrRangeString); err != nil {
+				return nil, fmt.Errorf("%s: invalid ip address %s", c.Name, cidrRangeString)
+			}
+			columnValue = &proto.Column{Value: &proto.Column_CidrRangeValue{CidrRangeValue: cidrRangeString}}
+		}
+		break
+	case proto.ColumnType_INET:
+		inetString := types.SafeString(val)
+		// treat an empty string as a null ip address
+		if inetString == "" {
+			columnValue = &proto.Column{Value: &proto.Column_NullValue{}}
+		} else {
+			if ip := net.ParseIP(inetString); ip == nil {
+				if _, _, err := net.ParseCIDR(inetString); err != nil {
+					return nil, fmt.Errorf("%s: invalid ip address %s", c.Name, inetString)
+				}
+			}
+			columnValue = &proto.Column{Value: &proto.Column_CidrRangeValue{CidrRangeValue: inetString}}
+		}
+	case proto.ColumnType_LTREE:
+		columnValue = &proto.Column{Value: &proto.Column_LtreeValue{LtreeValue: types.ToString(val)}}
+		break
+
+	default:
+		return nil, fmt.Errorf("unrecognised columnValue type '%s'", c.Type)
+	}
+
+	return columnValue, nil
+
+}
+
 func NewQueryColumn(column *Column, hydrateName string) *QueryColumn {
 	return &QueryColumn{column, hydrateName}
 }

plugin/table_column.go

@@ -2,16 +2,11 @@ package plugin
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
-	"log"
-	"net"
-
-	"github.com/golang/protobuf/ptypes"
 	"github.com/turbot/go-kit/helpers"
-	"github.com/turbot/go-kit/types"
 	"github.com/turbot/steampipe-plugin-sdk/v5/grpc/proto"
 	"github.com/turbot/steampipe-plugin-sdk/v5/plugin/transform"
+	"log"
 )
 
 // get the column object with the given name
@@ -61,7 +56,8 @@ func (t *Table) getColumnValue(ctx context.Context, rowData *rowData, column *Qu
 		}
 	}
 	// now convert the value to a protobuf column value
-	c, err := t.interfaceToColumnValue(column, value)
+	c, err := column.ToColumnValue(value)
+	//c, err := proto.InterfaceToColumnValue(column.Column.Name, column.Column.Type, column.Column.Default, value)
 	return c, err
 }
 
@@ -90,128 +86,3 @@ func (t *Table) getDefaultColumnTransform(column *QueryColumn) *transform.Column
 	}
 	return columnTransform
 }
-
-// convert a value of unknown type to a valid protobuf column value.type
-func (t *Table) interfaceToColumnValue(column *QueryColumn, val interface{}) (*proto.Column, error) {
-	defer func() {
-		if r := recover(); r != nil {
-			panic(fmt.Errorf("%s: %v", column.Name, r))
-		}
-	}()
-
-	// if the value is a pointer, get its value and use that
-	val = helpers.DereferencePointer(val)
-	if val == nil {
-		if column.Default != nil {
-			val = column.Default
-		} else {
-			// return nil
-			return &proto.Column{Value: &proto.Column_NullValue{}}, nil
-		}
-	}
-
-	var columnValue *proto.Column
-
-	switch column.Type {
-	case proto.ColumnType_STRING:
-		columnValue = &proto.Column{Value: &proto.Column_StringValue{StringValue: types.ToString(val)}}
-		break
-	case proto.ColumnType_BOOL:
-		b, err := types.ToBool(val)
-		if err != nil {
-			return nil, fmt.Errorf("interfaceToColumnValue failed for column '%s': %v", column.Name, err)
-		}
-		columnValue = &proto.Column{Value: &proto.Column_BoolValue{BoolValue: b}}
-		break
-	case proto.ColumnType_INT:
-		i, err := types.ToInt64(val)
-		if err != nil {
-			return nil, fmt.Errorf("interfaceToColumnValue failed for column '%s': %v", column.Name, err)
-		}
-
-		columnValue = &proto.Column{Value: &proto.Column_IntValue{IntValue: i}}
-		break
-	case proto.ColumnType_DOUBLE:
-		d, err := types.ToFloat64(val)
-		if err != nil {
-			return nil, fmt.Errorf("interfaceToColumnValue failed for column '%s': %v", column.Name, err)
-		}
-		columnValue = &proto.Column{Value: &proto.Column_DoubleValue{DoubleValue: d}}
-		break
-	case proto.ColumnType_JSON:
-		strValue, ok := val.(string)
-		if ok {
-			// NOTE: Strings are assumed to be raw JSON, so are passed through directly.
-			// This is the most common case, but means it's currently impossible to
-			// pass through a string and have it marshalled to be a JSON representation
-			// of a string.
-			columnValue = &proto.Column{Value: &proto.Column_JsonValue{JsonValue: []byte(strValue)}}
-		} else {
-			res, err := json.Marshal(val)
-			if err != nil {
-				log.Printf("[ERROR] failed to marshal value to json: %v\n", err)
-				return nil, fmt.Errorf("%s: %v ", column.Name, err)
-			}
-			columnValue = &proto.Column{Value: &proto.Column_JsonValue{JsonValue: res}}
-		}
-	case proto.ColumnType_DATETIME, proto.ColumnType_TIMESTAMP:
-		// cast val to time
-		var timeVal, err = types.ToTime(val)
-		if err != nil {
-			return nil, fmt.Errorf("interfaceToColumnValue failed for column '%s': %v", column.Name, err)
-		}
-		// now convert time to protobuf timestamp
-		timestamp, err := ptypes.TimestampProto(timeVal)
-		if err != nil {
-			return nil, fmt.Errorf("interfaceToColumnValue failed for column '%s': %v", column.Name, err)
-		}
-		columnValue = &proto.Column{Value: &proto.Column_TimestampValue{TimestampValue: timestamp}}
-		break
-	case proto.ColumnType_IPADDR:
-		ipString := types.SafeString(val)
-		// treat an empty string as a null ip address
-		if ipString == "" {
-			columnValue = &proto.Column{Value: &proto.Column_NullValue{}}
-		} else {
-			if ip := net.ParseIP(ipString); ip == nil {
-				return nil, fmt.Errorf("%s: invalid ip address %s", column.Name, ipString)
-			}
-			columnValue = &proto.Column{Value: &proto.Column_IpAddrValue{IpAddrValue: ipString}}
-		}
-		break
-	case proto.ColumnType_CIDR:
-		cidrRangeString := types.SafeString(val)
-		// treat an empty string as a null ip address
-		if cidrRangeString == "" {
-			columnValue = &proto.Column{Value: &proto.Column_NullValue{}}
-		} else {
-			if _, _, err := net.ParseCIDR(cidrRangeString); err != nil {
-				return nil, fmt.Errorf("%s: invalid ip address %s", column.Name, cidrRangeString)
-			}
-			columnValue = &proto.Column{Value: &proto.Column_CidrRangeValue{CidrRangeValue: cidrRangeString}}
-		}
-		break
-	case proto.ColumnType_INET:
-		inetString := types.SafeString(val)
-		// treat an empty string as a null ip address
-		if inetString == "" {
-			columnValue = &proto.Column{Value: &proto.Column_NullValue{}}
-		} else {
-			if ip := net.ParseIP(inetString); ip == nil {
-				if _, _, err := net.ParseCIDR(inetString); err != nil {
-					return nil, fmt.Errorf("%s: invalid ip address %s", column.Name, inetString)
-				}
-			}
-			columnValue = &proto.Column{Value: &proto.Column_CidrRangeValue{CidrRangeValue: inetString}}
-		}
-	case proto.ColumnType_LTREE:
-		columnValue = &proto.Column{Value: &proto.Column_LtreeValue{LtreeValue: types.ToString(val)}}
-		break
-
-	default:
-		return nil, fmt.Errorf("unrecognised columnValue type '%s'", column.Type)
-	}
-
-	return columnValue, nil
-
-}

plugin/table_schema.go

@@ -1,6 +1,7 @@
 package plugin
 
 import (
+	"github.com/turbot/go-kit/helpers"
 	"github.com/turbot/steampipe-plugin-sdk/v5/grpc/proto"
 	"strings"
 )
@@ -41,6 +42,20 @@ func (t *Table) GetSchema() (*proto.TableSchema, error) {
 				Type:        column.Type,
 				Description: column.Description,
 			}
+			if column.Hydrate != nil {
+				name, _ := column.Hydrate.getOriginalFuncName()
+				columnDef.Hydrate = name
+			}
+			if !helpers.IsNil(column.Default) {
+				// to convert the column default to a proto.Column, call ToColumnValue with a nil value
+				// - we will get the default
+				def, err := column.ToColumnValue(nil)
+				if err != nil {
+					return nil, err
+				}
+				columnDef.Default = def
+
+			}
 			schema.Columns = append(schema.Columns, columnDef)
 		}
 	}