Key columns should allow specifying supported operators. Closes #121

- deprecated KeyColumnSet
- can now specify key columns using an array of KeyColumn objects, which have an 'operators' and 'require' property

go.mod

@@ -7,23 +7,16 @@ require (
 	github.com/gertd/go-pluralize v0.1.7
 	github.com/ghodss/yaml v1.0.0
 	github.com/golang/protobuf v1.4.3
-	github.com/hashicorp/go-hclog v0.14.1
-	github.com/hashicorp/go-plugin v1.3.0
+	github.com/hashicorp/go-hclog v0.15.0
+	github.com/hashicorp/go-plugin v1.4.1
 	github.com/hashicorp/go-version v1.2.1
-	github.com/hashicorp/hcl/v2 v2.8.2
+	github.com/hashicorp/hcl/v2 v2.9.1
 	github.com/iancoleman/strcase v0.1.2
-	github.com/mattn/go-runewidth v0.0.9 // indirect
-	github.com/mitchellh/go-testing-interface v1.0.0 // indirect
 	github.com/olekukonko/tablewriter v0.0.4
 	github.com/sethvargo/go-retry v0.1.0
 	github.com/stevenle/topsort v0.0.0-20130922064739-8130c1d7596b
-	github.com/turbot/go-kit v0.1.1
-	github.com/zclconf/go-cty v1.7.1
-	golang.org/x/net v0.0.0-20201029221708-28c70e62bb1d // indirect
-	golang.org/x/sys v0.0.0-20201029080932-201ba4db2418 // indirect
-	golang.org/x/text v0.3.4 // indirect
-	google.golang.org/genproto v0.0.0-20201029200359-8ce4113da6f7 // indirect
+	github.com/turbot/go-kit v0.2.2-0.20210628165333-268ba0a30be3
+	github.com/zclconf/go-cty v1.8.2
 	google.golang.org/grpc v1.33.1
 	google.golang.org/protobuf v1.25.0
-	gopkg.in/yaml.v2 v2.2.8 // indirect
 )

grpc/proto/plugin.proto

@@ -99,17 +99,29 @@ message TableSchema
 {
   repeated ColumnDefinition columns = 1;
   string description = 2;
-  KeyColumnsSet getCallKeyColumns = 3;
-  KeyColumnsSet listCallKeyColumns = 4;
-  KeyColumnsSet listCallOptionalKeyColumns = 5;
+  KeyColumnsSet getCallKeyColumns = 3 [deprecated=true];
+  KeyColumnsSet listCallKeyColumns = 4 [deprecated=true];
+  KeyColumnsSet listCallOptionalKeyColumns = 5 [deprecated=true];
+
+  repeated KeyColumn getCallKeyColumnList = 6;
+  repeated KeyColumn listCallKeyColumnList = 7;
 }
 
-// a set of Key Columns, all of which are reuired
+// a set of Key Columns, required for get/list calls
+// deprecated - kept for compatibility
 message KeyColumnsSet
 {
-   string single = 1;
-   repeated string all = 2;
-   repeated string any = 3;
+  option deprecated = true;
+  string single = 1;
+  repeated string all = 2;
+  repeated string any = 3;
+}
+
+message KeyColumn
+{
+  string name = 1;
+  repeated string operators = 2;
+  string require=3;
 }
 
 message Schema {

plugin/key_column.go

@@ -0,0 +1,68 @@
+package plugin
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/gertd/go-pluralize"
+	"github.com/turbot/go-kit/helpers"
+	"github.com/turbot/steampipe-plugin-sdk/grpc/proto"
+)
+
+const (
+	Required = "required"
+	Optional = "optional"
+	AnyOf    = "any_of"
+)
+
+// KeyColumn is a struct representing the definition of a KeyColumn used to filter and Get/List call
+type KeyColumn struct {
+	Name      string
+	Operators []string
+	Require   string
+}
+
+func (k KeyColumn) String() string {
+	return fmt.Sprintf("column:'%s' %s: %s", k.Name, pluralize.NewClient().Pluralize("operator", len(k.Operators), false), strings.Join(k.Operators, ","))
+}
+
+// ToProtobuf converts the KeyColumn to a protobuf object
+func (k *KeyColumn) ToProtobuf() *proto.KeyColumn {
+	return &proto.KeyColumn{
+		Name:      k.Name,
+		Operators: k.Operators,
+		Require:   k.Require,
+	}
+}
+
+// SingleEqualsQual returns whether this key column has a single = operator
+func (k *KeyColumn) SingleEqualsQual() bool {
+	return len(k.Operators) == 1 && k.Operators[0] == "="
+}
+
+func (k *KeyColumn) Validate() []string {
+	// ensure operators are valid
+
+	// map "!=" operator to "<>"
+	validOperators := []string{"=", "<>", "<", "<=", ">", ">="}
+	validRequire := []string{Required, Optional, AnyOf}
+	var res []string
+
+	for _, op := range k.Operators {
+		// convert "!=" to "<>"
+		if op == "!=" {
+			op = "<>"
+		}
+		if !helpers.StringSliceContains(validOperators, op) {
+			res = append(res, fmt.Sprintf("operator %s is not valid, it must be one of: %s", op, strings.Join(validOperators, ",")))
+		}
+	}
+	// default Require to Required
+	if k.Require == "" {
+		k.Require = Required
+	}
+	if !helpers.StringSliceContains(validRequire, k.Require) {
+		res = append(res, fmt.Sprintf("Require value '%s' is not valid, it must be one of: %s", k.Require, strings.Join(validRequire, ",")))
+	}
+	return res
+}

plugin/key_column_qual.go

@@ -0,0 +1,29 @@
+package plugin
+
+import (
+	"github.com/turbot/steampipe-plugin-sdk/plugin/quals"
+
+	"github.com/turbot/go-kit/helpers"
+)
+
+// KeyColumnQuals is a struct representing all quals for a specific column
+type KeyColumnQuals struct {
+	Name  string
+	Quals quals.QualSlice
+}
+
+func (k KeyColumnQuals) SatisfiesKeyColumn(keyColumn *KeyColumn) bool {
+	if keyColumn.Name != k.Name {
+		return false
+	}
+	for _, q := range k.Quals {
+		if helpers.StringSliceContains(keyColumn.Operators, q.Operator) {
+			return true
+		}
+	}
+	return false
+}
+
+func (k KeyColumnQuals) SingleEqualsQual() bool {
+	return k.Quals.SingleEqualsQual()
+}

plugin/key_column_qual_map.go

@@ -0,0 +1,143 @@
+package plugin
+
+import (
+	"fmt"
+	"log"
+	"strings"
+
+	"github.com/turbot/go-kit/helpers"
+	"github.com/turbot/steampipe-plugin-sdk/grpc/proto"
+	"github.com/turbot/steampipe-plugin-sdk/plugin/quals"
+)
+
+// KeyColumnQualMap is a map of KeyColumnQuals keyed by column name
+type KeyColumnQualMap map[string]*KeyColumnQuals
+
+// ToEqualsQualValueMap converts a KeyColumnQualMap to a column-qual value map, including only the
+func (m KeyColumnQualMap) ToEqualsQualValueMap() map[string]*proto.QualValue {
+	res := make(map[string]*proto.QualValue, len(m))
+	for k, v := range m {
+		if v.SingleEqualsQual() {
+			res[k] = v.Quals[0].Value
+		}
+	}
+	return res
+}
+
+func (m KeyColumnQualMap) String() string {
+	strs := make([]string, len(m))
+	for _, k := range m {
+		var values = make([]interface{}, len(k.Quals))
+		for i, v := range k.Quals {
+			values[i] = v.Value
+		}
+		strs = append(strs, fmt.Sprintf("%s - %v", k.Name, values))
+	}
+	return strings.Join(strs, "\n")
+}
+
+func (m KeyColumnQualMap) SatisfiesKeyColumns(columns KeyColumnSlice) (bool, KeyColumnSlice) {
+	log.Printf("[WARN] SatisfiesKeyColumns %v", columns)
+
+	if columns == nil {
+		return true, nil
+	}
+	var unsatisfiedKeyColumns KeyColumnSlice
+	satisfiedCount := map[string]int{
+		Required: 0,
+		AnyOf:    0,
+		Optional: 0,
+	}
+	unsatisfiedCount := map[string]int{
+		Required: 0,
+		AnyOf:    0,
+		Optional: 0,
+	}
+
+	for _, keyColumn := range columns {
+		// look for this key column in our map
+		k := m[keyColumn.Name]
+		satisfied := k != nil && k.SatisfiesKeyColumn(keyColumn)
+		if satisfied {
+			satisfiedCount[keyColumn.Require]++
+
+			log.Printf("[TRACE] key column satisfied %v", keyColumn)
+
+		} else {
+			unsatisfiedCount[keyColumn.Require]++
+			unsatisfiedKeyColumns = append(unsatisfiedKeyColumns, keyColumn)
+			log.Printf("[TRACE] key column NOT satisfied %v", keyColumn)
+			// if this was NOT an optional key column, we are not satisfied
+		}
+	}
+
+	// we are satisfied if:
+	// all Required key columns are satisfied
+	// either there is at least 1 satisfied AnyOf key columns, or there are no AnyOf columns
+	res := unsatisfiedCount[Required] == 0 && (satisfiedCount[AnyOf] > 0 || unsatisfiedCount[AnyOf] == 0)
+
+	log.Printf("[WARN] SatisfiesKeyColumns result: %v, satisfiedCount %v, unsatisfiedCount %v, unsatisfiedKeyColumns %v", res, satisfiedCount, unsatisfiedCount, unsatisfiedKeyColumns)
+	return res, unsatisfiedKeyColumns
+}
+
+// ToQualMap converts the map into a simpler map of column to []Quals
+// this is needed to avoid the transform package needing to reference plugin
+func (m KeyColumnQualMap) ToQualMap() map[string]quals.QualSlice {
+	var res = make(map[string]quals.QualSlice)
+	for k, v := range m {
+		res[k] = v.Quals
+	}
+	return res
+}
+
+// NewKeyColumnQualValueMap creates a KeyColumnQualMap from a qual map and a KeyColumnSlice
+func NewKeyColumnQualValueMap(qualMap map[string]*proto.Quals, keyColumns KeyColumnSlice) KeyColumnQualMap {
+	res := KeyColumnQualMap{}
+
+	for _, col := range keyColumns {
+		matchingQuals := getMatchingQuals(col, qualMap)
+		for _, q := range matchingQuals {
+			// convert proto.Qual into a qual.Qual (which is easier to use)
+			qual := quals.NewQual(q)
+
+			// if there is already an entry for this column, add a value to the array
+			if mapEntry, mapEntryExists := res[col.Name]; mapEntryExists {
+				mapEntry.Quals = append(mapEntry.Quals, qual)
+				res[col.Name] = mapEntry
+			} else {
+				// crate a new map entry for this column
+				res[col.Name] = &KeyColumnQuals{
+					Name:  col.Name,
+					Quals: quals.QualSlice{qual},
+				}
+			}
+		}
+	}
+	return res
+}
+
+// look in a column-qual map for quals with column and operator matching the key column
+func getMatchingQuals(keyColumn *KeyColumn, qualMap map[string]*proto.Quals) []*proto.Qual {
+	log.Printf("[TRACE] getMatchingQuals keyColumn %s qualMap %s", keyColumn, qualMap)
+
+	quals, ok := qualMap[keyColumn.Name]
+	if !ok {
+		log.Printf("[TRACE] getMatchingQuals returning false - qualMap does not contain any quals for colums %s", keyColumn.Name)
+		return nil
+	}
+
+	var res []*proto.Qual
+	for _, q := range quals.Quals {
+		operator := q.GetStringValue()
+		if helpers.StringSliceContains(keyColumn.Operators, operator) {
+			res = append(res, q)
+		}
+	}
+	if len(res) > 0 {
+		log.Printf("[TRACE] getMatchingQuals found %d quals matching key column %s", len(res), keyColumn)
+	} else {
+		log.Printf("[TRACE] getMatchingQuals returning false - qualMap does not contain any matching quals for quals for key column %s", keyColumn)
+	}
+
+	return res
+}