Support tagged fields and taggable (#60)

* fix (filters/encrypt): make a copy of the event data before modifying it

since the node will be modifying the event data (aka redact/encrypt), we
need our own copy, otherwise we could be changing the event across other
pipelines and nodes and creating a host of problems and race conditions.

* fix (filters/encrypt): skip non-exported fields

When filtering fields, skip the non-exported fields which will
significantly improve prerformance when dealing with things
like protobufs which can have MANY unexported fields with
MANY values.

* feat (filters/encrypt): add support for filtering structpb.Value

Add support for tagging maps which are google.protobuf.Struct and use
structpb.Value for their values.

* feat (filters/encrypt): support structs with tagged fields and taggable interface

Support structs that have both tagged fields and have also
implemented the Taggable interface for its fields which
are maps

* fix (filters/encrypt): fix check for top level Taggable interface

We need to check if the top level payload interface{} for an
event is Taggable before we strip off the interface to the
underlying value.

* test (filters/encrypt): Add tests for protobuf payloads

* feat (filters/encrypt): Support filtering proto string/btye wrappers

* fix (filters/encrypt): Properly support Taggable filter operations

The initial implementation of Taggable required the
all impl. of Taggable to return the filter operation for each
tagged field and ignored the node's configured and overridden
filter operations.  This fixes that issue.  Going fwd the impl. of
Taggable "may" optionally return a filter op, and if none is returned
the node's filter operations and/or overrides are used.

filters/encrypt/Makefile

@@ -0,0 +1,27 @@
+
+THIS_FILE := $(lastword $(MAKEFILE_LIST))
+THIS_DIR := $(dir $(realpath $(firstword $(MAKEFILE_LIST))))
+
+TMP_DIR := $(shell mktemp -d)
+REPO_PATH := github.com/hashicorp/eventlogger/filters/encrypt
+
+
+# currently, protobufs are only used for testing the filtering of event payloads
+# which include protobufs.
+proto:  protobuild
+
+protobuild:
+	# To add a new directory containing a proto pass the  proto's root path in
+	# through the --proto_path flag.
+	@bash scripts/protoc_gen_plugin.bash \
+		"--proto_path=testing/proto" \
+		"--plugin_name=go" \
+		"--plugin_out=${TMP_DIR}"
+	# Move the generated files from the tmp file subdirectories into the current
+	# repo.
+	cp -R ${TMP_DIR}/${REPO_PATH}/* ${THIS_DIR}
+
+	# inject classification/filter tags
+	@protoc-go-inject-tag -input=./testing/resources/protopayload/proto_payload.pb.go
+
+

filters/encrypt/README.md

@@ -22,9 +22,8 @@ event payload, when they are tagged with a `class` tag:
 * `[]string`
 * `[]byte`
 * `[][]byte`
-
-encrypt.Filter also supports filtering any field of type `map[string]interface{}` that implements the
-`encrypt.Taggable` interface. 
+* `wrapperspb.StringValue`
+* `wrapperspb.BytesValue`
 
 The following DataClassifications are supported:
 * PublicClassification
@@ -40,9 +39,11 @@ The following FilterOperations are supported:
 
 
 # Taggable interface
-`map[string]interface{}` fields in an event payloads can be filtered using a
-`[]PointerTag` for the map. To be filtered a `map[string]interface{}` field is
-required to implement a single function interface:
+Go `maps` and `google.protobuf.Struct` in an event payloads can be filtered by
+implementing a single function `Taggable` interface, which returns a
+`[]PointerTag` for fields that must be filtered.  You may have payloads and/or
+payload fields which implement the `Taggable` interface and also contain fields
+that are tagged with the `class` tag.
 ```go
 // Taggable defines an interface for taggable maps
 type Taggable interface {
@@ -51,7 +52,7 @@ type Taggable interface {
 }
 
 // PointerTag provides the pointerstructure pointer string to get/set a key
-// within a map[string]interface{} along with its DataClassification and
+// within a map or struct.Value along with its DataClassification and
 // FilterOperation.
 type PointerTag struct {
 	// Pointer is the pointerstructure pointer string to get/set a key within a
@@ -63,7 +64,8 @@ type PointerTag struct {
 	Classification DataClassification
 
 	// Filter is the FilterOperation to apply to the data pointed to by the
-	// Pointer
+	// Pointer.  This is optional and the default operations (or overrides) will
+	// apply when not specified
 	Filter FilterOperation
 }
 ``` 

filters/encrypt/filter.go

@@ -13,8 +13,11 @@ import (
 
 	"github.com/hashicorp/eventlogger"
 	wrapping "github.com/hashicorp/go-kms-wrapping"
+	"github.com/mitchellh/copystructure"
 	"github.com/mitchellh/pointerstructure"
 	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/structpb"
+	"google.golang.org/protobuf/types/known/wrapperspb"
 )
 
 // Filter is an eventlogger Filter Node which will filter string and
@@ -155,10 +158,22 @@ func (ef *Filter) Process(ctx context.Context, e *eventlogger.Event) (*eventlogg
 		}
 	}
 
+	// since the node will be modifying the event data (aka redact/encrypt), we
+	// need our own copy, otherwise we could be changing the event across other
+	// pipelines and nodes and creating a host of problems and race conditions.
+	dup, err := copystructure.Copy(e)
+	if err != nil {
+		return nil, err
+	}
+	e = dup.(*eventlogger.Event)
+
 	// Get both the value and the type of what the payload points to. Value is
 	// used to mutate underlying data and Type is used to get the name of the
 	// field.
 	payloadValue := reflect.ValueOf(e.Payload)
+
+	taggedInterface, isTaggable := payloadValue.Interface().(Taggable)
+
 	switch payloadValue.Kind() {
 	case reflect.Ptr, reflect.Interface:
 		if payloadValue.IsNil() { // be sure it's not a nil interface
@@ -170,8 +185,6 @@ func (ef *Filter) Process(ctx context.Context, e *eventlogger.Event) (*eventlogg
 	pType := payloadValue.Type()
 	pKind := payloadValue.Kind()
 
-	taggedInterface, isTaggable := payloadValue.Interface().(Taggable)
-
 	// make a copy of the overrides before we begin processing this event, which
 	// will give us a consistent set of overrides for this event.
 	filterOverrides := ef.copyFilterOperationOverrides()
@@ -186,9 +199,18 @@ func (ef *Filter) Process(ctx context.Context, e *eventlogger.Event) (*eventlogg
 			return nil, fmt.Errorf("%s: %w", op, err)
 		}
 	case isTaggable:
-		if err := ef.filterTaggable(ctx, taggedInterface, opts...); err != nil {
+		if err := ef.filterTaggable(ctx, taggedInterface, filterOverrides, opts...); err != nil {
 			return nil, fmt.Errorf("%s: %w", op, err)
 		}
+		if pKind != reflect.Map {
+			// okay, we've dealt with the "Taggable" things, let's check for other
+			// fields that need to be filtered, but be sure to ignore taggable
+			// on the next recursion or will be in an infinite loop
+			opts := append(opts, withIgnoreTaggable())
+			if err := ef.filterField(ctx, payloadValue, filterOverrides, opts...); err != nil {
+				return nil, fmt.Errorf("%s: %w", op, err)
+			}
+		}
 	case pKind == reflect.Slice:
 		switch {
 		// if the field is a slice of string or slice of []byte
@@ -242,18 +264,46 @@ func (ef *Filter) filterField(ctx context.Context, v reflect.Value, filterOverri
 		return nil
 	}
 
+	opts := getOpts(opt...)
+	// we want to check if we should ignore taggable for this recursion, but
+	// then strip the option, so the next level of recursion can redact both
+	// taggable things and other fields.
+	if opts.withIgnoreTaggable {
+		var removeIdx int
+		for i := 0; i < len(opt)-1; i++ {
+			currentOpt := getOpts(opt[i])
+			if currentOpt.withIgnoreTaggable {
+				removeIdx = i
+				break
+			}
+		}
+		opt = append(opt[:removeIdx], opt[removeIdx+1:]...)
+	}
+
 	for i := 0; i < v.Type().NumField(); i++ {
 		field := v.Field(i)
+		fkind := field.Kind()
 
-		switch v.Field(i).Kind() {
+		// skip non-exported fields which cannot interface.
+		if !field.CanInterface() {
+			continue
+		}
+
+		switch fkind {
 		case reflect.Ptr, reflect.Interface:
 			field = v.Field(i).Elem()
 			if field == reflect.ValueOf(nil) {
 				continue
 			}
+			if field.Kind() == reflect.Ptr { // well, it was an interface and we sill need to determine what the ptr is...
+				field = field.Elem()
+				if field == reflect.ValueOf(nil) {
+					continue
+				}
+			}
+			fkind = field.Kind() // re-init to the kind after deferencing the pointer or interface...
 		}
 
-		fkind := field.Kind()
 		ftype := field.Type()
 
 		var taggedInterface Taggable
@@ -270,6 +320,11 @@ func (ef *Filter) filterField(ctx context.Context, v reflect.Value, filterOverri
 			if err := ef.filterValue(ctx, field, classificationTag, opt...); err != nil {
 				return fmt.Errorf("%s: %w", op, err)
 			}
+		case ftype == reflect.TypeOf(wrapperspb.StringValue{}) || ftype == reflect.TypeOf(wrapperspb.BytesValue{}):
+			classificationTag := getClassificationFromTag(v.Type().Field(i).Tag, withFilterOperations(filterOverrides))
+			if err := ef.filterValue(ctx, field.FieldByName("Value"), classificationTag, opt...); err != nil {
+				return err
+			}
 		// if the field is a slice
 		case fkind == reflect.Slice:
 			switch {
@@ -294,23 +349,33 @@ func (ef *Filter) filterField(ctx context.Context, v reflect.Value, filterOverri
 					}
 				}
 			}
+
+		case isTaggable && !opts.withIgnoreTaggable:
+			if err := ef.filterTaggable(ctx, taggedInterface, filterOverrides, opt...); err != nil {
+				return fmt.Errorf("%s: %w", op, err)
+			}
+			if fkind != reflect.Map {
+				// okay, we've dealt with the "Taggable" things, let's check for other
+				// fields that need to be filtered, but be sure to ignore taggable
+				// on the next recursion or will be in an infinite loop
+				opt = append(opt, withIgnoreTaggable())
+				if err := ef.filterField(ctx, field, filterOverrides, opt...); err != nil {
+					return fmt.Errorf("%s: %w", op, err)
+				}
+			}
+
 		// if the field is a struct
 		case fkind == reflect.Struct:
 			if err := ef.filterField(ctx, field, filterOverrides, opt...); err != nil {
 				return err
 			}
-
-		case isTaggable:
-			if err := ef.filterTaggable(ctx, taggedInterface, opt...); err != nil {
-				return fmt.Errorf("%s: %w", op, err)
-			}
 		}
 	}
 	return nil
 }
 
 // filterTaggable will filter data that implements the Taggable interface
-func (ef *Filter) filterTaggable(ctx context.Context, t Taggable, _ ...Option) error {
+func (ef *Filter) filterTaggable(ctx context.Context, t Taggable, filterOverrides map[DataClassification]FilterOperation, opt ...Option) error {
 	const op = "event.(Filter).filterTaggable"
 	if t == nil {
 		return fmt.Errorf("%s: missing taggable interface: %w", op, ErrInvalidParameter)
@@ -329,10 +394,7 @@ func (ef *Filter) filterTaggable(ctx context.Context, t Taggable, _ ...Option) e
 			}
 		}
 		rv := reflect.Indirect(reflect.ValueOf(value))
-		info := &tagInfo{
-			Classification: pt.Classification,
-			Operation:      pt.Filter,
-		}
+		info := getClassificationFromTagString(fmt.Sprintf("%s,%s", pt.Classification, pt.Filter), withFilterOperations(filterOverrides))
 		if err = ef.filterValue(ctx, rv, info, withPointer(t, pt.Pointer)); err != nil {
 			return fmt.Errorf("%s: %w", op, err)
 		}
@@ -423,7 +485,12 @@ func (ef *Filter) filterValue(ctx context.Context, fv reflect.Value, classificat
 			if err != nil {
 				return fmt.Errorf("%s: unable to get value from taggable interface using pointer: %s: %w", op, opts.withPointerstructureInfo.pointer, err)
 			}
-			raw = []byte(fmt.Sprintf("%s", i))
+			switch {
+			case reflect.TypeOf(i) == reflect.TypeOf(&structpb.Value{}):
+				raw = []byte(i.(*structpb.Value).GetStringValue())
+			default:
+				raw = []byte(fmt.Sprintf("%s", i))
+			}
 		case fv.Type() == reflect.TypeOf(""):
 			raw = []byte(fv.String())
 		case fv.Type() == reflect.TypeOf([]uint8(nil)):
@@ -450,8 +517,16 @@ func (ef *Filter) filterValue(ctx context.Context, fv reflect.Value, classificat
 			return fmt.Errorf("%s: unknown filter operation for field: %s: %w", op, classificationTag.Operation, ErrInvalidParameter)
 		}
 		if opts.withPointerstructureInfo != nil {
-			if _, err := pointerstructure.Set(opts.withPointerstructureInfo.i, opts.withPointerstructureInfo.pointer, data); err != nil {
-				return fmt.Errorf("%s: %w", op, err)
+			switch {
+			case ftype == reflect.TypeOf(structpb.Value{}):
+				// support for tagging maps which are google.protobuf.Struct and use structpb.Value for their values.
+				if _, err := pointerstructure.Set(opts.withPointerstructureInfo.i, opts.withPointerstructureInfo.pointer, structpb.NewStringValue(data)); err != nil {
+					return fmt.Errorf("%s: %w", op, err)
+				}
+			default:
+				if _, err := pointerstructure.Set(opts.withPointerstructureInfo.i, opts.withPointerstructureInfo.pointer, data); err != nil {
+					return fmt.Errorf("%s: %w", op, err)
+				}
 			}
 		} else {
 			if err := setValue(fv, data); err != nil {

filters/encrypt/filter_test.go

@@ -57,7 +57,7 @@ func TestFilter_filterTaggable(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			assert, require := assert.New(t), require.New(t)
-			err := tt.ef.filterTaggable(ctx, tt.t, tt.opt...)
+			err := tt.ef.filterTaggable(ctx, tt.t, tt.ef.copyFilterOperationOverrides(), tt.opt...)
 			if tt.wantErrIs != nil {
 				require.Error(err)
 				assert.ErrorIs(err, tt.wantErrIs)