From 8ddc2716ae10d3feff13102b809f81e9900b8447 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Thu, 29 Nov 2018 08:42:34 +0100 Subject: [PATCH 1/2] protosanitizer: handle secrets in oneof and maps As expected, oneof wasn't supported so far and requires special code: - JSON naming is not defined in Go structs and thus inconsistent with protobuf - JSON and Go struct are nested, protobuf fields aren't For maps from plain type to a message, the messages were not filtered. Because the nesting got a bit deep with support for both added, the code now avoids nesting by aborting the flow in `if` cases with continue or return and continuing unindented after that. Everything that could cause a panic gets checked, which protects against malformed protobuf data but should never trigger in practice. To determine where an assumption gets violated, strip() now returns an error and wraps that with pkg.errors, which eventually gets reports in the resulting string. This is similar to how fmt.Sprintf handles parameter errors. --- Gopkg.lock | 9 + protosanitizer/protosanitizer.go | 166 ++++++++++--- protosanitizer/protosanitizer_test.go | 6 +- vendor/github.com/pkg/errors/.gitignore | 24 ++ vendor/github.com/pkg/errors/.travis.yml | 11 + vendor/github.com/pkg/errors/LICENSE | 23 ++ vendor/github.com/pkg/errors/README.md | 52 +++++ vendor/github.com/pkg/errors/appveyor.yml | 32 +++ vendor/github.com/pkg/errors/errors.go | 269 ++++++++++++++++++++++ vendor/github.com/pkg/errors/stack.go | 178 ++++++++++++++ 10 files changed, 731 insertions(+), 39 deletions(-) create mode 100644 vendor/github.com/pkg/errors/.gitignore create mode 100644 vendor/github.com/pkg/errors/.travis.yml create mode 100644 vendor/github.com/pkg/errors/LICENSE create mode 100644 vendor/github.com/pkg/errors/README.md create mode 100644 vendor/github.com/pkg/errors/appveyor.yml create mode 100644 vendor/github.com/pkg/errors/errors.go create mode 100644 vendor/github.com/pkg/errors/stack.go diff --git a/Gopkg.lock b/Gopkg.lock index 8d4120d8..377b470f 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -26,6 +26,14 @@ revision = "aa810b61a9c79d51363740d207bb46cf8e620ed5" version = "v1.2.0" +[[projects]] + digest = "1:40e195917a951a8bf867cd05de2a46aaf1806c50cf92eebf4c16f78cd196f747" + name = "github.com/pkg/errors" + packages = ["."] + pruneopts = "UT" + revision = "645ef00459ed84a119197bfb8d8205042c6df63d" + version = "v0.8.0" + [[projects]] digest = "1:0028cb19b2e4c3112225cd871870f2d9cf49b9b4276531f03438a88e94be86fe" name = "github.com/pmezard/go-difflib" @@ -141,6 +149,7 @@ "github.com/golang/protobuf/protoc-gen-go/descriptor", "github.com/golang/protobuf/ptypes/timestamp", "github.com/golang/protobuf/ptypes/wrappers", + "github.com/pkg/errors", "github.com/stretchr/testify/assert", "golang.org/x/net/context", "google.golang.org/grpc", diff --git a/protosanitizer/protosanitizer.go b/protosanitizer/protosanitizer.go index af64a7b2..45ee9131 100644 --- a/protosanitizer/protosanitizer.go +++ b/protosanitizer/protosanitizer.go @@ -22,12 +22,14 @@ import ( "encoding/json" "fmt" "reflect" + "regexp" "strings" "github.com/golang/protobuf/descriptor" "github.com/golang/protobuf/proto" protobuf "github.com/golang/protobuf/protoc-gen-go/descriptor" protobufdescriptor "github.com/golang/protobuf/protoc-gen-go/descriptor" + "github.com/pkg/errors" ) // StripSecrets returns a wrapper around the original CSI gRPC message @@ -72,7 +74,9 @@ func (s *stripSecrets) String() string { } // Now remove secrets from the generic representation of the message. - s.strip(parsed, s.msg) + if err := s.strip(parsed, s.msg); err != nil { + return fmt.Sprintf("<>", err) + } // Re-encoded the stripped representation and return that. b, err = json.Marshal(parsed) @@ -82,18 +86,26 @@ func (s *stripSecrets) String() string { return string(b) } -func (s *stripSecrets) strip(parsed interface{}, msg interface{}) { +// isFieldName returns true for strings that start with a-zA-Z and +// are followed by those, digits or underscore. +func isFieldName(str string) bool { + return fieldNameRe.MatchString(str) +} + +var fieldNameRe = regexp.MustCompile(`[a-zA-Z][a-zA-Z0-9_]*`) + +func (s *stripSecrets) strip(parsed interface{}, msg interface{}) error { protobufMsg, ok := msg.(descriptor.Message) if !ok { - // Not a protobuf message, so we are done. - return + // Not a protobuf message, nothing to strip. + return nil } // The corresponding map in the parsed JSON representation. parsedFields, ok := parsed.(map[string]interface{}) if !ok { - // Probably nil. - return + // Probably nil, nothing to strip. + return nil } // Walk through all fields and replace those with ***stripped*** that @@ -110,42 +122,128 @@ func (s *stripSecrets) strip(parsed interface{}, msg interface{}) { if _, ok := parsedFields[field.GetName()]; ok { parsedFields[field.GetName()] = "***stripped***" } - } else if field.GetType() == protobuf.FieldDescriptorProto_TYPE_MESSAGE { - // When we get here, - // the type name is something like ".csi.v1.CapacityRange" (leading dot!) - // and looking up "csi.v1.CapacityRange" - // returns the type of a pointer to a pointer - // to CapacityRange. We need a pointer to such - // a value for recursive stripping. - typeName := field.GetTypeName() - if strings.HasPrefix(typeName, ".") { - typeName = typeName[1:] + continue + } + + // Not stripped. Decide whether we need to + // recursively strip the message(s) that the + // field contains. + + if field.GetType() != protobuf.FieldDescriptorProto_TYPE_MESSAGE { + // No need to recurse into plain types. + continue + } + + // When we get here, the type name is something + // like ".csi.v1.CapacityRange" (leading dot!) + // and looking up "csi.v1.CapacityRange" + // returns the type of a pointer to a pointer + // to CapacityRange. We need a pointer to such + // a value for recursive stripping. + typeName := field.GetTypeName() + if strings.HasPrefix(typeName, ".") { + typeName = typeName[1:] + } + t := proto.MessageType(typeName) + // Shouldn't happen, but better check + // anyway instead of panicking. + if t == nil { + return errors.Errorf("%s: unknown type", typeName) + } + var v reflect.Value + switch t.Kind() { + case reflect.Map: + ptrType := t.Elem() + if ptrType.Kind() != reflect.Ptr { + // map to plain type, nothing to recurse into + continue + } + v = reflect.New(t.Elem().Elem()) + case reflect.Ptr: + v = reflect.New(t.Elem()) + default: + return errors.Errorf("%s: has no elements", typeName) + } + i := v.Interface() + + if field.OneofIndex != nil { + // A oneof field doesn't have json tags in the generated .pb.go. + // Therefore the parsedFields is different and we need to recurse differently: + // - the entry is named like the Go field (upper first character, no underscores) + // - it contains a map with the name of the individual candidates, again + // with Go naming + // Example for proto field "volume" inside "VolumeContentSource": + // map[string]interface {} [ + // "Type": map[string]interface {} [ + // "Volume": *(*"interface {}")(0xc4201c0988), + // ], + // ] + if len(md.OneofDecl) <= int(*field.OneofIndex) { + // Shouldn't happen, bail out. + return errors.Errorf("invalid oneof index in %v", field) + } + oneof := md.OneofDecl[int(*field.OneofIndex)] + if oneof.Name == nil { + return errors.Errorf("invalid oneof for %s, no name: %v", typeName, oneof) } - t := proto.MessageType(typeName) - if t == nil || t.Kind() != reflect.Ptr { - // Shouldn't happen, but - // better check anyway instead - // of panicking. + jsonName := upperFirst(*oneof.Name) + entry, ok := parsedFields[jsonName] + if !ok || entry == nil { + // Not set. continue } - v := reflect.New(t.Elem()) - - // Recursively strip the message(s) that - // the field contains. - i := v.Interface() - entry := parsedFields[field.GetName()] - if slice, ok := entry.([]interface{}); ok { - // Array of values, like VolumeCapabilities in CreateVolumeRequest. - for _, entry := range slice { - s.strip(entry, i) + oneofMap, ok := entry.(map[string]interface{}) + if !ok { + return errors.Errorf("unexpected type %T in JSON for %s", entry, typeName) + } + if field.JsonName == nil { + return errors.Errorf("invalid field %v, no name", field) + } + entry, ok = oneofMap[upperFirst(*field.JsonName)] + if !ok { + // Oneof does not contain this particular field. + continue + } + // Finally recurse into a particular oneof value. + if err := s.strip(entry, i); err != nil { + return errors.Wrap(err, typeName) + } + continue + } + + entry := parsedFields[field.GetName()] + if slice, ok := entry.([]interface{}); ok { + // Array of values, like VolumeCapabilities in CreateVolumeRequest. + for _, entry := range slice { + if err := s.strip(entry, i); err != nil { + return errors.Wrap(err, typeName) } - } else { - // Single value. - s.strip(entry, i) } + continue + } + if mapping, ok := entry.(map[string]interface{}); ok { + // All maps in protobuf are string to something maps in JSON. + for _, entry := range mapping { + if err := s.strip(entry, i); err != nil { + return errors.Wrap(err, typeName) + } + } + } + + // Single value. + if err := s.strip(entry, i); err != nil { + return errors.Wrap(err, typeName) } } } + return nil +} + +func upperFirst(str string) string { + if len(str) >= 2 { + return strings.ToUpper(str[:1]) + str[1:] + } + return strings.ToUpper(str) } // isCSI1Secret uses the csi.E_CsiSecret extension from CSI 1.0 to diff --git a/protosanitizer/protosanitizer_test.go b/protosanitizer/protosanitizer_test.go index 944943c0..65b1e1d5 100644 --- a/protosanitizer/protosanitizer_test.go +++ b/protosanitizer/protosanitizer_test.go @@ -182,11 +182,7 @@ func TestStripSecrets(t *testing.T) { {createVolumeCSI03, `{"accessibility_requirements":{"requisite":[{"segments":{"foo":"bar","x":"y"}},{"segments":{"a":"b"}}]},"capacity_range":{"required_bytes":1024},"controller_create_secrets":"***stripped***","name":"foo","volume_capabilities":[{"AccessType":{"Mount":{"fs_type":"ext4"}}}]}`}, {&csitest.CreateVolumeRequest{}, `{}`}, {createVolumeFuture, - // Secrets are *not* removed from all fields yet. This will have to be fixed one way or another - // before the CSI spec can start using secrets there (currently it doesn't). - // The test is still useful because it shows that also complicated fields get serialized. - // `{"capacity_range":{"required_bytes":1024},"maybe_secret_map":{"1":{"AccessType":null,"array_secret":"***stripped***"},"2":{"AccessType":null,"array_secret":"***stripped***"}},"name":"foo","new_secret_int":"***stripped***","seecreets":"***stripped***","volume_capabilities":[{"AccessType":{"Mount":{"fs_type":"ext4"}},"array_secret":"***stripped***"},{"AccessType":null,"array_secret":"***stripped***"}],"volume_content_source":{"Type":{"Volume":{"oneof_secret_field":"***stripped***","volume_id":"abc"}},"nested_secret_field":"***stripped***"}}`, - `{"capacity_range":{"required_bytes":1024},"maybe_secret_map":{"1":{"AccessType":null,"array_secret":"aaa"},"2":{"AccessType":null,"array_secret":"bbb"}},"name":"foo","new_secret_int":"***stripped***","seecreets":"***stripped***","volume_capabilities":[{"AccessType":{"Mount":{"fs_type":"ext4"}},"array_secret":"***stripped***"},{"AccessType":null,"array_secret":"***stripped***"}],"volume_content_source":{"Type":{"Volume":{"oneof_secret_field":"hello","volume_id":"abc"}},"nested_secret_field":"***stripped***"}}`, + `{"capacity_range":{"required_bytes":1024},"maybe_secret_map":{"1":{"AccessType":null,"array_secret":"***stripped***"},"2":{"AccessType":null,"array_secret":"***stripped***"}},"name":"foo","new_secret_int":"***stripped***","seecreets":"***stripped***","volume_capabilities":[{"AccessType":{"Mount":{"fs_type":"ext4"}},"array_secret":"***stripped***"},{"AccessType":null,"array_secret":"***stripped***"}],"volume_content_source":{"Type":{"Volume":{"oneof_secret_field":"***stripped***","volume_id":"abc"}},"nested_secret_field":"***stripped***"}}`, }, } diff --git a/vendor/github.com/pkg/errors/.gitignore b/vendor/github.com/pkg/errors/.gitignore new file mode 100644 index 00000000..daf913b1 --- /dev/null +++ b/vendor/github.com/pkg/errors/.gitignore @@ -0,0 +1,24 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test +*.prof diff --git a/vendor/github.com/pkg/errors/.travis.yml b/vendor/github.com/pkg/errors/.travis.yml new file mode 100644 index 00000000..588ceca1 --- /dev/null +++ b/vendor/github.com/pkg/errors/.travis.yml @@ -0,0 +1,11 @@ +language: go +go_import_path: github.com/pkg/errors +go: + - 1.4.3 + - 1.5.4 + - 1.6.2 + - 1.7.1 + - tip + +script: + - go test -v ./... diff --git a/vendor/github.com/pkg/errors/LICENSE b/vendor/github.com/pkg/errors/LICENSE new file mode 100644 index 00000000..835ba3e7 --- /dev/null +++ b/vendor/github.com/pkg/errors/LICENSE @@ -0,0 +1,23 @@ +Copyright (c) 2015, Dave Cheney +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/pkg/errors/README.md b/vendor/github.com/pkg/errors/README.md new file mode 100644 index 00000000..273db3c9 --- /dev/null +++ b/vendor/github.com/pkg/errors/README.md @@ -0,0 +1,52 @@ +# errors [![Travis-CI](https://travis-ci.org/pkg/errors.svg)](https://travis-ci.org/pkg/errors) [![AppVeyor](https://ci.appveyor.com/api/projects/status/b98mptawhudj53ep/branch/master?svg=true)](https://ci.appveyor.com/project/davecheney/errors/branch/master) [![GoDoc](https://godoc.org/github.com/pkg/errors?status.svg)](http://godoc.org/github.com/pkg/errors) [![Report card](https://goreportcard.com/badge/github.com/pkg/errors)](https://goreportcard.com/report/github.com/pkg/errors) + +Package errors provides simple error handling primitives. + +`go get github.com/pkg/errors` + +The traditional error handling idiom in Go is roughly akin to +```go +if err != nil { + return err +} +``` +which applied recursively up the call stack results in error reports without context or debugging information. The errors package allows programmers to add context to the failure path in their code in a way that does not destroy the original value of the error. + +## Adding context to an error + +The errors.Wrap function returns a new error that adds context to the original error. For example +```go +_, err := ioutil.ReadAll(r) +if err != nil { + return errors.Wrap(err, "read failed") +} +``` +## Retrieving the cause of an error + +Using `errors.Wrap` constructs a stack of errors, adding context to the preceding error. Depending on the nature of the error it may be necessary to reverse the operation of errors.Wrap to retrieve the original error for inspection. Any error value which implements this interface can be inspected by `errors.Cause`. +```go +type causer interface { + Cause() error +} +``` +`errors.Cause` will recursively retrieve the topmost error which does not implement `causer`, which is assumed to be the original cause. For example: +```go +switch err := errors.Cause(err).(type) { +case *MyError: + // handle specifically +default: + // unknown error +} +``` + +[Read the package documentation for more information](https://godoc.org/github.com/pkg/errors). + +## Contributing + +We welcome pull requests, bug fixes and issue reports. With that said, the bar for adding new symbols to this package is intentionally set high. + +Before proposing a change, please discuss your change by raising an issue. + +## Licence + +BSD-2-Clause diff --git a/vendor/github.com/pkg/errors/appveyor.yml b/vendor/github.com/pkg/errors/appveyor.yml new file mode 100644 index 00000000..a932eade --- /dev/null +++ b/vendor/github.com/pkg/errors/appveyor.yml @@ -0,0 +1,32 @@ +version: build-{build}.{branch} + +clone_folder: C:\gopath\src\github.com\pkg\errors +shallow_clone: true # for startup speed + +environment: + GOPATH: C:\gopath + +platform: + - x64 + +# http://www.appveyor.com/docs/installed-software +install: + # some helpful output for debugging builds + - go version + - go env + # pre-installed MinGW at C:\MinGW is 32bit only + # but MSYS2 at C:\msys64 has mingw64 + - set PATH=C:\msys64\mingw64\bin;%PATH% + - gcc --version + - g++ --version + +build_script: + - go install -v ./... + +test_script: + - set PATH=C:\gopath\bin;%PATH% + - go test -v ./... + +#artifacts: +# - path: '%GOPATH%\bin\*.exe' +deploy: off diff --git a/vendor/github.com/pkg/errors/errors.go b/vendor/github.com/pkg/errors/errors.go new file mode 100644 index 00000000..842ee804 --- /dev/null +++ b/vendor/github.com/pkg/errors/errors.go @@ -0,0 +1,269 @@ +// Package errors provides simple error handling primitives. +// +// The traditional error handling idiom in Go is roughly akin to +// +// if err != nil { +// return err +// } +// +// which applied recursively up the call stack results in error reports +// without context or debugging information. The errors package allows +// programmers to add context to the failure path in their code in a way +// that does not destroy the original value of the error. +// +// Adding context to an error +// +// The errors.Wrap function returns a new error that adds context to the +// original error by recording a stack trace at the point Wrap is called, +// and the supplied message. For example +// +// _, err := ioutil.ReadAll(r) +// if err != nil { +// return errors.Wrap(err, "read failed") +// } +// +// If additional control is required the errors.WithStack and errors.WithMessage +// functions destructure errors.Wrap into its component operations of annotating +// an error with a stack trace and an a message, respectively. +// +// Retrieving the cause of an error +// +// Using errors.Wrap constructs a stack of errors, adding context to the +// preceding error. Depending on the nature of the error it may be necessary +// to reverse the operation of errors.Wrap to retrieve the original error +// for inspection. Any error value which implements this interface +// +// type causer interface { +// Cause() error +// } +// +// can be inspected by errors.Cause. errors.Cause will recursively retrieve +// the topmost error which does not implement causer, which is assumed to be +// the original cause. For example: +// +// switch err := errors.Cause(err).(type) { +// case *MyError: +// // handle specifically +// default: +// // unknown error +// } +// +// causer interface is not exported by this package, but is considered a part +// of stable public API. +// +// Formatted printing of errors +// +// All error values returned from this package implement fmt.Formatter and can +// be formatted by the fmt package. The following verbs are supported +// +// %s print the error. If the error has a Cause it will be +// printed recursively +// %v see %s +// %+v extended format. Each Frame of the error's StackTrace will +// be printed in detail. +// +// Retrieving the stack trace of an error or wrapper +// +// New, Errorf, Wrap, and Wrapf record a stack trace at the point they are +// invoked. This information can be retrieved with the following interface. +// +// type stackTracer interface { +// StackTrace() errors.StackTrace +// } +// +// Where errors.StackTrace is defined as +// +// type StackTrace []Frame +// +// The Frame type represents a call site in the stack trace. Frame supports +// the fmt.Formatter interface that can be used for printing information about +// the stack trace of this error. For example: +// +// if err, ok := err.(stackTracer); ok { +// for _, f := range err.StackTrace() { +// fmt.Printf("%+s:%d", f) +// } +// } +// +// stackTracer interface is not exported by this package, but is considered a part +// of stable public API. +// +// See the documentation for Frame.Format for more details. +package errors + +import ( + "fmt" + "io" +) + +// New returns an error with the supplied message. +// New also records the stack trace at the point it was called. +func New(message string) error { + return &fundamental{ + msg: message, + stack: callers(), + } +} + +// Errorf formats according to a format specifier and returns the string +// as a value that satisfies error. +// Errorf also records the stack trace at the point it was called. +func Errorf(format string, args ...interface{}) error { + return &fundamental{ + msg: fmt.Sprintf(format, args...), + stack: callers(), + } +} + +// fundamental is an error that has a message and a stack, but no caller. +type fundamental struct { + msg string + *stack +} + +func (f *fundamental) Error() string { return f.msg } + +func (f *fundamental) Format(s fmt.State, verb rune) { + switch verb { + case 'v': + if s.Flag('+') { + io.WriteString(s, f.msg) + f.stack.Format(s, verb) + return + } + fallthrough + case 's': + io.WriteString(s, f.msg) + case 'q': + fmt.Fprintf(s, "%q", f.msg) + } +} + +// WithStack annotates err with a stack trace at the point WithStack was called. +// If err is nil, WithStack returns nil. +func WithStack(err error) error { + if err == nil { + return nil + } + return &withStack{ + err, + callers(), + } +} + +type withStack struct { + error + *stack +} + +func (w *withStack) Cause() error { return w.error } + +func (w *withStack) Format(s fmt.State, verb rune) { + switch verb { + case 'v': + if s.Flag('+') { + fmt.Fprintf(s, "%+v", w.Cause()) + w.stack.Format(s, verb) + return + } + fallthrough + case 's': + io.WriteString(s, w.Error()) + case 'q': + fmt.Fprintf(s, "%q", w.Error()) + } +} + +// Wrap returns an error annotating err with a stack trace +// at the point Wrap is called, and the supplied message. +// If err is nil, Wrap returns nil. +func Wrap(err error, message string) error { + if err == nil { + return nil + } + err = &withMessage{ + cause: err, + msg: message, + } + return &withStack{ + err, + callers(), + } +} + +// Wrapf returns an error annotating err with a stack trace +// at the point Wrapf is call, and the format specifier. +// If err is nil, Wrapf returns nil. +func Wrapf(err error, format string, args ...interface{}) error { + if err == nil { + return nil + } + err = &withMessage{ + cause: err, + msg: fmt.Sprintf(format, args...), + } + return &withStack{ + err, + callers(), + } +} + +// WithMessage annotates err with a new message. +// If err is nil, WithMessage returns nil. +func WithMessage(err error, message string) error { + if err == nil { + return nil + } + return &withMessage{ + cause: err, + msg: message, + } +} + +type withMessage struct { + cause error + msg string +} + +func (w *withMessage) Error() string { return w.msg + ": " + w.cause.Error() } +func (w *withMessage) Cause() error { return w.cause } + +func (w *withMessage) Format(s fmt.State, verb rune) { + switch verb { + case 'v': + if s.Flag('+') { + fmt.Fprintf(s, "%+v\n", w.Cause()) + io.WriteString(s, w.msg) + return + } + fallthrough + case 's', 'q': + io.WriteString(s, w.Error()) + } +} + +// Cause returns the underlying cause of the error, if possible. +// An error value has a cause if it implements the following +// interface: +// +// type causer interface { +// Cause() error +// } +// +// If the error does not implement Cause, the original error will +// be returned. If the error is nil, nil will be returned without further +// investigation. +func Cause(err error) error { + type causer interface { + Cause() error + } + + for err != nil { + cause, ok := err.(causer) + if !ok { + break + } + err = cause.Cause() + } + return err +} diff --git a/vendor/github.com/pkg/errors/stack.go b/vendor/github.com/pkg/errors/stack.go new file mode 100644 index 00000000..6b1f2891 --- /dev/null +++ b/vendor/github.com/pkg/errors/stack.go @@ -0,0 +1,178 @@ +package errors + +import ( + "fmt" + "io" + "path" + "runtime" + "strings" +) + +// Frame represents a program counter inside a stack frame. +type Frame uintptr + +// pc returns the program counter for this frame; +// multiple frames may have the same PC value. +func (f Frame) pc() uintptr { return uintptr(f) - 1 } + +// file returns the full path to the file that contains the +// function for this Frame's pc. +func (f Frame) file() string { + fn := runtime.FuncForPC(f.pc()) + if fn == nil { + return "unknown" + } + file, _ := fn.FileLine(f.pc()) + return file +} + +// line returns the line number of source code of the +// function for this Frame's pc. +func (f Frame) line() int { + fn := runtime.FuncForPC(f.pc()) + if fn == nil { + return 0 + } + _, line := fn.FileLine(f.pc()) + return line +} + +// Format formats the frame according to the fmt.Formatter interface. +// +// %s source file +// %d source line +// %n function name +// %v equivalent to %s:%d +// +// Format accepts flags that alter the printing of some verbs, as follows: +// +// %+s path of source file relative to the compile time GOPATH +// %+v equivalent to %+s:%d +func (f Frame) Format(s fmt.State, verb rune) { + switch verb { + case 's': + switch { + case s.Flag('+'): + pc := f.pc() + fn := runtime.FuncForPC(pc) + if fn == nil { + io.WriteString(s, "unknown") + } else { + file, _ := fn.FileLine(pc) + fmt.Fprintf(s, "%s\n\t%s", fn.Name(), file) + } + default: + io.WriteString(s, path.Base(f.file())) + } + case 'd': + fmt.Fprintf(s, "%d", f.line()) + case 'n': + name := runtime.FuncForPC(f.pc()).Name() + io.WriteString(s, funcname(name)) + case 'v': + f.Format(s, 's') + io.WriteString(s, ":") + f.Format(s, 'd') + } +} + +// StackTrace is stack of Frames from innermost (newest) to outermost (oldest). +type StackTrace []Frame + +func (st StackTrace) Format(s fmt.State, verb rune) { + switch verb { + case 'v': + switch { + case s.Flag('+'): + for _, f := range st { + fmt.Fprintf(s, "\n%+v", f) + } + case s.Flag('#'): + fmt.Fprintf(s, "%#v", []Frame(st)) + default: + fmt.Fprintf(s, "%v", []Frame(st)) + } + case 's': + fmt.Fprintf(s, "%s", []Frame(st)) + } +} + +// stack represents a stack of program counters. +type stack []uintptr + +func (s *stack) Format(st fmt.State, verb rune) { + switch verb { + case 'v': + switch { + case st.Flag('+'): + for _, pc := range *s { + f := Frame(pc) + fmt.Fprintf(st, "\n%+v", f) + } + } + } +} + +func (s *stack) StackTrace() StackTrace { + f := make([]Frame, len(*s)) + for i := 0; i < len(f); i++ { + f[i] = Frame((*s)[i]) + } + return f +} + +func callers() *stack { + const depth = 32 + var pcs [depth]uintptr + n := runtime.Callers(3, pcs[:]) + var st stack = pcs[0:n] + return &st +} + +// funcname removes the path prefix component of a function's name reported by func.Name(). +func funcname(name string) string { + i := strings.LastIndex(name, "/") + name = name[i+1:] + i = strings.Index(name, ".") + return name[i+1:] +} + +func trimGOPATH(name, file string) string { + // Here we want to get the source file path relative to the compile time + // GOPATH. As of Go 1.6.x there is no direct way to know the compiled + // GOPATH at runtime, but we can infer the number of path segments in the + // GOPATH. We note that fn.Name() returns the function name qualified by + // the import path, which does not include the GOPATH. Thus we can trim + // segments from the beginning of the file path until the number of path + // separators remaining is one more than the number of path separators in + // the function name. For example, given: + // + // GOPATH /home/user + // file /home/user/src/pkg/sub/file.go + // fn.Name() pkg/sub.Type.Method + // + // We want to produce: + // + // pkg/sub/file.go + // + // From this we can easily see that fn.Name() has one less path separator + // than our desired output. We count separators from the end of the file + // path until it finds two more than in the function name and then move + // one character forward to preserve the initial path segment without a + // leading separator. + const sep = "/" + goal := strings.Count(name, sep) + 2 + i := len(file) + for n := 0; n < goal; n++ { + i = strings.LastIndex(file[:i], sep) + if i == -1 { + // not enough separators found, set i so that the slice expression + // below leaves file unmodified + i = -len(sep) + break + } + } + // get back to 0 or trim the leading separator + file = file[i+len(sep):] + return file +} From e1b57dcce7e111680b8bc13cf1dee1ae8003b145 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Thu, 29 Nov 2018 16:43:28 +0100 Subject: [PATCH 2/2] protosanitizer: nicer marshaling of stripped messages The output is now closer to the original marshaling done by gRPC: - no distracting quotation marks around field names - angle brackets for structs and maps Some differences are intentional to increase readability: - no redundant space after the last element in a struct or map - square brackets around arrays One remaining drawback is the loss of the original field ordering. Elements are now sorted alphabetically because the conversion into the generic data structure via json.Marshal/Unmarshal drops the ordering when converting to a map. --- protosanitizer/protosanitizer.go | 74 ++++++++++++++++++++++++--- protosanitizer/protosanitizer_test.go | 16 +++--- 2 files changed, 75 insertions(+), 15 deletions(-) diff --git a/protosanitizer/protosanitizer.go b/protosanitizer/protosanitizer.go index 45ee9131..163328fc 100644 --- a/protosanitizer/protosanitizer.go +++ b/protosanitizer/protosanitizer.go @@ -19,10 +19,13 @@ limitations under the License. package protosanitizer import ( + "bytes" "encoding/json" "fmt" + "io" "reflect" "regexp" + "sort" "strings" "github.com/golang/protobuf/descriptor" @@ -34,7 +37,7 @@ import ( // StripSecrets returns a wrapper around the original CSI gRPC message // which has a Stringer implementation that serializes the message -// as one-line JSON, but without including secret information. +// similar to gRPC, but without including secret information. // Instead of the secret value(s), the string "***stripped***" is // included in the result. // @@ -55,6 +58,12 @@ func StripSecretsCSI03(msg interface{}) fmt.Stringer { return &stripSecrets{msg, isCSI03Secret} } +type stripped struct{} + +func (s stripped) String() string { + return "***stripped***" +} + type stripSecrets struct { msg interface{} @@ -79,11 +88,64 @@ func (s *stripSecrets) String() string { } // Re-encoded the stripped representation and return that. - b, err = json.Marshal(parsed) - if err != nil { - return fmt.Sprintf("<>", s.msg, err) + var buf bytes.Buffer + marshal(&buf, parsed) + return buf.String() +} + +// marshall is mimicking the output of the compact text marshaller +// from https://github.com/golang/protobuf/blob/master/proto/text.go. It +// can't be exactly the same because it works on the result of json.Unmarshal +// into generic types (map, array, int/string/bool). +func marshal(out io.Writer, parsed interface{}) { + switch parsed := parsed.(type) { + case map[string]interface{}: + out.Write([]byte("<")) + var keys []string + for k := range parsed { + keys = append(keys, k) + } + // Ensure consistent alphabetic ordering. + // Ordering by field number would be nicer, but we don't + // have that information here. + sort.Strings(keys) + for i, k := range keys { + if isFieldName(k) { + // No quotation marks round simple + // strings that are likely to be field + // names. We can't be sure 100%, + // because both structs and real maps + // end up as maps after + // json.Unmarshal. + out.Write([]byte(k)) + } else { + out.Write([]byte(fmt.Sprintf("%q", k))) + } + out.Write([]byte(":")) + marshal(out, parsed[k]) + // Avoid redundant space after last element. + if i+1 < len(keys) { + out.Write([]byte(" ")) + } + } + out.Write([]byte(">")) + case []interface{}: + // gRPC uses < for repeating elements. We use + // [ ] because it is a bit more readable. + out.Write([]byte("[")) + for i, v := range parsed { + marshal(out, v) + // Avoid redundant space after last element. + if i+1 < len(parsed) { + out.Write([]byte(" ")) + } + } + out.Write([]byte("]")) + case string: + fmt.Fprintf(out, "%q", parsed) + default: + fmt.Fprint(out, parsed) } - return string(b) } // isFieldName returns true for strings that start with a-zA-Z and @@ -120,7 +182,7 @@ func (s *stripSecrets) strip(parsed interface{}, msg interface{}) error { if s.isSecretField(field) { // Overwrite only if already set. if _, ok := parsedFields[field.GetName()]; ok { - parsedFields[field.GetName()] = "***stripped***" + parsedFields[field.GetName()] = stripped{} } continue } diff --git a/protosanitizer/protosanitizer_test.go b/protosanitizer/protosanitizer_test.go index 65b1e1d5..63901fd1 100644 --- a/protosanitizer/protosanitizer_test.go +++ b/protosanitizer/protosanitizer_test.go @@ -147,12 +147,12 @@ func TestStripSecrets(t *testing.T) { } cases := []testcase{ - {nil, "null"}, + {nil, ""}, {1, "1"}, {"hello world", `"hello world"`}, {true, "true"}, {false, "false"}, - {&csi.CreateVolumeRequest{}, `{}`}, + {&csi.CreateVolumeRequest{}, `<>`}, // Test case from https://github.com/kubernetes-csi/csi-lib-utils/pull/1#pullrequestreview-180126394. {&csi.CreateVolumeRequest{ Name: "test-volume", @@ -177,13 +177,11 @@ func TestStripSecrets(t *testing.T) { Parameters: map[string]string{"param1": "param1", "param2": "param2"}, VolumeContentSource: &csi.VolumeContentSource{}, AccessibilityRequirements: &csi.TopologyRequirement{}, - }, `{"accessibility_requirements":{},"capacity_range":{"limit_bytes":1024,"required_bytes":1024},"name":"test-volume","parameters":{"param1":"param1","param2":"param2"},"secrets":"***stripped***","volume_capabilities":[{"AccessType":{"Mount":{"fs_type":"ext4","mount_flags":["flag1","flag2","flag3"]}},"access_mode":{"mode":5}}],"volume_content_source":{"Type":null}}`}, - {createVolume, `{"accessibility_requirements":{"requisite":[{"segments":{"foo":"bar","x":"y"}},{"segments":{"a":"b"}}]},"capacity_range":{"required_bytes":1024},"name":"foo","secrets":"***stripped***","volume_capabilities":[{"AccessType":{"Mount":{"fs_type":"ext4"}}}]}`}, - {createVolumeCSI03, `{"accessibility_requirements":{"requisite":[{"segments":{"foo":"bar","x":"y"}},{"segments":{"a":"b"}}]},"capacity_range":{"required_bytes":1024},"controller_create_secrets":"***stripped***","name":"foo","volume_capabilities":[{"AccessType":{"Mount":{"fs_type":"ext4"}}}]}`}, - {&csitest.CreateVolumeRequest{}, `{}`}, + }, ` capacity_range: name:"test-volume" parameters: secrets:***stripped*** volume_capabilities:[> access_mode:>] volume_content_source:>>`}, + {createVolume, `> >]> capacity_range: name:"foo" secrets:***stripped*** volume_capabilities:[>>]>`}, + {createVolumeCSI03, `> >]> capacity_range: controller_create_secrets:***stripped*** name:"foo" volume_capabilities:[>>]>`}, {createVolumeFuture, - `{"capacity_range":{"required_bytes":1024},"maybe_secret_map":{"1":{"AccessType":null,"array_secret":"***stripped***"},"2":{"AccessType":null,"array_secret":"***stripped***"}},"name":"foo","new_secret_int":"***stripped***","seecreets":"***stripped***","volume_capabilities":[{"AccessType":{"Mount":{"fs_type":"ext4"}},"array_secret":"***stripped***"},{"AccessType":null,"array_secret":"***stripped***"}],"volume_content_source":{"Type":{"Volume":{"oneof_secret_field":"***stripped***","volume_id":"abc"}},"nested_secret_field":"***stripped***"}}`, - }, + ` maybe_secret_map:<"1": array_secret:***stripped***> "2": array_secret:***stripped***>> name:"foo" new_secret_int:***stripped*** seecreets:***stripped*** volume_capabilities:[> array_secret:***stripped***> array_secret:***stripped***>] volume_content_source:> nested_secret_field:***stripped***>>`}, } // Message from revised spec as received by a sidecar based on the current spec. @@ -193,7 +191,7 @@ func TestStripSecrets(t *testing.T) { if assert.NoError(t, err, "marshall future message") && assert.NoError(t, proto.Unmarshal(data, unknownFields), "unmarshal with unknown fields") { cases = append(cases, testcase{unknownFields, - `{"capacity_range":{"required_bytes":1024},"name":"foo","secrets":"***stripped***","volume_capabilities":[{"AccessType":{"Mount":{"fs_type":"ext4"}}},{"AccessType":null}],"volume_content_source":{"Type":{"Volume":{"volume_id":"abc"}}}}`, + ` name:"foo" secrets:***stripped*** volume_capabilities:[>> >] volume_content_source:>>>`, }) }