From c2cef877f41dbd3afbbf73f237e084dfb5d1dd7c Mon Sep 17 00:00:00 2001 From: Jeff Mitchell Date: Thu, 30 Nov 2017 09:43:07 -0500 Subject: [PATCH] Port over some changes --- Makefile | 2 + api/sys_auth.go | 2 + command/auth.go | 4 +- command/auth_enable.go | 9 +++- command/mount.go | 9 +++- command/mounts.go | 6 +-- helper/locksutil/locks.go | 28 +++++++++++++ physical/file/file.go | 15 +++++-- physical/inmem/inmem.go | 7 +++- physical/transactions.go | 10 ++--- physical/types.pb.go | 87 +++++++++++++++++++++++++++++++++++++++ physical/types.proto | 13 ++++++ vault/barrier.go | 3 +- vault/barrier_aes_gcm.go | 12 +++--- vault/testing.go | 23 +++++++++++ 15 files changed, 203 insertions(+), 27 deletions(-) create mode 100644 physical/types.pb.go create mode 100644 physical/types.proto diff --git a/Makefile b/Makefile index ccb556ec2e99..4e228c19ca29 100644 --- a/Makefile +++ b/Makefile @@ -82,8 +82,10 @@ proto: protoc -I helper/forwarding -I vault -I ../../.. vault/*.proto --go_out=plugins=grpc:vault protoc -I helper/storagepacker helper/storagepacker/types.proto --go_out=plugins=grpc:helper/storagepacker protoc -I helper/forwarding -I vault -I ../../.. helper/forwarding/types.proto --go_out=plugins=grpc:helper/forwarding + protoc -I physical physical/types.proto --go_out=plugins=grpc:physical protoc -I helper/identity -I ../../.. helper/identity/types.proto --go_out=plugins=grpc:helper/identity sed -i -e 's/Idp/IDP/' -e 's/Url/URL/' -e 's/Id/ID/' -e 's/EntityId/EntityID/' -e 's/Api/API/' -e 's/Qr/QR/' -e 's/protobuf:"/sentinel:"" protobuf:"/' helper/identity/types.pb.go helper/storagepacker/types.pb.go + sed -i -e 's/Iv/IV/' -e 's/Hmac/HMAC/' physical/types.pb.go fmtcheck: @sh -c "'$(CURDIR)/scripts/gofmtcheck.sh'" diff --git a/api/sys_auth.go b/api/sys_auth.go index 32f4bbddc058..7b8674bb8bb9 100644 --- a/api/sys_auth.go +++ b/api/sys_auth.go @@ -87,6 +87,7 @@ type EnableAuthOptions struct { Config AuthConfigInput `json:"config" structs:"config"` Local bool `json:"local" structs:"local"` PluginName string `json:"plugin_name,omitempty" structs:"plugin_name,omitempty"` + SealWrap bool `json:"seal_wrap" structs:"seal_wrap" mapstructure:"seal_wrap"` } type AuthConfigInput struct { @@ -99,6 +100,7 @@ type AuthMount struct { Accessor string `json:"accessor" structs:"accessor" mapstructure:"accessor"` Config AuthConfigOutput `json:"config" structs:"config" mapstructure:"config"` Local bool `json:"local" structs:"local" mapstructure:"local"` + SealWrap bool `json:"seal_wrap" structs:"seal_wrap" mapstructure:"seal_wrap"` } type AuthConfigOutput struct { diff --git a/command/auth.go b/command/auth.go index 00b21ce41461..5beabff5cbd2 100644 --- a/command/auth.go +++ b/command/auth.go @@ -377,7 +377,7 @@ func (c *AuthCommand) listMethods() int { } sort.Strings(paths) - columns := []string{"Path | Type | Accessor | Default TTL | Max TTL | Replication Behavior | Description"} + columns := []string{"Path | Type | Accessor | Default TTL | Max TTL | Replication Behavior | Seal Wrap | Description"} for _, path := range paths { auth := auth[path] defTTL := "system" @@ -393,7 +393,7 @@ func (c *AuthCommand) listMethods() int { replicatedBehavior = "local" } columns = append(columns, fmt.Sprintf( - "%s | %s | %s | %s | %s | %s | %s", path, auth.Type, auth.Accessor, defTTL, maxTTL, replicatedBehavior, auth.Description)) + "%s | %s | %s | %s | %s | %s | %t | %s", path, auth.Type, auth.Accessor, defTTL, maxTTL, replicatedBehavior, auth.SealWrap, auth.Description)) } c.Ui.Output(columnize.SimpleFormat(columns)) diff --git a/command/auth_enable.go b/command/auth_enable.go index e6b7f20f24f1..09227a749d9b 100644 --- a/command/auth_enable.go +++ b/command/auth_enable.go @@ -16,12 +16,13 @@ type AuthEnableCommand struct { func (c *AuthEnableCommand) Run(args []string) int { var description, path, pluginName string - var local bool + var local, sealWrap bool flags := c.Meta.FlagSet("auth-enable", meta.FlagSetDefault) flags.StringVar(&description, "description", "", "") flags.StringVar(&path, "path", "", "") flags.StringVar(&pluginName, "plugin-name", "", "") flags.BoolVar(&local, "local", false, "") + flags.BoolVar(&sealWrap, "seal-wrap", false, "") flags.Usage = func() { c.Ui.Error(c.Help()) } if err := flags.Parse(args); err != nil { return 1 @@ -60,7 +61,8 @@ func (c *AuthEnableCommand) Run(args []string) int { Config: api.AuthConfigInput{ PluginName: pluginName, }, - Local: local, + Local: local, + SealWrap: sealWrap, }); err != nil { c.Ui.Error(fmt.Sprintf( "Error: %s", err)) @@ -110,6 +112,8 @@ Auth Enable Options: -local Mark the mount as a local mount. Local mounts are not replicated nor (if a secondary) removed by replication. + + -seal-wrap Turn on seal wrapping for the mount. ` return strings.TrimSpace(helpText) } @@ -137,5 +141,6 @@ func (c *AuthEnableCommand) AutocompleteFlags() complete.Flags { "-path": complete.PredictNothing, "-plugin-name": complete.PredictNothing, "-local": complete.PredictNothing, + "-seal-wrap": complete.PredictNothing, } } diff --git a/command/mount.go b/command/mount.go index 895e7b8bc0f7..b97392fa33f7 100644 --- a/command/mount.go +++ b/command/mount.go @@ -16,7 +16,7 @@ type MountCommand struct { func (c *MountCommand) Run(args []string) int { var description, path, defaultLeaseTTL, maxLeaseTTL, pluginName string - var local, forceNoCache bool + var local, forceNoCache, sealWrap bool flags := c.Meta.FlagSet("mount", meta.FlagSetDefault) flags.StringVar(&description, "description", "", "") flags.StringVar(&path, "path", "", "") @@ -25,6 +25,7 @@ func (c *MountCommand) Run(args []string) int { flags.StringVar(&pluginName, "plugin-name", "", "") flags.BoolVar(&forceNoCache, "force-no-cache", false, "") flags.BoolVar(&local, "local", false, "") + flags.BoolVar(&sealWrap, "seal-wrap", false, "") flags.Usage = func() { c.Ui.Error(c.Help()) } if err := flags.Parse(args); err != nil { return 1 @@ -66,7 +67,8 @@ func (c *MountCommand) Run(args []string) int { ForceNoCache: forceNoCache, PluginName: pluginName, }, - Local: local, + Local: local, + SealWrap: sealWrap, } if err := client.Sys().Mount(path, mountInfo); err != nil { @@ -131,6 +133,8 @@ Mount Options: -local Mark the mount as a local mount. Local mounts are not replicated nor (if a secondary) removed by replication. + + -seal-wrap Turn on seal wrapping for the mount. ` return strings.TrimSpace(helpText) } @@ -160,5 +164,6 @@ func (c *MountCommand) AutocompleteFlags() complete.Flags { "-force-no-cache": complete.PredictNothing, "-plugin-name": complete.PredictNothing, "-local": complete.PredictNothing, + "-seal-wrap": complete.PredictNothing, } } diff --git a/command/mounts.go b/command/mounts.go index 403d9d4d912c..b14bbd832000 100644 --- a/command/mounts.go +++ b/command/mounts.go @@ -42,7 +42,7 @@ func (c *MountsCommand) Run(args []string) int { } sort.Strings(paths) - columns := []string{"Path | Type | Accessor | Plugin | Default TTL | Max TTL | Force No Cache | Replication Behavior | Description"} + columns := []string{"Path | Type | Accessor | Plugin | Default TTL | Max TTL | Force No Cache | Replication Behavior | Seal Wrap | Description"} for _, path := range paths { mount := mounts[path] pluginName := "n/a" @@ -70,8 +70,8 @@ func (c *MountsCommand) Run(args []string) int { replicatedBehavior = "local" } columns = append(columns, fmt.Sprintf( - "%s | %s | %s | %s | %s | %s | %v | %s | %s", path, mount.Type, mount.Accessor, pluginName, defTTL, maxTTL, - mount.Config.ForceNoCache, replicatedBehavior, mount.Description)) + "%s | %s | %s | %s | %s | %s | %v | %s | %t | %s", path, mount.Type, mount.Accessor, pluginName, defTTL, maxTTL, + mount.Config.ForceNoCache, replicatedBehavior, mount.SealWrap, mount.Description)) } c.Ui.Output(columnize.SimpleFormat(columns)) diff --git a/helper/locksutil/locks.go b/helper/locksutil/locks.go index dcf1b4b82d93..8561318dfb1a 100644 --- a/helper/locksutil/locks.go +++ b/helper/locksutil/locks.go @@ -13,6 +13,18 @@ type LockEntry struct { sync.RWMutex } +// CreateLocks returns an array so that the locks can be itterated over in +// order. +// +// This is only threadsafe if a process is using a single lock, or iterating +// over the entire lock slice in order. Using a consistant order avoids +// deadlocks because you can never have the following: +// +// Lock A, Lock B +// Lock B, Lock A +// +// Where process 1 is now deadlocked trying to lock B, and process 2 deadlocked trying to lock A +// func CreateLocks() []*LockEntry { ret := make([]*LockEntry, LockCount) for i := range ret { @@ -30,3 +42,19 @@ func LockIndexForKey(key string) uint8 { func LockForKey(locks []*LockEntry, key string) *LockEntry { return locks[LockIndexForKey(key)] } + +func LocksForKeys(locks []*LockEntry, keys []string) []*LockEntry { + lockIndexes := make(map[uint8]struct{}, len(keys)) + for _, k := range keys { + lockIndexes[LockIndexForKey(k)] = struct{}{} + } + + locksToReturn := make([]*LockEntry, 0, len(keys)) + for i, l := range locks { + if _, ok := lockIndexes[uint8(i)]; ok { + locksToReturn = append(locksToReturn, l) + } + } + + return locksToReturn +} diff --git a/physical/file/file.go b/physical/file/file.go index f7f51928af61..18eaee9d3d35 100644 --- a/physical/file/file.go +++ b/physical/file/file.go @@ -34,6 +34,10 @@ type TransactionalFileBackend struct { FileBackend } +type fileEntry struct { + Value []byte +} + // NewFileBackend constructs a FileBackend using the given directory func NewFileBackend(conf map[string]string, logger log.Logger) (physical.Backend, error) { path, ok := conf["path"] @@ -163,12 +167,15 @@ func (b *FileBackend) GetInternal(k string) (*physical.Entry, error) { return nil, err } - var entry physical.Entry + var entry fileEntry if err := jsonutil.DecodeJSONFromReader(f, &entry); err != nil { return nil, err } - return &entry, nil + return &physical.Entry{ + Key: k, + Value: entry.Value, + }, nil } func (b *FileBackend) Put(entry *physical.Entry) error { @@ -205,7 +212,9 @@ func (b *FileBackend) PutInternal(entry *physical.Entry) error { return err } enc := json.NewEncoder(f) - return enc.Encode(entry) + return enc.Encode(&fileEntry{ + Value: entry.Value, + }) } func (b *FileBackend) List(prefix string) ([]string, error) { diff --git a/physical/inmem/inmem.go b/physical/inmem/inmem.go index 7bbf1801a855..9f478e9ffe35 100644 --- a/physical/inmem/inmem.go +++ b/physical/inmem/inmem.go @@ -59,7 +59,7 @@ func (i *InmemBackend) Put(entry *physical.Entry) error { } func (i *InmemBackend) PutInternal(entry *physical.Entry) error { - i.root.Insert(entry.Key, entry) + i.root.Insert(entry.Key, entry.Value) return nil } @@ -76,7 +76,10 @@ func (i *InmemBackend) Get(key string) (*physical.Entry, error) { func (i *InmemBackend) GetInternal(key string) (*physical.Entry, error) { if raw, ok := i.root.Get(key); ok { - return raw.(*physical.Entry), nil + return &physical.Entry{ + Key: key, + Value: raw.([]byte), + }, nil } return nil, nil } diff --git a/physical/transactions.go b/physical/transactions.go index 81882df8afca..006840905fdc 100644 --- a/physical/transactions.go +++ b/physical/transactions.go @@ -52,9 +52,8 @@ TxnWalk: rollbackEntry := &TxnEntry{ Operation: PutOperation, Entry: &Entry{ - Key: entry.Key, - Value: entry.Value, - SealWrap: entry.SealWrap, + Key: entry.Key, + Value: entry.Value, }, } err = t.DeleteInternal(txn.Entry.Key) @@ -85,9 +84,8 @@ TxnWalk: rollbackEntry = &TxnEntry{ Operation: PutOperation, Entry: &Entry{ - Key: entry.Key, - Value: entry.Value, - SealWrap: entry.SealWrap, + Key: entry.Key, + Value: entry.Value, }, } } diff --git a/physical/types.pb.go b/physical/types.pb.go new file mode 100644 index 000000000000..ccfa2ec08c0f --- /dev/null +++ b/physical/types.pb.go @@ -0,0 +1,87 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: types.proto + +/* +Package physical is a generated protocol buffer package. + +It is generated from these files: + types.proto + +It has these top-level messages: + SealWrapEntry +*/ +package physical + +import proto "github.com/golang/protobuf/proto" +import fmt "fmt" +import math "math" + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package + +type SealWrapEntry struct { + Ciphertext []byte `protobuf:"bytes,1,opt,name=ciphertext,proto3" json:"ciphertext,omitempty"` + IV []byte `protobuf:"bytes,2,opt,name=iv,proto3" json:"iv,omitempty"` + HMAC []byte `protobuf:"bytes,3,opt,name=hmac,proto3" json:"hmac,omitempty"` + Wrapped bool `protobuf:"varint,4,opt,name=wrapped" json:"wrapped,omitempty"` +} + +func (m *SealWrapEntry) Reset() { *m = SealWrapEntry{} } +func (m *SealWrapEntry) String() string { return proto.CompactTextString(m) } +func (*SealWrapEntry) ProtoMessage() {} +func (*SealWrapEntry) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } + +func (m *SealWrapEntry) GetCiphertext() []byte { + if m != nil { + return m.Ciphertext + } + return nil +} + +func (m *SealWrapEntry) GetIV() []byte { + if m != nil { + return m.IV + } + return nil +} + +func (m *SealWrapEntry) GetHMAC() []byte { + if m != nil { + return m.HMAC + } + return nil +} + +func (m *SealWrapEntry) GetWrapped() bool { + if m != nil { + return m.Wrapped + } + return false +} + +func init() { + proto.RegisterType((*SealWrapEntry)(nil), "physical.SealWrapEntry") +} + +func init() { proto.RegisterFile("types.proto", fileDescriptor0) } + +var fileDescriptor0 = []byte{ + // 138 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x2e, 0xa9, 0x2c, 0x48, + 0x2d, 0xd6, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x28, 0xc8, 0xa8, 0x2c, 0xce, 0x4c, 0x4e, + 0xcc, 0x51, 0xca, 0xe5, 0xe2, 0x0d, 0x4e, 0x4d, 0xcc, 0x09, 0x2f, 0x4a, 0x2c, 0x70, 0xcd, 0x2b, + 0x29, 0xaa, 0x14, 0x92, 0xe3, 0xe2, 0x4a, 0xce, 0x2c, 0xc8, 0x48, 0x2d, 0x2a, 0x49, 0xad, 0x28, + 0x91, 0x60, 0x54, 0x60, 0xd4, 0xe0, 0x09, 0x42, 0x12, 0x11, 0xe2, 0xe3, 0x62, 0xca, 0x2c, 0x93, + 0x60, 0x02, 0x8b, 0x33, 0x65, 0x96, 0x09, 0x09, 0x71, 0xb1, 0x64, 0xe4, 0x26, 0x26, 0x4b, 0x30, + 0x83, 0x45, 0xc0, 0x6c, 0x21, 0x09, 0x2e, 0xf6, 0xf2, 0xa2, 0xc4, 0x82, 0x82, 0xd4, 0x14, 0x09, + 0x16, 0x05, 0x46, 0x0d, 0x8e, 0x20, 0x18, 0x37, 0x89, 0x0d, 0x6c, 0xbf, 0x31, 0x20, 0x00, 0x00, + 0xff, 0xff, 0x8b, 0xab, 0x5f, 0x50, 0x8e, 0x00, 0x00, 0x00, +} diff --git a/physical/types.proto b/physical/types.proto new file mode 100644 index 000000000000..43eac0bf0237 --- /dev/null +++ b/physical/types.proto @@ -0,0 +1,13 @@ +syntax = "proto3"; + +package physical; + +message SealWrapEntry { + bytes ciphertext = 1; + + bytes iv = 2; + + bytes hmac = 3; + + bool wrapped = 4; +} diff --git a/vault/barrier.go b/vault/barrier.go index f8474616f0f7..8faa3459b89c 100644 --- a/vault/barrier.go +++ b/vault/barrier.go @@ -30,7 +30,8 @@ const ( // keyringPath is the location of the keyring data. This is encrypted // by the master key. - keyringPath = "core/keyring" + keyringPath = "core/keyring" + keyringPrefix = "core/" // keyringUpgradePrefix is the path used to store keyring update entries. // When running in HA mode, the active instance will install the new key diff --git a/vault/barrier_aes_gcm.go b/vault/barrier_aes_gcm.go index e8053d291c28..429e524944b8 100644 --- a/vault/barrier_aes_gcm.go +++ b/vault/barrier_aes_gcm.go @@ -13,6 +13,7 @@ import ( "github.com/armon/go-metrics" "github.com/hashicorp/vault/helper/jsonutil" + "github.com/hashicorp/vault/helper/strutil" "github.com/hashicorp/vault/physical" ) @@ -79,16 +80,16 @@ func NewAESGCMBarrier(physical physical.Backend) (*AESGCMBarrier, error) { // and has a master key set. func (b *AESGCMBarrier) Initialized() (bool, error) { // Read the keyring file - out, err := b.backend.Get(keyringPath) + keys, err := b.backend.List(keyringPrefix) if err != nil { return false, fmt.Errorf("failed to check for initialization: %v", err) } - if out != nil { + if strutil.StrListContains(keys, "keyring") { return true, nil } // Fallback, check for the old sentinel file - out, err = b.backend.Get(barrierInitPath) + out, err := b.backend.Get(barrierInitPath) if err != nil { return false, fmt.Errorf("failed to check for initialization: %v", err) } @@ -490,9 +491,8 @@ func (b *AESGCMBarrier) CreateUpgrade(term uint32) error { value := b.encrypt(key, prevTerm, primary, buf) // Create upgrade key pe := &physical.Entry{ - Key: key, - Value: value, - SealWrap: true, + Key: key, + Value: value, } return b.backend.Put(pe) } diff --git a/vault/testing.go b/vault/testing.go index 97028a02f899..f6ac69c90547 100644 --- a/vault/testing.go +++ b/vault/testing.go @@ -760,6 +760,29 @@ func (c *TestCluster) ensureCoresSealed() error { return nil } +func (c *TestCluster) UnsealWithStoredKeys(t testing.T) error { + for _, core := range c.Cores { + if err := core.UnsealWithStoredKeys(); err != nil { + return err + } + timeout := time.Now().Add(60 * time.Second) + for { + if time.Now().After(timeout) { + return fmt.Errorf("timeout waiting for core to unseal") + } + sealed, err := core.Sealed() + if err != nil { + return err + } + if !sealed { + break + } + time.Sleep(250 * time.Millisecond) + } + } + return nil +} + type TestListener struct { net.Listener Address *net.TCPAddr