From 5b4e674da3ddfc9f9ca90038d00118036af5a80d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?No=C3=A9mi=20V=C3=A1nyi?= Date: Thu, 3 Mar 2022 17:00:15 +0100 Subject: [PATCH] Move `github.com/elastic/beats/v7/libbeat/keystore` and `github.com/elastic/beats/v7/libbeat/common/file.FileInfo` (#20) --- README.md | 1 + file/fileinfo.go | 72 +++++ file/fileinfo_test.go | 96 +++++++ file/fileinfo_unix.go | 47 ++++ file/fileinfo_windows.go | 35 +++ go.mod | 1 + go.sum | 1 + keystore/config.go | 29 ++ keystore/file_keystore.go | 480 +++++++++++++++++++++++++++++++++ keystore/file_keystore_test.go | 372 +++++++++++++++++++++++++ keystore/keystore.go | 117 ++++++++ keystore/keystore_test.go | 52 ++++ keystore/secure_string.go | 49 ++++ keystore/secure_string_test.go | 55 ++++ 14 files changed, 1407 insertions(+) create mode 100644 file/fileinfo.go create mode 100644 file/fileinfo_test.go create mode 100644 file/fileinfo_unix.go create mode 100644 file/fileinfo_windows.go create mode 100644 keystore/config.go create mode 100644 keystore/file_keystore.go create mode 100644 keystore/file_keystore_test.go create mode 100644 keystore/keystore.go create mode 100644 keystore/keystore_test.go create mode 100644 keystore/secure_string.go create mode 100644 keystore/secure_string_test.go diff --git a/README.md b/README.md index c383130..861919b 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ Provided packages: * `github.com/elastic/elastic-agent-libs/cloudid` is used for parsing `cloud.id` and `cloud.auth` when connecting to the Elastic stack. * `github.com/elastic/elastic-agent-libs/config` the previous `config.go` file from `github.com/elastic/beats/v7/libbeat/common`. A minimal wrapper around `github.com/elastic/go-ucfg`. It contains helpers for merging and accessing configuration objects and flags. * `github.com/elastic/elastic-agent-libs/file` is responsible for rotating and writing input and output files. +* `github.com/elastic/elastic-agent-libs/keystore` interface for keystores and file keystore implementation. * `github.com/elastic/elastic-agent-libs/logp` is the well known logger from libbeat. * `github.com/elastic/elastic-agent-libs/logp/cfgwarn` provides logging utilities for warning users about deprecated settings. * `github.com/elastic/elastic-agent-libs/mapstr` is the old `github.com/elastic/beats/v7/libbeat/common.MapStr`. It is an extra layer on top of `map[string]interface{}`. diff --git a/file/fileinfo.go b/file/fileinfo.go new file mode 100644 index 0000000..c1a8ce6 --- /dev/null +++ b/file/fileinfo.go @@ -0,0 +1,72 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package file + +import ( + "errors" + "os" +) + +// A FileInfo describes a file and is returned by Stat and Lstat. +type FileInfo interface { + os.FileInfo + UID() (int, error) // UID of the file owner. Returns an error on non-POSIX file systems. + GID() (int, error) // GID of the file owner. Returns an error on non-POSIX file systems. +} + +// Stat returns a FileInfo describing the named file. +// If there is an error, it will be of type *PathError. +func Stat(name string) (FileInfo, error) { + return stat(name, os.Stat) +} + +// Lstat returns a FileInfo describing the named file. +// If the file is a symbolic link, the returned FileInfo +// describes the symbolic link. Lstat makes no attempt to follow the link. +// If there is an error, it will be of type *PathError. +func Lstat(name string) (FileInfo, error) { + return stat(name, os.Lstat) +} + +// Wrap wraps the given os.FileInfo and returns a FileInfo in order to expose +// the UID and GID in a uniform manner across operating systems. +func Wrap(info os.FileInfo) (FileInfo, error) { + return wrap(info) +} + +type fileInfo struct { + os.FileInfo + uid *int + gid *int +} + +func (f fileInfo) UID() (int, error) { + if f.uid == nil { + return -1, errors.New("uid not implemented") + } + + return *f.uid, nil +} + +func (f fileInfo) GID() (int, error) { + if f.gid == nil { + return -1, errors.New("gid not implemented") + } + + return *f.gid, nil +} diff --git a/file/fileinfo_test.go b/file/fileinfo_test.go new file mode 100644 index 0000000..d1aacb5 --- /dev/null +++ b/file/fileinfo_test.go @@ -0,0 +1,96 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build !windows && !openbsd +// +build !windows,!openbsd + +// Test for openbsd are excluded here as info.GID() returns 0 instead of the actual value +// As the code does not seem to be used in any of the beats, this should be ok +// Still it would be interesting to know why it returns 0. + +package file_test + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/elastic/elastic-agent-libs/file" +) + +func TestStat(t *testing.T) { + f, err := ioutil.TempFile("", "teststat") + if err != nil { + t.Fatal(err) + } + defer os.Remove(f.Name()) + + link := filepath.Join(os.TempDir(), "teststat-link") + if err := os.Symlink(f.Name(), link); err != nil { + t.Fatal(err) + } + defer os.Remove(link) + + info, err := file.Stat(link) + if err != nil { + t.Fatal(err) + } + + require.True(t, info.Mode().IsRegular()) + + uid, err := info.UID() + if err != nil { + t.Fatal(err) + } + require.EqualValues(t, os.Geteuid(), uid) + + gid, err := info.GID() + if err != nil { + t.Fatal(err) + } + require.EqualValues(t, os.Getegid(), gid) +} + +func TestLstat(t *testing.T) { + link := filepath.Join(os.TempDir(), "link") + if err := os.Symlink("dummy", link); err != nil { + t.Fatal(err) + } + defer os.Remove(link) + + info, err := file.Lstat(link) + if err != nil { + t.Fatal(err) + } + + require.True(t, info.Mode()&os.ModeSymlink > 0) + + uid, err := info.UID() + if err != nil { + t.Fatal(err) + } + require.EqualValues(t, os.Geteuid(), uid) + + gid, err := info.GID() + if err != nil { + t.Fatal(err) + } + require.EqualValues(t, os.Getegid(), gid) +} diff --git a/file/fileinfo_unix.go b/file/fileinfo_unix.go new file mode 100644 index 0000000..9a0ce88 --- /dev/null +++ b/file/fileinfo_unix.go @@ -0,0 +1,47 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +//go:build !windows +// +build !windows + +package file + +import ( + "errors" + "os" + "syscall" +) + +func stat(name string, statFunc func(name string) (os.FileInfo, error)) (FileInfo, error) { + info, err := statFunc(name) + if err != nil { + return nil, err + } + + return wrap(info) +} + +func wrap(info os.FileInfo) (FileInfo, error) { + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return nil, errors.New("failed to get uid/gid") + } + + uid := int(stat.Uid) + gid := int(stat.Gid) + return fileInfo{FileInfo: info, uid: &uid, gid: &gid}, nil +} diff --git a/file/fileinfo_windows.go b/file/fileinfo_windows.go new file mode 100644 index 0000000..855eb70 --- /dev/null +++ b/file/fileinfo_windows.go @@ -0,0 +1,35 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package file + +import ( + "os" +) + +func stat(name string, statFunc func(name string) (os.FileInfo, error)) (FileInfo, error) { + info, err := statFunc(name) + if err != nil { + return nil, err + } + + return wrap(info) +} + +func wrap(info os.FileInfo) (FileInfo, error) { + return fileInfo{FileInfo: info}, nil +} diff --git a/go.mod b/go.mod index ec72d6b..362fee1 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/stretchr/testify v1.7.0 go.elastic.co/ecszap v1.0.0 go.uber.org/zap v1.21.0 + golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 golang.org/x/sys v0.0.0-20220209214540-3681064d5158 ) diff --git a/go.sum b/go.sum index 1038815..c1ff1f7 100644 --- a/go.sum +++ b/go.sum @@ -384,6 +384,7 @@ golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5 h1:HWj/xjIHfjYU5nVXpTM0s39J9CbLn7Cc5a7IC5rwsMQ= golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= diff --git a/keystore/config.go b/keystore/config.go new file mode 100644 index 0000000..724795a --- /dev/null +++ b/keystore/config.go @@ -0,0 +1,29 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package keystore + +// Config Define keystore configurable options +type Config struct { + Path string `config:"path"` +} + +func defaultConfig() Config { + return Config{ + Path: "", + } +} diff --git a/keystore/file_keystore.go b/keystore/file_keystore.go new file mode 100644 index 0000000..00c5226 --- /dev/null +++ b/keystore/file_keystore.go @@ -0,0 +1,480 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package keystore + +import ( + "bytes" + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "crypto/sha512" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + "runtime" + "sync" + + "golang.org/x/crypto/pbkdf2" + + "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/elastic-agent-libs/file" +) + +const ( + filePermission = 0600 + + // Encryption Related constants + iVLength = 12 + saltLength = 64 + iterationsCount = 10000 + keyLength = 32 +) + +// Version of the keystore format, will be added at the beginning of the file. +var version = []byte("v1") + +// Packager defines a keystore that we can read the raw bytes and be packaged in an artifact. +type Packager interface { + Package() ([]byte, error) + ConfiguredPath() string +} + +// FileKeystore Allows to store key / secrets pair securely into an encrypted local file. +type FileKeystore struct { + sync.RWMutex + Path string + secrets map[string]serializableSecureString + dirty bool + password *SecureString + isStrictPerms bool +} + +// Allow the original SecureString type to be correctly serialized to json. +type serializableSecureString struct { + *SecureString + Value []byte `json:"value"` +} + +// Factory Create the right keystore with the configured options. +func Factory(c *config.C, defaultPath string, strictPerms bool) (Keystore, error) { + cfg := defaultConfig() + + if c == nil { + c = config.NewConfig() + } + err := c.Unpack(&cfg) + + if err != nil { + return nil, fmt.Errorf("could not read keystore configuration, err: %w", err) + } + + if cfg.Path == "" { + cfg.Path = defaultPath + } + + keystore, err := NewFileKeystoreWithStrictPerms(cfg.Path, strictPerms) + return keystore, err +} + +// NewFileKeystore returns an new File based keystore or an error, currently users cannot set their +// own password on the keystore, the default password will be an empty string. When the keystore +// is initialized the secrets are automatically loaded into memory. +func NewFileKeystore(keystoreFile string) (Keystore, error) { + return NewFileKeystoreWithStrictPerms(keystoreFile, false) +} + +// NewFileKeystore returns an new File based keystore or an error, currently users cannot set their +// own password on the keystore, the default password will be an empty string. When the keystore +// is initialized the secrets are automatically loaded into memory. +func NewFileKeystoreWithStrictPerms(keystoreFile string, strictPerms bool) (Keystore, error) { + return NewFileKeystoreWithPasswordAndStrictPerms(keystoreFile, NewSecureString([]byte("")), strictPerms) +} + +// NewFileKeystoreWithPassword return a new File based keystore or an error, allow to define what +// password to use to create the keystore. +func NewFileKeystoreWithPasswordAndStrictPerms(keystoreFile string, password *SecureString, strictPerms bool) (Keystore, error) { + keystore := FileKeystore{ + Path: keystoreFile, + dirty: false, + password: password, + secrets: make(map[string]serializableSecureString), + isStrictPerms: strictPerms, + } + + err := keystore.load() + if err != nil { + return nil, err + } + + return &keystore, nil +} + +// NewFileKeystoreWithPassword return a new File based keystore or an error, allow to define what +// password to use to create the keystore. +func NewFileKeystoreWithPassword(keystoreFile string, password *SecureString) (Keystore, error) { + return NewFileKeystoreWithPasswordAndStrictPerms(keystoreFile, password, false) +} + +// Retrieve return a SecureString instance that will contains both the key and the secret. +func (k *FileKeystore) Retrieve(key string) (*SecureString, error) { + k.RLock() + defer k.RUnlock() + + secret, ok := k.secrets[key] + if !ok { + return nil, ErrKeyDoesntExists + } + return NewSecureString(secret.Value), nil +} + +// Store add the key pair to the secret store and mark the store as dirty. +func (k *FileKeystore) Store(key string, value []byte) error { + k.Lock() + defer k.Unlock() + + k.secrets[key] = serializableSecureString{Value: value} + k.dirty = true + return nil +} + +// Delete an existing key from the store and mark the store as dirty. +func (k *FileKeystore) Delete(key string) error { + k.Lock() + defer k.Unlock() + + delete(k.secrets, key) + k.dirty = true + return nil +} + +// Save persists the in memory data to disk if needed. +func (k *FileKeystore) Save() error { + k.Lock() + err := k.doSave(true) + k.Unlock() + return err +} + +// List return the availables keys. +func (k *FileKeystore) List() ([]string, error) { + k.RLock() + defer k.RUnlock() + + keys := make([]string, 0, len(k.secrets)) + for key := range k.secrets { + keys = append(keys, key) + } + + return keys, nil +} + +// GetConfig returns config.C representation of the key / secret pair to be merged with other +// loaded configuration. +func (k *FileKeystore) GetConfig() (*config.C, error) { + k.RLock() + defer k.RUnlock() + + configHash := make(map[string]interface{}) + for key, secret := range k.secrets { + configHash[key] = string(secret.Value) + } + + return config.NewConfigFrom(configHash) +} + +// Create create an empty keystore, if the store already exist we will return an error. +func (k *FileKeystore) Create(override bool) error { + k.Lock() + k.secrets = make(map[string]serializableSecureString) + k.dirty = true + err := k.doSave(override) + k.Unlock() + return err +} + +// IsPersisted return if the keystore is physically persisted on disk. +func (k *FileKeystore) IsPersisted() bool { + k.Lock() + defer k.Unlock() + + // We just check if the file is present on disk, we don't need to do any validation + // for a file based keystore, since all the keys will be fetched when we initialize the object + // if the file is invalid it will already fails. Creating a new FileKeystore will raise + // any errors concerning the permissions + f, err := os.OpenFile(k.Path, os.O_RDONLY, filePermission) + if err != nil { + return false + } + f.Close() + return true +} + +// doSave lock/unlocking of the resource need to be done by the caller. +func (k *FileKeystore) doSave(override bool) error { + if !k.dirty { + return nil + } + + temporaryPath := fmt.Sprintf("%s.tmp", k.Path) + + w := new(bytes.Buffer) + jsonEncoder := json.NewEncoder(w) + if err := jsonEncoder.Encode(k.secrets); err != nil { + return fmt.Errorf("cannot serialize the keystore before saving it to disk: %w", err) + } + + encrypted, err := k.encrypt(w) + if err != nil { + return fmt.Errorf("cannot encrypt the keystore: %w", err) + } + + flags := os.O_RDWR | os.O_CREATE + if override { + flags |= os.O_TRUNC + } else { + flags |= os.O_EXCL + } + + f, err := os.OpenFile(temporaryPath, flags, filePermission) + if err != nil { + return fmt.Errorf("cannot open file to save the keystore to '%s', error: %w", k.Path, err) + } + + _, _ = f.Write(version) + base64Encoder := base64.NewEncoder(base64.StdEncoding, f) + _, _ = io.Copy(base64Encoder, encrypted) + base64Encoder.Close() + _ = f.Sync() + f.Close() + + err = file.SafeFileRotate(k.Path, temporaryPath) + if err != nil { + os.Remove(temporaryPath) + return fmt.Errorf("cannot replace the existing keystore, with the new keystore file at '%s', error: %w", k.Path, err) + } + os.Remove(temporaryPath) + + k.dirty = false + return nil +} + +func (k *FileKeystore) loadRaw() ([]byte, error) { + f, err := os.OpenFile(k.Path, os.O_RDONLY, filePermission) + if err != nil { + if os.IsNotExist(err) { + return nil, nil + } + return nil, err + } + defer f.Close() + + if k.isStrictPerms { + if err := k.checkPermissions(k.Path); err != nil { + return nil, err + } + } + + raw, err := ioutil.ReadAll(f) + if err != nil { + return nil, err + } + + v := raw[0:len(version)] + if !bytes.Equal(v, version) { + return nil, fmt.Errorf("keystore format doesn't match expected version: '%s' got '%s'", version, v) + } + + if len(raw) <= len(version) { + return nil, fmt.Errorf("corrupt or empty keystore") + } + + return raw, nil +} + +func (k *FileKeystore) load() error { + k.Lock() + defer k.Unlock() + + raw, err := k.loadRaw() + if err != nil { + return err + } + + if len(raw) == 0 { + return nil + } + + base64Decoder := base64.NewDecoder(base64.StdEncoding, bytes.NewReader(raw[len(version):])) + plaintext, err := k.decrypt(base64Decoder) + if err != nil { + return fmt.Errorf("could not decrypt the keystore: %w", err) + } + jsonDecoder := json.NewDecoder(plaintext) + return jsonDecoder.Decode(&k.secrets) +} + +// Encrypt the data payload using a derived keys and the AES-256-GCM algorithm. +func (k *FileKeystore) encrypt(reader io.Reader) (io.Reader, error) { + // randomly generate the salt and the initialization vector, this information will be saved + // on disk in the file as part of the header + iv, err := randomBytes(iVLength) + + if err != nil { + return nil, err + } + + salt, err := randomBytes(saltLength) + if err != nil { + return nil, err + } + + // Stretch the user provided key + password, _ := k.password.Get() + passwordBytes := k.hashPassword(password, salt) + + // Select AES-256: because len(passwordBytes) == 32 bytes + block, err := aes.NewCipher(passwordBytes) + if err != nil { + return nil, fmt.Errorf("could not create the keystore cipher to encrypt, error: %w", err) + } + + aesgcm, err := cipher.NewGCM(block) + if err != nil { + return nil, fmt.Errorf("could not create the keystore cipher to encrypt, error: %w", err) + } + + data, err := ioutil.ReadAll(reader) + if err != nil { + return nil, fmt.Errorf("could not read unencrypted data, error: %w", err) + } + + encodedBytes := aesgcm.Seal(nil, iv, data, nil) + + // Generate the payload with all the additional information required to decrypt the + // output format of the document: VERSION|SALT|IV|PAYLOAD + buf := bytes.NewBuffer(salt) + buf.Write(iv) + buf.Write(encodedBytes) + + return buf, nil +} + +// should receive an io.reader... +func (k *FileKeystore) decrypt(reader io.Reader) (io.Reader, error) { + data, err := ioutil.ReadAll(reader) + if err != nil { + return nil, fmt.Errorf("could not read all the data from the encrypted file, error: %w", err) + } + + if len(data) < saltLength+iVLength+1 { + return nil, fmt.Errorf("missing information in the file for decrypting the keystore") + } + + // extract the necessary information to decrypt the data from the data payload + salt := data[0:saltLength] + iv := data[saltLength : saltLength+iVLength] + encodedBytes := data[saltLength+iVLength:] + + password, _ := k.password.Get() + passwordBytes := k.hashPassword(password, salt) + + block, err := aes.NewCipher(passwordBytes) + if err != nil { + return nil, fmt.Errorf("could not create the keystore cipher to decrypt the data: %w", err) + } + + aesgcm, err := cipher.NewGCM(block) + if err != nil { + return nil, fmt.Errorf("could not create the keystore cipher to decrypt the data: %w", err) + } + + decodedBytes, err := aesgcm.Open(nil, iv, encodedBytes, nil) + if err != nil { + return nil, fmt.Errorf("could not decrypt keystore data: %w", err) + } + + return bytes.NewReader(decodedBytes), nil +} + +// checkPermission enforces permission on the keystore file itself, the file should have strict +// permission (0600) and the keystore should refuses to start if its not the case. +func (k *FileKeystore) checkPermissions(f string) error { + if runtime.GOOS == "windows" { //nolint: goconst // OS checking in cleaner this way + return nil + } + + info, err := file.Stat(f) + if err != nil { + return err + } + + euid := os.Geteuid() + fileUID, _ := info.UID() + perm := info.Mode().Perm() + + if fileUID != 0 && euid != fileUID { + return fmt.Errorf(`config file ("%v") must be owned by the user identifier `+ + `(uid=%v) or root`, f, euid) + } + + // Test if group or other have write permissions. + if perm != filePermission { + nameAbs, err := filepath.Abs(f) + if err != nil { + nameAbs = f + } + return fmt.Errorf(`file ("%v") can only be writable and readable by the `+ + `owner but the permissions are "%v" (to fix the permissions use: `+ + `'chmod go-wrx %v')`, + f, perm, nameAbs) + } + + return nil +} + +// Package returns the bytes of the encrypted keystore. +func (k *FileKeystore) Package() ([]byte, error) { + k.Lock() + defer k.Unlock() + return k.loadRaw() +} + +// ConfiguredPath returns the path to the keystore. +func (k *FileKeystore) ConfiguredPath() string { + return k.Path +} + +func (k *FileKeystore) hashPassword(password, salt []byte) []byte { + return pbkdf2.Key(password, salt, iterationsCount, keyLength, sha512.New) +} + +// randomBytes return a slice of random bytes of the defined length +func randomBytes(length int) ([]byte, error) { + r := make([]byte, length) + _, err := rand.Read(r) + + if err != nil { + return nil, err + } + + return r, nil +} diff --git a/keystore/file_keystore_test.go b/keystore/file_keystore_test.go new file mode 100644 index 0000000..d14fdbb --- /dev/null +++ b/keystore/file_keystore_test.go @@ -0,0 +1,372 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package keystore + +import ( + "bytes" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "runtime" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var keyValue = "output.elasticsearch.password" +var secretValue = []byte("secret") + +func TestCanCreateAKeyStore(t *testing.T) { + path := GetTemporaryKeystoreFile() + defer os.Remove(path) + + keyStore, err := NewFileKeystore(path) + require.NoError(t, err) + + writableKeystore, err := AsWritableKeystore(keyStore) + require.NoError(t, err) + + require.Nil(t, writableKeystore.Store(keyValue, secretValue)) + require.Nil(t, writableKeystore.Save()) +} + +func TestCanReadAnExistingKeyStoreWithEmptyString(t *testing.T) { + path := GetTemporaryKeystoreFile() + defer os.Remove(path) + + CreateAnExistingKeystore(path) + + keystoreRead, err := NewFileKeystore(path) + require.NoError(t, err) + + secure, err := keystoreRead.Retrieve(keyValue) + require.NoError(t, err) + + v, err := secure.Get() + require.NoError(t, err) + require.Equal(t, v, secretValue) +} + +func TestCanDeleteAKeyFromTheStoreAndPersistChanges(t *testing.T) { + path := GetTemporaryKeystoreFile() + defer os.Remove(path) + + CreateAnExistingKeystore(path) + + keyStore, _ := NewFileKeystore(path) + _, err := keyStore.Retrieve(keyValue) + require.NoError(t, err) + + writableKeystore, err := AsWritableKeystore(keyStore) + require.NoError(t, err) + + err = writableKeystore.Delete(keyValue) + require.NoError(t, err) + _, err = keyStore.Retrieve(keyValue) + require.Error(t, err) + + _ = writableKeystore.Save() + newKeystore, err := NewFileKeystore(path) + require.NoError(t, err) + _, err = newKeystore.Retrieve(keyValue) + require.Error(t, err) +} + +func TestFilePermissionOnCreate(t *testing.T) { + // Skip check on windows + if runtime.GOOS == "windows" { + t.Skip("Permission check is not running on windows") + } + + path := GetTemporaryKeystoreFile() + defer os.Remove(path) + CreateAnExistingKeystore(path) + + stats, err := os.Stat(path) + require.NoError(t, err) + permissions := stats.Mode().Perm() + if permissions != 0600 { + t.Fatalf("Expecting the file what only readable/writable by the owner, permission found: %v", permissions) + } +} + +func TestFilePermissionOnUpdate(t *testing.T) { + // Skip check on windows + if runtime.GOOS == "windows" { + t.Skip("Permission check is not running on windows") + } + + path := GetTemporaryKeystoreFile() + defer os.Remove(path) + + keyStore := CreateAnExistingKeystore(path) + + writableKeystore, err := AsWritableKeystore(keyStore) + require.NoError(t, err) + + err = writableKeystore.Store("newkey", []byte("newsecret")) + require.NoError(t, err) + err = writableKeystore.Save() + require.NoError(t, err) + stats, err := os.Stat(path) + require.NoError(t, err) + permissions := stats.Mode().Perm() + if permissions != 0600 { + t.Fatalf("Expecting the file what only readable/writable by the owner, permission found: %v", permissions) + } +} + +func TestFilePermissionOnLoadWhenStrictIsOn(t *testing.T) { + // Skip check on windows + if runtime.GOOS == "windows" { + t.Skip("Permission check is not running on windows") + } + + path := GetTemporaryKeystoreFile() + defer os.Remove(path) + + // Create a world readable keystore file + fd, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666) + require.NoError(t, err) + _, _ = fd.WriteString("bad permission") + require.NoError(t, fd.Close()) + _, err = NewFileKeystore(path) + require.Error(t, err) +} + +func TestReturnsUsedKeysInTheStore(t *testing.T) { + path := GetTemporaryKeystoreFile() + defer os.Remove(path) + + keyStore := CreateAnExistingKeystore(path) + + listingKeystore, err := AsListingKeystore(keyStore) + require.NoError(t, err) + + keys, err := listingKeystore.List() + + require.NoError(t, err) + require.Equal(t, len(keys), 1) + require.Equal(t, keys[0], keyValue) +} + +func TestCannotDecryptKeyStoreWithWrongPassword(t *testing.T) { + path := GetTemporaryKeystoreFile() + defer os.Remove(path) + + keyStore, err := NewFileKeystoreWithPassword(path, NewSecureString([]byte("password"))) + require.NoError(t, err) + + writableKeystore, err := AsWritableKeystore(keyStore) + require.NoError(t, err) + + err = writableKeystore.Store("hello", []byte("world")) + require.NoError(t, err) + err = writableKeystore.Save() + require.NoError(t, err) + + _, err = NewFileKeystoreWithPassword(path, NewSecureString([]byte("wrongpassword"))) + if assert.Error(t, err, "should fail to decrypt the keystore") { + m := `could not decrypt the keystore: could not decrypt keystore data: ` + + `cipher: message authentication failed` + assert.Equal(t, m, err.Error()) + } +} + +func TestUserDefinedPasswordUTF8(t *testing.T) { + createAndReadKeystoreWithPassword(t, []byte("mysecret¥¥password")) +} + +func TestUserDefinedPasswordASCII(t *testing.T) { + createAndReadKeystoreWithPassword(t, []byte("mysecret")) +} + +func TestSecretWithUTF8EncodedSecret(t *testing.T) { + content := []byte("ありがとうございます") // translation: thank you + createAndReadKeystoreSecret(t, []byte("mysuperpassword"), "mykey", content) +} + +func TestSecretWithASCIIEncodedSecret(t *testing.T) { + content := []byte("good news everyone") // translation: thank you + createAndReadKeystoreSecret(t, []byte("mysuperpassword"), "mykey", content) +} + +func TestGetConfig(t *testing.T) { + path := GetTemporaryKeystoreFile() + defer os.Remove(path) + + keyStore := CreateAnExistingKeystore(path) + + writableKeystore, err := AsWritableKeystore(keyStore) + require.NoError(t, err) + + // Add a bit more data of different type + err = writableKeystore.Store("super.nested", []byte("hello")) + require.NoError(t, err) + err = writableKeystore.Save() + require.NoError(t, err) + + cfg, err := keyStore.GetConfig() + require.NotNil(t, cfg) + require.NoError(t, err) + + secret, err := cfg.String("output.elasticsearch.password", 0) + require.NoError(t, err) + require.Equal(t, secret, "secret") + + port, err := cfg.String("super.nested", 0) + require.NoError(t, err) + require.Equal(t, port, "hello") +} + +func TestShouldRaiseAndErrorWhenVersionDontMatch(t *testing.T) { + temporaryPath := GetTemporaryKeystoreFile() + defer os.Remove(temporaryPath) + + badVersion := `v2D/EQwnDNO7yZsjsRFVWGgbkZudhPxVhBkaQAVud66+tK4HRdfPrNrNNgSmhioDGrQ0z/VZpvbw68gb0G + G2QHxlP5s4HGRU/GQge3Nsnx0+kDIcb/37gPN1D1TOPHSiRrzzPn2vInmgaLUfEgBgoa9tuXLZEKdh3JPh/q` + + f, err := os.OpenFile(temporaryPath, os.O_CREATE|os.O_WRONLY, 0600) + require.NoError(t, err) + _, _ = f.WriteString(badVersion) + err = f.Close() + require.NoError(t, err) + + _, err = NewFileKeystoreWithPassword(temporaryPath, NewSecureString([]byte(""))) + if assert.Error(t, err, "Expect version check error") { + assert.Equal(t, err, fmt.Errorf("keystore format doesn't match expected version: 'v1' got 'v2'")) + } +} + +func TestMissingEncryptedBlock(t *testing.T) { + temporaryPath := GetTemporaryKeystoreFile() + defer os.Remove(temporaryPath) + + badVersion := "v1" + + f, err := os.OpenFile(temporaryPath, os.O_CREATE|os.O_WRONLY, 0600) + require.NoError(t, err) + _, _ = f.WriteString(badVersion) + err = f.Close() + require.NoError(t, err) + + _, err = NewFileKeystoreWithPassword(temporaryPath, NewSecureString([]byte(""))) + if assert.Error(t, err) { + assert.Equal(t, err, fmt.Errorf("corrupt or empty keystore")) + } +} + +func createAndReadKeystoreSecret(t *testing.T, password []byte, key string, value []byte) { + path := GetTemporaryKeystoreFile() + defer os.Remove(path) + + keyStore, err := NewFileKeystoreWithPassword(path, NewSecureString(password)) + require.NoError(t, err) + + writableKeystore, err := AsWritableKeystore(keyStore) + require.NoError(t, err) + + err = writableKeystore.Store(key, value) + require.NoError(t, err) + err = writableKeystore.Save() + require.NoError(t, err) + + newStore, err := NewFileKeystoreWithPassword(path, NewSecureString(password)) + require.NoError(t, err) + s, _ := newStore.Retrieve(key) + v, _ := s.Get() + require.Equal(t, v, value) +} + +func createAndReadKeystoreWithPassword(t *testing.T, password []byte) { + path := GetTemporaryKeystoreFile() + defer os.Remove(path) + + keyStore, err := NewFileKeystoreWithPassword(path, NewSecureString(password)) + require.NoError(t, err) + + writableKeystore, err := AsWritableKeystore(keyStore) + require.NoError(t, err) + + err = writableKeystore.Store("hello", []byte("world")) + require.NoError(t, err) + err = writableKeystore.Save() + require.NoError(t, err) + + newStore, err := NewFileKeystoreWithPassword(path, NewSecureString(password)) + require.NoError(t, err) + s, _ := newStore.Retrieve("hello") + v, _ := s.Get() + + require.Equal(t, v, []byte("world")) +} + +// CreateAnExistingKeystore creates a keystore with an existing key +/// `output.elasticsearch.password` with the value `secret`. +func CreateAnExistingKeystore(path string) Keystore { + keyStore, err := NewFileKeystore(path) + // Fail fast in the test suite + if err != nil { + panic(err) + } + + writableKeystore, err := AsWritableKeystore(keyStore) + if err != nil { + panic(err) + } + + err = writableKeystore.Store(keyValue, secretValue) + if err != nil { + panic(err) + } + err = writableKeystore.Save() + if err != nil { + panic(err) + } + return keyStore +} + +// GetTemporaryKeystoreFile create a temporary file on disk to save the keystore. +func GetTemporaryKeystoreFile() string { + path, err := ioutil.TempDir("", "testing") + if err != nil { + panic(err) + } + return filepath.Join(path, "keystore") +} + +func TestRandomBytesLength(t *testing.T) { + r1, _ := randomBytes(5) + require.Equal(t, len(r1), 5) + + r2, _ := randomBytes(4) + require.Equal(t, len(r2), 4) + require.NotEqual(t, string(r1[:]), string(r2[:])) +} + +func TestRandomBytes(t *testing.T) { + v1, err := randomBytes(10) + require.NoError(t, err) + v2, err := randomBytes(10) + require.NoError(t, err) + + // unlikely to get 2 times the same results + require.False(t, bytes.Equal(v1, v2)) +} diff --git a/keystore/keystore.go b/keystore/keystore.go new file mode 100644 index 0000000..884ab58 --- /dev/null +++ b/keystore/keystore.go @@ -0,0 +1,117 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package keystore + +import ( + "errors" + + "github.com/elastic/elastic-agent-libs/config" + "github.com/elastic/go-ucfg" + "github.com/elastic/go-ucfg/parse" +) + +var ( + // ErrAlreadyExists is returned when the file already exist at the location. + ErrAlreadyExists = errors.New("cannot create a new keystore a valid keystore already exist at the location") + + // ErrKeyDoesntExists is returned when the key doesn't exist in the store + ErrKeyDoesntExists = errors.New("cannot retrieve the key") + + // ErrNotWritable is returned when the keystore is not writable + ErrNotWritable = errors.New("the configured keystore is not writable") + + // ErrNotWritable is returned when the keystore is not writable + ErrNotListing = errors.New("the configured keystore is not listing") +) + +// Keystore implement a way to securely saves and retrieves secrets to be used in the configuration +// Currently all credentials are loaded upfront and are not lazy retrieved, we will eventually move +// to that concept, so we can deal with tokens that has a limited duration or can be revoked by a +// remote keystore. +type Keystore interface { + // Retrieve returns a SecureString instance of the searched key or an error. + Retrieve(key string) (*SecureString, error) + + // GetConfig returns the key value pair in the config format to be merged with other configuration. + GetConfig() (*config.C, error) + + // IsPersisted check if the current keystore is persisted. + IsPersisted() bool +} + +type WritableKeystore interface { + // Store add keys to the keystore, wont be persisted until we save. + Store(key string, secret []byte) error + + // Delete removes a specific key from the keystore. + Delete(key string) error + + // Create Allow to create an empty keystore. + Create(override bool) error + + // Save persist the changes to the keystore. + Save() error +} + +type ListingKeystore interface { + // List returns the list of keys in the keystore, return an empty list if none is found. + List() ([]string, error) +} + +// ResolverWrap wrap a config resolver around an existing keystore. +func ResolverWrap(keystore Keystore) func(string) (string, parse.Config, error) { + return func(keyName string) (string, parse.Config, error) { + key, err := keystore.Retrieve(keyName) + + if err != nil { + // If we cannot find the key, its a non fatal error + // and we pass to other resolver. + if errors.Is(err, ErrKeyDoesntExists) { + return "", parse.DefaultConfig, ucfg.ErrMissing + } + return "", parse.DefaultConfig, err + } + + v, err := key.Get() + if err != nil { + return "", parse.DefaultConfig, err + } + + return string(v), parse.DefaultConfig, nil + } +} + +// AsWritableKeystore casts a keystore to WritableKeystore, returning an ErrNotWritable error if the given keystore does not implement +// WritableKeystore interface +func AsWritableKeystore(store Keystore) (WritableKeystore, error) { + w, ok := store.(WritableKeystore) + if !ok { + return nil, ErrNotWritable + } + return w, nil +} + +// AsListingKeystore casts a keystore to ListingKeystore, returning an ErrNotListing error if the given keystore does not implement +// ListingKeystore interface +func AsListingKeystore(store Keystore) (ListingKeystore, error) { + w, ok := store.(ListingKeystore) + if !ok { + return nil, ErrNotListing + } + return w, nil +} diff --git a/keystore/keystore_test.go b/keystore/keystore_test.go new file mode 100644 index 0000000..36c96f3 --- /dev/null +++ b/keystore/keystore_test.go @@ -0,0 +1,52 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package keystore + +import ( + "os" + "testing" + + "github.com/stretchr/testify/assert" + + ucfg "github.com/elastic/go-ucfg" + "github.com/elastic/go-ucfg/parse" +) + +func TestResolverWhenTheKeyDoesntExist(t *testing.T) { + path := GetTemporaryKeystoreFile() + defer os.Remove(path) + + keystore := CreateAnExistingKeystore(path) + + resolver := ResolverWrap(keystore) + _, _, err := resolver("donotexist") + assert.Equal(t, err, ucfg.ErrMissing) +} + +func TestResolverWhenTheKeyExist(t *testing.T) { + path := GetTemporaryKeystoreFile() + defer os.Remove(path) + + keystore := CreateAnExistingKeystore(path) + + resolver := ResolverWrap(keystore) + v, pCfg, err := resolver("output.elasticsearch.password") + assert.NoError(t, err) + assert.Equal(t, pCfg, parse.DefaultConfig) + assert.Equal(t, v, "secret") +} diff --git a/keystore/secure_string.go b/keystore/secure_string.go new file mode 100644 index 0000000..d6f2ce9 --- /dev/null +++ b/keystore/secure_string.go @@ -0,0 +1,49 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package keystore + +// SecureString Initial implementation for a SecureString representation in +// beats, currently we keep the password into a Bytes array, we need to implement a way +// to safely clean that array. +// +// Investigate memguard: https://github.com/awnumar/memguard +type SecureString struct { + value []byte +} + +// NewSecureString return a struct representing a secrets string. +func NewSecureString(value []byte) *SecureString { + return &SecureString{ + value: value, + } +} + +// Get returns the byte value of the secret, or an error if we cannot return it. +func (s *SecureString) Get() ([]byte, error) { + return s.value, nil +} + +// String custom string implementation to make sure we don't bleed this struct into a string. +func (s SecureString) String() string { + return "" +} + +// GoString implements the GoStringer interface to hide the secret value. +func (s SecureString) GoString() string { + return s.String() +} diff --git a/keystore/secure_string_test.go b/keystore/secure_string_test.go new file mode 100644 index 0000000..a8bf2f6 --- /dev/null +++ b/keystore/secure_string_test.go @@ -0,0 +1,55 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package keystore + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +var secret = []byte("mysecret") + +func TestGet(t *testing.T) { + s := NewSecureString(secret) + v, err := s.Get() + assert.Equal(t, secret, v) + assert.NoError(t, err) +} + +func TestStringMarshalingS(t *testing.T) { + s := NewSecureString(secret) + v := fmt.Sprintf("%s", s) //nolint: gosimple // the goal of the test is to check if the string is not printed + + assert.Equal(t, v, "") +} + +func TestStringMarshalingF(t *testing.T) { + s := NewSecureString(secret) + v := fmt.Sprintf("%v", s) + + assert.Equal(t, v, "") +} + +func TestStringGoStringerMarshaling(t *testing.T) { + s := NewSecureString(secret) + v := fmt.Sprintf("%#v", s) + + assert.Equal(t, v, "") +}