diff --git a/pkg/secrets/filedriver/filedriver.go b/pkg/secrets/filedriver/filedriver.go new file mode 100644 index 000000000..b58db6c63 --- /dev/null +++ b/pkg/secrets/filedriver/filedriver.go @@ -0,0 +1,164 @@ +package filedriver + +import ( + "encoding/json" + "io/ioutil" + "os" + "path/filepath" + "time" + + "github.com/containers/storage/pkg/lockfile" + "github.com/pkg/errors" +) + +// secretsDataFile is the file where secrets data/payload will be stored +var secretsDataFile = "secretsdata.json" + +//ErrNoSecretData indicates that there is not data assocaited with an id +var ErrNoSecretData = errors.New("No secret data with ID") + +//ErrNoSecretData indicates that there is secret data already associated with an id +var ErrSecretIDExists = errors.New("Secret data with ID already exists") + +// fileDriver is the filedriver object +type FileDriver struct { + // filePath is the path to the secretsfile + secretsDataFilePath string + // drivertype is the string representation of a filedriver + drivertype string + // lockfile is the filedriver lockfile + lockfile lockfile.Locker + // secretData is an in-memory rep of secrets data + secretData map[string][]byte + // lastModified is the time when the database was last modified in memory + lastModified time.Time +} + +//NewFileDriver creates a new file driver +func NewFileDriver(dirPath string) (*FileDriver, error) { + fileDriver := new(FileDriver) + fileDriver.secretsDataFilePath = filepath.Join(dirPath, secretsDataFile) + fileDriver.drivertype = "file" + lockfile, err := lockfile.GetLockfile(filepath.Join(dirPath, "secretsdata.lock")) + if err != nil { + return nil, err + } + fileDriver.lockfile = lockfile + fileDriver.secretData = make(map[string][]byte) + + return fileDriver, nil +} + +// DriverType retuns the string represntation of the filedriver +func (d *FileDriver) DriverType() string { + return d.drivertype +} + +// List returns all secret id's +func (d *FileDriver) List() ([]string, error) { + d.lockfile.Lock() + defer d.lockfile.Unlock() + err := d.loadData() + if err != nil { + return nil, err + } + var allID []string + for k := range d.secretData { + allID = append(allID, k) + } + + return allID, err +} + +// Lookup returns the bytes associated with a secret id +func (d *FileDriver) Lookup(id string) ([]byte, error) { + + d.lockfile.Lock() + defer d.lockfile.Unlock() + + err := d.loadData() + if err != nil { + return nil, err + } + if data, ok := d.secretData[id]; ok { + return data, nil + } else { + return nil, errors.Wrapf(ErrNoSecretData, "%s", id) + } + +} + +// Store stores the bytes associated with an id +func (d *FileDriver) Store(id string, data []byte) error { + d.lockfile.Lock() + defer d.lockfile.Unlock() + + err := d.loadData() + if err != nil { + return err + } + if _, ok := d.secretData[id]; ok { + return errors.Wrapf(ErrSecretIDExists, "%s", id) + } + d.secretData[id] = data + marshalled, err := json.MarshalIndent(d.secretData, "", " ") + if err != nil { + return err + } + err = ioutil.WriteFile(d.secretsDataFilePath, marshalled, 0644) + if err != nil { + return err + } + return nil + +} + +// Delete deletes a secret's data associated with an id +func (d *FileDriver) Delete(id string) error { + d.lockfile.Lock() + defer d.lockfile.Unlock() + err := d.loadData() + if err != nil { + return err + } + if _, ok := d.secretData[id]; ok { + delete(d.secretData, id) + } else { + return errors.Wrap(ErrNoSecretData, id) + } + marshalled, err := json.MarshalIndent(d.secretData, "", " ") + if err != nil { + return err + } + err = ioutil.WriteFile(d.secretsDataFilePath, marshalled, 0644) + if err != nil { + return err + } + return nil +} + +// load loads the secret data into memory if it has been modified +func (d *FileDriver) loadData() error { + fileInfo, err := os.Stat(d.secretsDataFilePath) + if err != nil { + if os.IsNotExist(err) { + return nil + } else { + return err + } + } + + if !d.lastModified.Equal(fileInfo.ModTime()) { + file, err := os.Open(d.secretsDataFilePath) + defer file.Close() + + byteValue, err := ioutil.ReadAll(file) + if err != nil { + return err + } + json.Unmarshal([]byte(byteValue), &d.secretData) + d.lastModified = fileInfo.ModTime() + } + return nil + +} diff --git a/pkg/secrets/filedriver/filedriver_test.go b/pkg/secrets/filedriver/filedriver_test.go new file mode 100644 index 000000000..5c2cc72ac --- /dev/null +++ b/pkg/secrets/filedriver/filedriver_test.go @@ -0,0 +1,94 @@ +package filedriver + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +func setup() (*FileDriver, error) { + tmppath, err := ioutil.TempDir("", "secretsdata") + if err != nil { + return nil, err + } + return NewFileDriver(tmppath) +} + +func TestStoreAndLookupSecretData(t *testing.T) { + tstdriver, err := setup() + assert.NoError(t, err) + defer os.Remove(tstdriver.secretsDataFilePath) + + err = tstdriver.Store("unique_id", []byte("somedata")) + assert.NoError(t, err) + + secretData, err := tstdriver.Lookup("unique_id") + assert.NoError(t, err) + assert.Equal(t, secretData, []byte("somedata")) + +} + +func TestStoreDupID(t *testing.T) { + tstdriver, err := setup() + assert.NoError(t, err) + defer os.Remove(tstdriver.secretsDataFilePath) + + err = tstdriver.Store("unique_id", []byte("somedata")) + assert.NoError(t, err) + + err = tstdriver.Store("unique_id", []byte("somedata")) + assert.Error(t, err) + +} + +func TestLookupBogus(t *testing.T) { + tstdriver, err := setup() + assert.NoError(t, err) + defer os.Remove(tstdriver.secretsDataFilePath) + + _, err = tstdriver.Lookup("bogus") + assert.Error(t, err) + +} + +func TestDeleteSecretData(t *testing.T) { + tstdriver, err := setup() + assert.NoError(t, err) + defer os.Remove(tstdriver.secretsDataFilePath) + + err = tstdriver.Store("unique_id", []byte("somedata")) + assert.NoError(t, err) + err = tstdriver.Delete("unique_id") + assert.NoError(t, err) + data, err := tstdriver.Lookup("unique_id") + assert.Error(t, err) + assert.Nil(t, data) + +} +func TestDeleteSecretDataNotExist(t *testing.T) { + tstdriver, err := setup() + assert.NoError(t, err) + defer os.Remove(tstdriver.secretsDataFilePath) + + err = tstdriver.Delete("bogus") + assert.Error(t, err) + +} + +func TestList(t *testing.T) { + tstdriver, err := setup() + assert.NoError(t, err) + defer os.Remove(tstdriver.secretsDataFilePath) + + err = tstdriver.Store("unique_id", []byte("somedata")) + assert.NoError(t, err) + err = tstdriver.Store("unique_id2", []byte("moredata")) + assert.NoError(t, err) + + data, err := tstdriver.List() + assert.NoError(t, err) + assert.Len(t, data, 2) + +} diff --git a/pkg/secrets/secrets.go b/pkg/secrets/secrets.go new file mode 100644 index 000000000..20ddb3212 --- /dev/null +++ b/pkg/secrets/secrets.go @@ -0,0 +1,211 @@ +package secrets + +import ( + "path/filepath" + "time" + + "github.com/containers/storage/pkg/lockfile" + "github.com/containers/storage/pkg/stringid" + "github.com/pkg/errors" +) + +// ErrInvalidPath indicates that the secrets path is invalid +var ErrInvalidPath = errors.New("Invalid secrets path") + +// ErrNoSuchSecret indicates that the the secret does not exist +var ErrNoSuchSecret = errors.New("no such secret") + +// ErrSecretNameInUse indicates that the secret name is already in use +var ErrSecretNameInUse = errors.New("secret name in use") + +// ErrInvalidSecretName indicates that the secret name is invalid +var ErrInvalidSecretName = errors.New("invalid secret name") + +// secretsFile is the name of the file that the secrets database will be stored in +var secretsFile = "secrets.json" + +// SecretsManager holds information on handling the secret +type SecretsManager struct { + // secretsPath is the path to the db file where secrets are stored + secretsDBPath string + // driver is the secrets data driver + driver SecretsDriver + // lockfile is the locker for the secrets file + lockfile lockfile.Locker + // db is a working in memory represnataion of the database of secrets + db *db +} + +// Secret defines a secret +type Secret struct { + // Name is the name of the secret + Name string `json:"name"` + // ID is the unique secret ID + ID string `json:"id"` + // Metadata stores other metadata on the secret + Metadata map[string]string `json:"metadata,omitempty"` + // CreatedAt is when the secret was created + CreatedAt time.Time + // Driver is the driver used to store secret data + Driver string +} + +// SecretsDriver interfaces with the secrets data store. +// Currently only the unencrypted filedriver and an in-memory +// driver option is implemented +type SecretsDriver interface { + // List lists all secret ids in the secrets data store + List() ([]string, error) + // Lookup gets the secret's data + Lookup(id string) ([]byte, error) + // Store stores the secret's data + Store(id string, data []byte) error + // Delete deletes a secret's data + Delete(id string) error + // DriverType returns the driver type + DriverType() string +} + +// NewManager creates a new secrets manager +func NewManager(dirPath string, driver SecretsDriver) (*SecretsManager, error) { + manager := new(SecretsManager) + + if !filepath.IsAbs(dirPath) { + return nil, errors.Wrap(ErrInvalidPath, "path must be absolute") + } + lockfile, err := lockfile.GetLockfile(filepath.Join(dirPath, "secrets.lock")) + if err != nil { + return nil, err + } + manager.lockfile = lockfile + manager.driver = driver + manager.secretsDBPath = filepath.Join(dirPath, secretsFile) + manager.db = new(db) + manager.db.Secrets = make(map[string]Secret) + manager.db.NameToID = make(map[string]string) + manager.db.IDToName = make(map[string]string) + return manager, nil +} + +// Store creates a secret and stores it, given a name. +// It stores secret metadata as well as the secret payload +func (s *SecretsManager) Store(name string, data []byte) (string, error) { //store, make consistent + err := validateSecretName(name) + if err != nil { + return "", err + } + + s.lockfile.Lock() + defer s.lockfile.Unlock() + + exist, err := s.secretExists(name) + if err != nil { + return "", err + } + if exist { + return "", errors.Wrapf(ErrSecretNameInUse, name) + } + + secr := new(Secret) + secr.Name = name + + for { + newID := stringid.GenerateNonCryptoID() + _, err := s.lookupSecret(newID) + if err != nil { + if errors.Cause(err) == ErrNoSuchSecret { + secr.ID = newID + break + } else { + return "", err + } + } + } + + secr.Driver = s.driver.DriverType() + secr.Metadata = make(map[string]string) + secr.CreatedAt = time.Now() + + err = s.driver.Store(secr.ID, data) + if err != nil { + return "", errors.Wrapf(err, "error creating secret %s", name) + } + + err = s.store(*secr) + if err != nil { + return "", errors.Wrapf(err, "error creating secret %s", name) + } + + return secr.ID, nil +} + +// Delete removes a secret +// It removes secret metadata as well as the secret data +func (s *SecretsManager) Delete(nameOrID string) (string, error) { + err := validateSecretName(nameOrID) + if err != nil { + return "", err + } + + s.lockfile.Lock() + defer s.lockfile.Unlock() + + _, id, err := s.getNameAndID(nameOrID) + if err != nil { + return "", err + } + + err = s.driver.Delete(id) + if err != nil { + return "", errors.Wrapf(err, "error deleting secret %s", nameOrID) + } + + _, err = s.delete(id) + if err != nil { + return "", errors.Wrapf(err, "error deleting secret %s", nameOrID) + } + return id, nil +} + +// Lookup gives a secret's metadata +func (s *SecretsManager) Lookup(namesOrIDs []string) ([]*Secret, error) { + if len(namesOrIDs) == 0 { + return nil, ErrInvalidSecretName + } + var lookups []*Secret + for _, nameOrID := range namesOrIDs { + secret, err := s.lookupSecret(nameOrID) + if err != nil { + return nil, err + } + lookups = append(lookups, secret) + } + + return lookups, nil +} + +// List lists all secrets +func (s *SecretsManager) List() ([]Secret, error) { + s.lockfile.Lock() + defer s.lockfile.Unlock() + + secrets, err := s.lookupAll() + if err != nil { + return nil, err + } + var ls []Secret + for _, v := range secrets { + ls = append(ls, v) + + } + return ls, nil + +} + +// validateSecretName checks if the secret name is valid +func validateSecretName(name string) error { + if name == "" { + return errors.Wrap(ErrInvalidSecretName, name) + } + return nil +} diff --git a/pkg/secrets/secrets.lock b/pkg/secrets/secrets.lock new file mode 100644 index 000000000..e69de29bb diff --git a/pkg/secrets/secrets_test.go b/pkg/secrets/secrets_test.go new file mode 100644 index 000000000..4b4795ca1 --- /dev/null +++ b/pkg/secrets/secrets_test.go @@ -0,0 +1,240 @@ +package secrets + +import ( + "bytes" + "io/ioutil" + "os" + "testing" + + "github.com/stretchr/testify/assert" +) + +type testDriver struct { + data map[string][]byte + driverType string +} + +func (d *testDriver) List() ([]string, error) { + var allID []string + for k := range d.data { + allID = append(allID, k) + } + return allID, nil +} + +func (d *testDriver) Lookup(id string) ([]byte, error) { + return d.data[id], nil +} +func (d *testDriver) Store(id string, data []byte) error { + d.data[id] = data + return nil +} + +func (d *testDriver) Delete(id string) error { + delete(d.data, id) + return nil +} +func (d *testDriver) DriverType() string { + return d.driverType +} +func (d *testDriver) dataInit() error { + return nil +} + +func NewTestDriver() (*testDriver, error) { + tstdriver := &testDriver{} + tstdriver.data = make(map[string][]byte) + tstdriver.driverType = "test" + return tstdriver, nil +} + +func setup() (*SecretsManager, error) { + driv, err := NewTestDriver() + tmppath, err := ioutil.TempDir("", "secretsdata") + if err != nil { + return nil, err + } + manager, err := NewManager(tmppath, driv) + return manager, err +} + +func TestAddSecret(t *testing.T) { + + manager, err := setup() + assert.NoError(t, err) + defer os.Remove(manager.secretsDBPath) + + id, err := manager.Store("mysecret", []byte("mydata")) + assert.NoError(t, err) + + _, err = manager.lookupSecret("mysecret") + assert.NoError(t, err) + + data, err := manager.driver.Lookup(id) + if !bytes.Equal(data, []byte("mydata")) { + t.Errorf("error: secret data not equal") + } + +} + +func TestAddMultipleSecrets(t *testing.T) { + + manager, err := setup() + assert.NoError(t, err) + defer os.Remove(manager.secretsDBPath) + + id, err := manager.Store("mysecret", []byte("mydata")) + assert.NoError(t, err) + + id2, err := manager.Store("mysecret2", []byte("mydata")) + assert.NoError(t, err) + + secrets, err := manager.List() + assert.Len(t, secrets, 2) + + _, err = manager.lookupSecret("mysecret") + assert.NoError(t, err) + + _, err = manager.lookupSecret("mysecret2") + assert.NoError(t, err) + + data, err := manager.driver.Lookup(id) + if !bytes.Equal(data, []byte("mydata")) { + t.Errorf("error: secret data not equal") + } + + data, err = manager.driver.Lookup(id2) + if !bytes.Equal(data, []byte("mydata")) { + t.Errorf("error: secret data not equal") + } + +} + +func TestAddSecretDupName(t *testing.T) { + manager, err := setup() + assert.NoError(t, err) + defer os.Remove(manager.secretsDBPath) + + _, err = manager.Store("mysecret", []byte("mydata")) + assert.NoError(t, err) + + _, err = manager.Store("mysecret", []byte("mydata")) + assert.Error(t, err) +} + +func TestRemoveSecret(t *testing.T) { + manager, err := setup() + assert.NoError(t, err) + defer os.Remove(manager.secretsDBPath) + + _, err = manager.Store("mysecret", []byte("mydata")) + assert.NoError(t, err) + + _, err = manager.lookupSecret("mysecret") + assert.NoError(t, err) + + _, err = manager.Delete("mysecret") + assert.NoError(t, err) + + _, err = manager.lookupSecret("mysecret") + assert.Error(t, err) + +} + +func TestRemoveSecretNoExist(t *testing.T) { + manager, err := setup() + assert.NoError(t, err) + defer os.Remove(manager.secretsDBPath) + + _, err = manager.Delete("mysecret") + assert.Error(t, err) +} + +func TestLookupSecret(t *testing.T) { + manager, err := setup() + assert.NoError(t, err) + defer os.Remove(manager.secretsDBPath) + + id, err := manager.Store("mysecret", []byte("mydata")) + assert.NoError(t, err) + + _, err = manager.lookupSecret("mysecret") + assert.NoError(t, err) + + var secret []string + secret = append(secret, "mysecret") + lookup, err := manager.Lookup(secret) + assert.NoError(t, err) + assert.Equal(t, lookup[0].ID, id) + assert.Equal(t, lookup[0].Name, "mysecret") +} + +func TestInspectSecretId(t *testing.T) { + manager, err := setup() + assert.NoError(t, err) + defer os.Remove(manager.secretsDBPath) + + id, err := manager.Store("mysecret", []byte("mydata")) + assert.NoError(t, err) + + _, err = manager.lookupSecret("mysecret") + assert.NoError(t, err) + + //inspect using secret id + var secret []string + secret = append(secret, id) + lookup, err := manager.Lookup(secret) + assert.NoError(t, err) + assert.Equal(t, lookup[0].ID, id) + assert.Equal(t, lookup[0].Name, "mysecret") +} + +func TestInspectSecretBogus(t *testing.T) { + manager, err := setup() + assert.NoError(t, err) + defer os.Remove(manager.secretsDBPath) + + var secret []string + secret = append(secret, "bogus") + _, err = manager.Lookup(secret) + assert.Error(t, err) +} + +func TestSecretList(t *testing.T) { + manager, err := setup() + assert.NoError(t, err) + defer os.Remove(manager.secretsDBPath) + + _, err = manager.Store("mysecret", []byte("mydata")) + assert.NoError(t, err) + _, err = manager.Store("mysecret2", []byte("mydata")) + assert.NoError(t, err) + + allSecrets, err := manager.List() + assert.NoError(t, err) + assert.Len(t, allSecrets, 2) + +} + +func TestSecretSync(t *testing.T) { + manager, err := setup() + assert.NoError(t, err) + defer os.Remove(manager.secretsDBPath) + + id, err := manager.Store("mysecret", []byte("mydata")) + assert.NoError(t, err) + + // unsync databases + manager.driver.Delete(id) + manager.driver.Store("notindatabase", []byte("testbytes")) + + err = manager.sync() + + allSecrets, err := manager.List() + assert.NoError(t, err) + assert.Len(t, allSecrets, 0) + allids, err := manager.driver.List() + assert.NoError(t, err) + assert.Len(t, allids, 0) + +} diff --git a/pkg/secrets/secretsdb.go b/pkg/secrets/secretsdb.go new file mode 100644 index 000000000..45b1e5b54 --- /dev/null +++ b/pkg/secrets/secretsdb.go @@ -0,0 +1,194 @@ +package secrets + +import ( + "encoding/json" + "io/ioutil" + "os" + "time" + + "github.com/pkg/errors" +) + +type db struct { + // Secrets maps a secret id to secret metadata + Secrets map[string]Secret `json:"secrets"` + // NameToID maps a secret name to a secret id + NameToID map[string]string `json:"nametoid"` + // IDToName maps a secret id to a secret name + IDToName map[string]string `json:"idtoname"` + // lastModified is the time when the database was last modified on the file system + lastModified time.Time +} + +// loadDB loads database data into memory if it has been modified +func (s *SecretsManager) loadDB() error { + fileInfo, err := os.Stat(secretsFile) + if err != nil { + if os.IsNotExist(err) { + return nil + } else { + return err + } + } + if !s.db.lastModified.Equal(fileInfo.ModTime()) { + file, err := os.Open(s.secretsDBPath) + defer file.Close() + if err != nil { + return err + } + + byteValue, err := ioutil.ReadAll(file) + if err != nil { + return err + } + unmarshalled := new(db) + if err = json.Unmarshal(byteValue, unmarshalled); err != nil { + return err + } + s.db = unmarshalled + s.db.lastModified = fileInfo.ModTime() + } + return nil +} + +// getNameAndID takes a secret's name or id and returns both its +// identifying name and id +func (s *SecretsManager) getNameAndID(nameOrID string) (name string, id string, err error) { + err = s.loadDB() + if err != nil { + return "", "", err + } + if id, ok := s.db.NameToID[nameOrID]; ok { + name := nameOrID + return name, id, nil + } + + if name, ok := s.db.IDToName[nameOrID]; ok { + id := nameOrID + return name, id, nil + } + return "", "", errors.Wrapf(ErrNoSuchSecret, "No secret with name or id %s", nameOrID) +} + +// secretExists checks if the secret exists +func (s *SecretsManager) secretExists(nameOrID string) (bool, error) { + _, _, err := s.getNameAndID(nameOrID) + if err != nil { + if errors.Cause(err) == ErrNoSuchSecret { + return false, nil + } + return false, err + } + return true, nil +} + +// lookupAll gets all secrets stored +func (s *SecretsManager) lookupAll() (map[string]Secret, error) { + err := s.loadDB() + if err != nil { + return nil, err + } + return s.db.Secrets, nil +} + +// lookupSecret returns a secret with the given name or id +func (s *SecretsManager) lookupSecret(nameOrID string) (*Secret, error) { + _, id, err := s.getNameAndID(nameOrID) + if err != nil { + return nil, err + } + allSecrets, err := s.lookupAll() + if err != nil { + return nil, err + } + if secret, ok := allSecrets[id]; ok { + return &secret, nil + } + return nil, ErrNoSuchSecret +} + +// Store creates a new secret in the secrets database +// it deals with only storing metadata, not data payload +func (s *SecretsManager) store(entry Secret) error { + err := s.loadDB() + if err != nil { + return err + } + + s.db.Secrets[entry.ID] = entry + s.db.NameToID[entry.Name] = entry.ID + s.db.IDToName[entry.ID] = entry.Name + + marshalled, err := json.MarshalIndent(s.db, "", " ") + if err != nil { + return err + } + err = ioutil.WriteFile(s.secretsDBPath, marshalled, 0644) + if err != nil { + return err + } + + return nil + +} + +// delete deletes a secret from the secrets database +// it deals with only deleting metadata, not data payload +func (s *SecretsManager) delete(nameOrID string) (string, error) { + name, id, err := s.getNameAndID(nameOrID) + if err != nil { + return "", err + } + err = s.loadDB() + if err != nil { + return "", err + } + delete(s.db.Secrets, id) + delete(s.db.NameToID, name) + delete(s.db.IDToName, id) + marshalled, err := json.MarshalIndent(s.db, "", " ") + if err != nil { + return "", err + } + err = ioutil.WriteFile(s.secretsDBPath, marshalled, 0644) + if err != nil { + return "", err + } + return id, nil + +} + +// sync removes secret entries that do not have a matching entry +// in the driver, and vice versa +func (s *SecretsManager) sync() error { + s.lockfile.Lock() + defer s.lockfile.Unlock() + secrets, err := s.lookupAll() + if err != nil { + return err + } + secretsIDs, err := s.driver.List() + if err != nil { + return err + } + + // Delete entries from secret database if it does not have a matching driver entry + for id := range secrets { //if its in the secrets db + _, err := s.driver.Lookup(id) + if err == nil { //but not in the driver db + if secrets[id].Driver == s.driver.DriverType() { //check if the driver matches + s.delete(id) + } + + } + } + + // Delete entries from driver if it does not have matching entry in database + for _, id := range secretsIDs { + if _, ok := secrets[id]; !ok { + s.driver.Delete(id) + } + } + return nil + +}