diff --git a/cmd/kops/delete_secret.go b/cmd/kops/delete_secret.go
index 5797ef8268da3..681c3007dc2fc 100644
--- a/cmd/kops/delete_secret.go
+++ b/cmd/kops/delete_secret.go
@@ -18,7 +18,6 @@ package main
 
 import (
 	"fmt"
-
 	"io"
 
 	"github.com/spf13/cobra"
@@ -128,7 +127,12 @@ func RunDeleteSecret(f *util.Factory, out io.Writer, options *DeleteSecretOption
 		return fmt.Errorf("found multiple matching secrets; specify the id of the key")
 	}
 
-	err = keyStore.DeleteSecret(secrets[0])
+	switch secrets[0].Type {
+	case fi.SecretTypeSecret:
+		err = secretStore.DeleteSecret(secrets[0])
+	default:
+		err = keyStore.DeleteSecret(secrets[0])
+	}
 	if err != nil {
 		return fmt.Errorf("error deleting secret: %v", err)
 	}
diff --git a/upup/pkg/fi/secrets.go b/upup/pkg/fi/secrets.go
index 51bb6de964c4a..d2dc66cb7e51a 100644
--- a/upup/pkg/fi/secrets.go
+++ b/upup/pkg/fi/secrets.go
@@ -27,6 +27,8 @@ import (
 type SecretStore interface {
 	// Get a secret.  Returns an error if not found
 	Secret(id string) (*Secret, error)
+	// DeleteSecret deletes the specified secret
+	DeleteSecret(item *KeystoreItem) error
 	// Find a secret, if exists.  Returns nil,nil if not found
 	FindSecret(id string) (*Secret, error)
 	// Create or replace a secret
diff --git a/upup/pkg/fi/secrets/vfs_secretstore.go b/upup/pkg/fi/secrets/vfs_secretstore.go
index c8cfbdaf9d805..8613ba59a63d8 100644
--- a/upup/pkg/fi/secrets/vfs_secretstore.go
+++ b/upup/pkg/fi/secrets/vfs_secretstore.go
@@ -38,8 +38,8 @@ func NewVFSSecretStore(basedir vfs.Path) fi.SecretStore {
 	return c
 }
 
-func (s *VFSSecretStore) VFSPath() vfs.Path {
-	return s.basedir
+func (c *VFSSecretStore) VFSPath() vfs.Path {
+	return c.basedir
 }
 
 func (c *VFSSecretStore) buildSecretPath(id string) vfs.Path {
@@ -55,6 +55,18 @@ func (c *VFSSecretStore) FindSecret(id string) (*fi.Secret, error) {
 	return s, nil
 }
 
+// DeleteSecret implements fi.SecretStore DeleteSecret
+func (c *VFSSecretStore) DeleteSecret(item *fi.KeystoreItem) error {
+	switch item.Type {
+	case fi.SecretTypeSecret:
+		p := c.buildSecretPath(item.Name)
+		return p.Remove()
+
+	default:
+		return fmt.Errorf("deletion of secretstore items of type %v not (yet) supported", item.Type)
+	}
+}
+
 func (c *VFSSecretStore) ListSecrets() ([]string, error) {
 	files, err := c.basedir.ReadDir()
 	if err != nil {
diff --git a/upup/pkg/fi/vfs_castore.go b/upup/pkg/fi/vfs_castore.go
index 026d7c3ad93e6..06e3ffe211f56 100644
--- a/upup/pkg/fi/vfs_castore.go
+++ b/upup/pkg/fi/vfs_castore.go
@@ -754,6 +754,21 @@ func (c *VFSCAStore) DeleteSecret(item *KeystoreItem) error {
 		p := c.buildSSHPublicKeyPath(item.Name, item.Id)
 		return p.Remove()
 
+	case SecretTypeKeypair:
+		version, ok := big.NewInt(0).SetString(item.Id, 10)
+		if !ok {
+			return fmt.Errorf("keypair had non-integer version: %q", item.Id)
+		}
+		p := c.buildCertificatePath(item.Name, version)
+		if err := p.Remove(); err != nil {
+			return fmt.Errorf("error deleting certificate: %v", err)
+		}
+		p = c.buildPrivateKeyPath(item.Name, version)
+		if err := p.Remove(); err != nil {
+			return fmt.Errorf("error deleting private key: %v", err)
+		}
+		return nil
+
 	default:
 		// Primarily because we need to make sure users can recreate them!
 		return fmt.Errorf("deletion of keystore items of type %v not (yet) supported", item.Type)
diff --git a/util/pkg/vfs/s3fs.go b/util/pkg/vfs/s3fs.go
index 665be1239c6b4..6a6c7a80f1b92 100644
--- a/util/pkg/vfs/s3fs.go
+++ b/util/pkg/vfs/s3fs.go
@@ -76,6 +76,8 @@ func (p *S3Path) Remove() error {
 		return err
 	}
 
+	glog.V(8).Infof("removing file %s", p)
+
 	request := &s3.DeleteObjectInput{}
 	request.Bucket = aws.String(p.bucket)
 	request.Key = aws.String(p.key)