diff --git a/go.mod b/go.mod index 1a758fefcc4..4cb9a6c4e82 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/DataDog/datadog-go v2.2.0+incompatible github.com/GeertJohan/go.rice v1.0.0 github.com/PuerkitoBio/goquery v1.5.1 + github.com/aquarapid/vaultlib v0.5.1 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6 github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 github.com/aws/aws-sdk-go v1.28.8 diff --git a/go.sum b/go.sum index 62825b13b93..50b907dda98 100644 --- a/go.sum +++ b/go.sum @@ -79,6 +79,8 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo= github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/aquarapid/vaultlib v0.5.1 h1:vuLWR6bZzLHybjJBSUYPgZlIp6KZ+SXeHLRRYTuk6d4= +github.com/aquarapid/vaultlib v0.5.1/go.mod h1:yT7AlEXtuabkxylOc/+Ulyp18tff1+QjgNLTnFWTlOs= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e h1:QEF07wC0T1rKkctt1RINW/+RMTVmiwxETico2l3gxJA= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6 h1:G1bPvciwNyF7IUmKXNt9Ak3m6u9DE1rF+RmtIkBpVdA= @@ -472,6 +474,8 @@ github.com/mattn/go-sqlite3 v1.14.0 h1:mLyGNKR8+Vv9CAU7PphKa2hkEqxxhn8i32J6FPj1/ github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mch1307/vaultlib v0.5.0 h1:+tI8YCG033aVI+kAKwo0fwrUylFs+wO6DB7DM5qXJzU= +github.com/mch1307/vaultlib v0.5.0/go.mod h1:phFbO1oIDL1xTqUrNXbrAG0VdcYEKP8TNa9FJd7hFic= github.com/miekg/dns v1.0.14 h1:9jZdLNd/P4+SfEJ0TNyxYpsK8N4GtfylBLqtbYN1sbA= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.26 h1:gPxPSwALAeHJSjarOs00QjVdV9QoBvc1D2ujQUr5BzU= diff --git a/go/cmd/vtgate/plugin_auth_vault.go b/go/cmd/vtgate/plugin_auth_vault.go new file mode 100644 index 00000000000..b43e471be36 --- /dev/null +++ b/go/cmd/vtgate/plugin_auth_vault.go @@ -0,0 +1,28 @@ +/* +Copyright 2020 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreedto in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +// This plugin imports InitAuthServerVault to register the HashiCorp Vault implementation of AuthServer. + +import ( + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/vt/vtgate" +) + +func init() { + vtgate.RegisterPluginInitializer(func() { mysql.InitAuthServerVault() }) +} diff --git a/go/cmd/vttablet/vttablet.go b/go/cmd/vttablet/vttablet.go index e9530953da6..b860efcde13 100644 --- a/go/cmd/vttablet/vttablet.go +++ b/go/cmd/vttablet/vttablet.go @@ -72,7 +72,7 @@ func main() { log.Exitf("failed to parse -tablet-path: %v", err) } - // config and mycnf intializations are intertwined. + // config and mycnf initializations are intertwined. config, mycnf := initConfig(tabletAlias) ts := topo.Open() @@ -105,7 +105,7 @@ func main() { VREngine: vreplication.NewEngine(config, ts, tabletAlias.Cell, mysqld), } if err := tm.Start(tablet, config.Healthcheck.IntervalSeconds.Get()); err != nil { - log.Exitf("failed to parse -tablet-path: %v", err) + log.Exitf("failed to parse -tablet-path or initialize DB credentials: %v", err) } servenv.OnClose(func() { // Close the tm so that our topo entry gets pruned properly and any diff --git a/go/mysql/auth_server_vault.go b/go/mysql/auth_server_vault.go new file mode 100644 index 00000000000..7f72cd72e19 --- /dev/null +++ b/go/mysql/auth_server_vault.go @@ -0,0 +1,302 @@ +/* +Copyright 2020 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mysql + +import ( + "bytes" + "flag" + "fmt" + "io/ioutil" + "net" + "os" + "os/signal" + "strings" + "sync" + "syscall" + "time" + + vaultapi "github.com/aquarapid/vaultlib" + + "vitess.io/vitess/go/vt/log" +) + +var ( + vaultAddr = flag.String("mysql_auth_vault_addr", "", "URL to Vault server") + vaultTimeout = flag.Duration("mysql_auth_vault_timeout", 10*time.Second, "Timeout for vault API operations") + vaultCACert = flag.String("mysql_auth_vault_tls_ca", "", "Path to CA PEM for validating Vault server certificate") + vaultPath = flag.String("mysql_auth_vault_path", "", "Vault path to vtgate credentials JSON blob, e.g.: secret/data/prod/vtgatecreds") + vaultCacheTTL = flag.Duration("mysql_auth_vault_ttl", 30*time.Minute, "How long to cache vtgate credentials from the Vault server") + vaultTokenFile = flag.String("mysql_auth_vault_tokenfile", "", "Path to file containing Vault auth token; token can also be passed using VAULT_TOKEN environment variable") + vaultRoleID = flag.String("mysql_auth_vault_roleid", "", "Vault AppRole id; can also be passed using VAULT_ROLEID environment variable") + vaultRoleSecretIDFile = flag.String("mysql_auth_vault_role_secretidfile", "", "Path to file containing Vault AppRole secret_id; can also be passed using VAULT_SECRETID environment variable") + vaultRoleMountPoint = flag.String("mysql_auth_vault_role_mountpoint", "approle", "Vault AppRole mountpoint; can also be passed using VAULT_MOUNTPOINT environment variable") +) + +// AuthServerVault implements AuthServer with a config loaded from Vault. +type AuthServerVault struct { + mu sync.Mutex + // method can be set to: + // - MysqlNativePassword + // - MysqlClearPassword + // - MysqlDialog + // It defaults to MysqlNativePassword. + method string + // users, passwords and user data + // We use the same JSON format as for -mysql_auth_server_static + // Acts as a cache for the in-Vault data + entries map[string][]*AuthServerStaticEntry + vaultCacheExpireTicker *time.Ticker + vaultClient *vaultapi.Client + vaultPath string + vaultTTL time.Duration + + sigChan chan os.Signal +} + +// InitAuthServerVault - entrypoint for initialization of Vault AuthServer implementation +func InitAuthServerVault() { + // Check critical parameters. + if *vaultAddr == "" { + log.Infof("Not configuring AuthServerVault, as -mysql_auth_vault_addr is empty.") + return + } + if *vaultPath == "" { + log.Exitf("If using Vault auth server, -mysql_auth_vault_path is required.") + } + + registerAuthServerVault(*vaultAddr, *vaultTimeout, *vaultCACert, *vaultPath, *vaultCacheTTL, *vaultTokenFile, *vaultRoleID, *vaultRoleSecretIDFile, *vaultRoleMountPoint) +} + +func registerAuthServerVault(addr string, timeout time.Duration, caCertPath string, path string, ttl time.Duration, tokenFilePath string, roleID string, secretIDPath string, roleMountPoint string) { + authServerVault, err := newAuthServerVault(addr, timeout, caCertPath, path, ttl, tokenFilePath, roleID, secretIDPath, roleMountPoint) + if err != nil { + log.Exitf("%s", err) + } + RegisterAuthServerImpl("vault", authServerVault) +} + +func newAuthServerVault(addr string, timeout time.Duration, caCertPath string, path string, ttl time.Duration, tokenFilePath string, roleID string, secretIDPath string, roleMountPoint string) (*AuthServerVault, error) { + // Validate more parameters + token, err := readFromFile(tokenFilePath) + if err != nil { + return nil, fmt.Errorf("No Vault token in provided filename for -mysql_auth_vault_tokenfile") + } + secretID, err := readFromFile(secretIDPath) + if err != nil { + return nil, fmt.Errorf("No Vault secret_id in provided filename for -mysql_auth_vault_role_secretidfile") + } + + config := vaultapi.NewConfig() + + // All these can be overriden by environment + // so we need to check if they have been set by NewConfig + if config.Address == "" { + config.Address = addr + } + if config.Timeout == (0 * time.Second) { + config.Timeout = timeout + } + if config.CACert == "" { + config.CACert = caCertPath + } + if config.Token == "" { + config.Token = token + } + if config.AppRoleCredentials.RoleID == "" { + config.AppRoleCredentials.RoleID = roleID + } + if config.AppRoleCredentials.SecretID == "" { + config.AppRoleCredentials.SecretID = secretID + } + if config.AppRoleCredentials.MountPoint == "" { + config.AppRoleCredentials.MountPoint = roleMountPoint + } + + if config.CACert != "" { + // If we provide a CA, ensure we actually use it + config.InsecureSSL = false + } + + client, err := vaultapi.NewClient(config) + if err != nil || client == nil { + log.Errorf("Error in vault client initialization, will retry: %v", err) + } + + a := &AuthServerVault{ + vaultClient: client, + vaultPath: path, + vaultTTL: ttl, + method: MysqlNativePassword, + entries: make(map[string][]*AuthServerStaticEntry), + } + + a.reloadVault() + a.installSignalHandlers() + return a, nil +} + +func (a *AuthServerVault) setTTLTicker(ttl time.Duration) { + a.mu.Lock() + defer a.mu.Unlock() + if a.vaultCacheExpireTicker == nil { + a.vaultCacheExpireTicker = time.NewTicker(ttl) + go func() { + for range a.vaultCacheExpireTicker.C { + a.sigChan <- syscall.SIGHUP + } + }() + } else { + a.vaultCacheExpireTicker.Reset(ttl) + } +} + +// Reload JSON auth key from Vault. Return true if successful, false if not +func (a *AuthServerVault) reloadVault() error { + a.mu.Lock() + secret, err := a.vaultClient.GetSecret(a.vaultPath) + a.mu.Unlock() + a.setTTLTicker(10 * time.Second) // Reload frequently on error + + if err != nil { + return fmt.Errorf("Error in vtgate Vault auth server params: %v", err) + } + + if secret.JSONSecret == nil { + return fmt.Errorf("Empty vtgate credentials retrieved from Vault server") + } + + entries := make(map[string][]*AuthServerStaticEntry) + if err := parseConfig(secret.JSONSecret, &entries); err != nil { + return fmt.Errorf("Error parsing vtgate Vault auth server config: %v", err) + } + if len(entries) == 0 { + return fmt.Errorf("vtgate credentials from Vault empty! Not updating previously cached values") + } + + log.Infof("reloadVault(): success. Client status: %s", a.vaultClient.GetStatus()) + a.mu.Lock() + a.entries = entries + a.mu.Unlock() + a.setTTLTicker(a.vaultTTL) + return nil +} + +func (a *AuthServerVault) installSignalHandlers() { + a.mu.Lock() + defer a.mu.Unlock() + + a.sigChan = make(chan os.Signal, 1) + signal.Notify(a.sigChan, syscall.SIGHUP) + go func() { + for range a.sigChan { + err := a.reloadVault() + if err != nil { + log.Errorf("%s", err) + } + + } + }() +} + +func (a *AuthServerVault) close() { + log.Warningf("Closing AuthServerVault instance.") + a.mu.Lock() + defer a.mu.Unlock() + if a.vaultCacheExpireTicker != nil { + a.vaultCacheExpireTicker.Stop() + } + if a.sigChan != nil { + signal.Stop(a.sigChan) + } +} + +// AuthMethod is part of the AuthServer interface. +func (a *AuthServerVault) AuthMethod(user string) (string, error) { + return a.method, nil +} + +// Salt is part of the AuthServer interface. +func (a *AuthServerVault) Salt() ([]byte, error) { + return NewSalt() +} + +// ValidateHash is part of the AuthServer interface. +func (a *AuthServerVault) ValidateHash(salt []byte, user string, authResponse []byte, remoteAddr net.Addr) (Getter, error) { + a.mu.Lock() + userEntries, ok := a.entries[user] + a.mu.Unlock() + + if !ok { + return &StaticUserData{}, NewSQLError(ERAccessDeniedError, SSAccessDeniedError, "Access denied for user '%v'", user) + } + + for _, entry := range userEntries { + if entry.MysqlNativePassword != "" { + isPass := isPassScrambleMysqlNativePassword(authResponse, salt, entry.MysqlNativePassword) + if matchSourceHost(remoteAddr, entry.SourceHost) && isPass { + return &StaticUserData{entry.UserData, entry.Groups}, nil + } + } else { + computedAuthResponse := ScramblePassword(salt, []byte(entry.Password)) + // Validate the password. + if matchSourceHost(remoteAddr, entry.SourceHost) && bytes.Equal(authResponse, computedAuthResponse) { + return &StaticUserData{entry.UserData, entry.Groups}, nil + } + } + } + return &StaticUserData{}, NewSQLError(ERAccessDeniedError, SSAccessDeniedError, "Access denied for user '%v'", user) +} + +// Negotiate is part of the AuthServer interface. +// It will be called if method is anything else than MysqlNativePassword. +// We only recognize MysqlClearPassword and MysqlDialog here. +func (a *AuthServerVault) Negotiate(c *Conn, user string, remoteAddr net.Addr) (Getter, error) { + // Finish the negotiation. + password, err := AuthServerNegotiateClearOrDialog(c, a.method) + if err != nil { + return nil, err + } + + a.mu.Lock() + userEntries, ok := a.entries[user] + a.mu.Unlock() + + if !ok { + return &StaticUserData{}, NewSQLError(ERAccessDeniedError, SSAccessDeniedError, "Access denied for user '%v'", user) + } + for _, entry := range userEntries { + // Validate the password. + if matchSourceHost(remoteAddr, entry.SourceHost) && entry.Password == password { + return &StaticUserData{entry.UserData, entry.Groups}, nil + } + } + return &StaticUserData{}, NewSQLError(ERAccessDeniedError, SSAccessDeniedError, "Access denied for user '%v'", user) +} + +// We ignore most errors here, to allow us to retry cleanly +// or ignore the cases where the input is not passed by file, but via env +func readFromFile(filePath string) (string, error) { + if filePath == "" { + return "", nil + } + fileBytes, err := ioutil.ReadFile(filePath) + if err != nil { + log.Errorf("Could not read file: %s", filePath) + return "", err + } + return strings.TrimSpace(string(fileBytes)), nil +} diff --git a/go/mysql/auth_server_vault_test.go b/go/mysql/auth_server_vault_test.go new file mode 100644 index 00000000000..34a8ff9352e --- /dev/null +++ b/go/mysql/auth_server_vault_test.go @@ -0,0 +1,47 @@ +/* +copyright 2020 The Vitess Authors. + +licensed under the apache license, version 2.0 (the "license"); +you may not use this file except in compliance with the license. +you may obtain a copy of the license at + + http://www.apache.org/licenses/license-2.0 + +unless required by applicable law or agreed to in writing, software +distributed under the license is distributed on an "as is" basis, +without warranties or conditions of any kind, either express or implied. +see the license for the specific language governing permissions and +limitations under the license. +*/ + +package mysql + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestErrorConditions(t *testing.T) { + // Bad token file path + _, err := newAuthServerVault("localhost", 1*time.Second, "", "/path/to/secret/in/vault", 10*time.Second, "/tmp/this_file_does_not_exist", "", "", "") + assert.Contains(t, err.Error(), "No Vault token in provided filename") + + // Bad secretID file path + _, err = newAuthServerVault("localhost", 1*time.Second, "", "/path/to/secret/in/vault", 10*time.Second, "", "", "/tmp/this_file_does_not_exist", "") + assert.Contains(t, err.Error(), "No Vault secret_id in provided filename") + + // Bad init ; but we should just retry + a, err := newAuthServerVault("https://localhost:828", 1*time.Second, "", "/path/to/secret/in/vault", 10*time.Second, "", "", "", "") + assert.NotEqual(t, a, nil) + assert.Equal(t, err, nil) + + // Test reload, should surface error, since we don't have a Vault + // instance on port 828 + err = a.reloadVault() + assert.Contains(t, err.Error(), "Error in vtgate Vault auth server params") + assert.Contains(t, err.Error(), "connection refused") + + a.close() +} diff --git a/go/test/endtoend/vault/ca.pem b/go/test/endtoend/vault/ca.pem new file mode 100644 index 00000000000..2c19d3b0ca3 --- /dev/null +++ b/go/test/endtoend/vault/ca.pem @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDSDCCAjCgAwIBAgIUGTcQb8NrTsJxh3Ef2lGd4YFDN7owDQYJKoZIhvcNAQEL +BQAwFTETMBEGA1UEAwwKVGVzdGluZyBDQTAeFw0yMTAxMDEwMDAzMTBaFw0zMDEy +MzAwMDAzMTBaMBUxEzARBgNVBAMMClRlc3RpbmcgQ0EwggEiMA0GCSqGSIb3DQEB +AQUAA4IBDwAwggEKAoIBAQDE42b11YcFIPmUHzRo0GhUaulwQ78RDVQfCy3WrWWt +w1Fv5F7xSiONl+eCRDTHUD1Lt4lMee1zFKySXg+kSRLzrrCuDfsK8MZ/4fKcrRVS +XliI9eS0QZijczYzlwgIjodnoAqcSmMJi0inC3oRTDu2/zRUEMKCtM4F2A7Fh1vu +BEdO6b0SmIGBFAb1Eqn++U9dES1xhMrwESy81Vs/C9qyclZCqaFqtEUBgepqkTGQ +stpMqdubhrJxYq4Lj3dcN+N/jubpPTHPJZrHO7fIJiRQaBUrqlLxUafqHvOrvwEG +aqZgaNT34ylCOdYtzuJpeHjZ1ptewMEdqXmHKC3gmpNDAgMBAAGjgY8wgYwwHQYD +VR0OBBYEFCgXq7LJ5AI40DYi6rnQH/suQ4srMFAGA1UdIwRJMEeAFCgXq7LJ5AI4 +0DYi6rnQH/suQ4sroRmkFzAVMRMwEQYDVQQDDApUZXN0aW5nIENBghQZNxBvw2tO +wnGHcR/aUZ3hgUM3ujAMBgNVHRMEBTADAQH/MAsGA1UdDwQEAwIBBjANBgkqhkiG +9w0BAQsFAAOCAQEAHDqcA40Q56giI8JybsE56+uk89MCfbamuk54/p2dIlKxP6sd +W7y9zhFCPJFH4LUj9Dyp9mxmafmxUtnFGRYRvnOs3peZbwQo2He86uw4TCxtrT6y +0lxNyxfoCtnzQUl5XGxl6WC0RsotUWp2paHfuh2YDQi8wMOa9CIczS+XZpVV7ByS +y0cvyl+1Hc6MaIsQRf8WERIx+rpG4qWwDeE0vqelRLoLpud1t+g4qe/o3YGWO1g3 ++fUExbhlqqo/DwYn4acnkTZJnZNyKSAQe9GtKwLXmpZSjHeea+dY4Ukdri5DlEpQ +MLDVIe6eFjtcrzPbdESKCpnb+tYP1epizj87Xw== +-----END CERTIFICATE----- diff --git a/go/test/endtoend/vault/dbcreds_policy.hcl b/go/test/endtoend/vault/dbcreds_policy.hcl new file mode 100644 index 00000000000..1b0a768fb34 --- /dev/null +++ b/go/test/endtoend/vault/dbcreds_policy.hcl @@ -0,0 +1,28 @@ +# Allow tokens to look up their own properties +path "auth/token/lookup-self" { + capabilities = ["read"] +} + +# Allow tokens to renew themselves +path "auth/token/renew-self" { + capabilities = ["update"] +} + +# Allow tokens to revoke themselves +path "auth/token/revoke-self" { + capabilities = ["update"] +} + +# Allow a token to look up its own capabilities on a path +path "sys/capabilities-self" { + capabilities = ["update"] +} + +path "kv/data/prod/dbcreds" { + capabilities = ["read"] +} + +path "kv/data/prod/vtgatecreds" { + capabilities = ["read"] +} + diff --git a/go/test/endtoend/vault/dbcreds_secret.json b/go/test/endtoend/vault/dbcreds_secret.json new file mode 100644 index 00000000000..96fff38bdcd --- /dev/null +++ b/go/test/endtoend/vault/dbcreds_secret.json @@ -0,0 +1,17 @@ +{ + "vt_app": [ + "password" + ], + "vt_dba": [ + "password" + ], + "vt_repl": [ + "password" + ], + "vt_appdebug": [ + "password" + ], + "vt_filtered": [ + "password" + ] +} diff --git a/go/test/endtoend/vault/vault-cert.pem b/go/test/endtoend/vault/vault-cert.pem new file mode 100644 index 00000000000..79dc973ca2b --- /dev/null +++ b/go/test/endtoend/vault/vault-cert.pem @@ -0,0 +1,87 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + e8:d3:99:40:87:c1:7d:f3:93:7f:4f:da:96:0e:26:86 + Signature Algorithm: sha256WithRSAEncryption + Issuer: CN=Testing CA + Validity + Not Before: Jan 1 00:04:15 2021 GMT + Not After : Dec 29 00:04:15 2030 GMT + Subject: CN=vault-server + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + RSA Public-Key: (2048 bit) + Modulus: + 00:bb:96:88:43:7f:c7:4b:9a:e7:4c:1a:62:fc:c0: + 56:cb:1e:f7:42:d6:ca:18:16:41:22:f4:ca:ee:98: + c9:32:44:56:fa:06:54:5f:a1:2c:0a:1a:1b:7d:17: + f6:b9:cc:c6:c6:cd:a4:23:1d:03:c3:d5:2b:a7:4c: + ed:37:22:22:91:11:3f:96:40:f4:26:ce:e0:db:be: + f4:10:5d:7b:ca:91:22:78:33:2c:22:95:5f:98:c7: + 69:ce:33:6b:f7:f0:4f:8e:b1:9a:da:63:a5:89:64: + ce:c8:24:fb:56:29:f5:a5:14:32:18:46:20:5a:73: + 77:42:bc:1d:90:52:5e:6a:55:30:46:ee:4d:24:bb: + 93:85:6b:92:19:09:f5:25:0c:f1:02:ac:eb:41:85: + 55:3b:2c:d1:92:a1:7f:73:d7:5e:3a:40:f6:ff:63: + 3d:f7:b6:7c:f0:ca:bb:ae:25:7c:2e:fc:e2:95:55: + a6:73:3a:57:bb:70:d4:f6:0a:45:3c:46:87:f0:4f: + bf:41:62:f3:ff:cf:b6:64:7f:3d:58:69:e4:a1:96: + 24:eb:36:d2:94:83:14:75:5f:6e:e8:3d:f5:2c:3f: + 45:8b:7a:c6:c6:a8:5f:75:50:d1:b8:c8:d3:c9:51: + 94:e9:6b:35:28:1d:af:1f:13:0d:03:a3:3d:46:eb: + a9:bd + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Basic Constraints: + CA:FALSE + X509v3 Subject Key Identifier: + 2C:5F:48:C0:DD:04:AE:62:87:37:5E:15:96:B6:98:61:41:1F:4C:7F + X509v3 Authority Key Identifier: + keyid:28:17:AB:B2:C9:E4:02:38:D0:36:22:EA:B9:D0:1F:FB:2E:43:8B:2B + DirName:/CN=Testing CA + serial:19:37:10:6F:C3:6B:4E:C2:71:87:71:1F:DA:51:9D:E1:81:43:37:BA + + X509v3 Extended Key Usage: + TLS Web Server Authentication + X509v3 Key Usage: + Digital Signature, Key Encipherment + X509v3 Subject Alternative Name: + DNS:localhost, IP Address:127.0.0.1 + Signature Algorithm: sha256WithRSAEncryption + 70:11:fa:36:28:6b:0a:dc:7c:f5:2c:7e:a5:1a:e0:3c:c0:0d: + 0d:a2:2f:ed:50:71:79:46:0b:e9:af:d4:6b:f8:b0:95:f0:7e: + fc:9c:01:cd:57:92:27:46:3b:96:27:9b:2b:bc:d5:79:7f:fa: + 2b:02:f0:92:39:04:b8:3a:a1:09:2c:10:bf:48:8f:f9:43:31: + d5:d4:87:15:7f:82:72:ff:93:79:4f:02:53:cc:fd:e4:7d:c4: + 50:df:dd:59:86:66:dd:4a:ea:17:ce:b4:d9:9c:ed:b0:e0:13: + 22:66:58:95:33:46:af:5b:a6:fa:22:ac:aa:ce:dd:05:0d:85: + 68:b0:4a:72:97:a8:3a:39:3e:1d:79:1b:98:f6:a9:6e:35:ff: + 9c:0f:56:7c:65:52:8a:55:b7:6f:4f:90:3d:9b:6a:b0:45:56: + 74:80:c1:de:da:10:c2:d1:1b:52:9f:32:4f:a1:0f:90:28:02: + cb:2b:ab:fc:9a:18:a6:c8:c6:15:ba:55:f5:69:23:96:c2:c0: + c7:71:a5:c2:29:17:f7:ab:4a:a0:a5:58:e5:66:1d:c0:e1:e2: + 6e:05:7b:a9:dd:a6:19:9c:4f:ee:01:e4:a5:e8:4d:b5:2a:0f: + 42:af:5f:9b:b2:ab:c3:6d:4b:c9:5c:b9:7d:36:21:ce:99:2f: + 7a:0b:10:0d +-----BEGIN CERTIFICATE----- +MIIDdTCCAl2gAwIBAgIRAOjTmUCHwX3zk39P2pYOJoYwDQYJKoZIhvcNAQELBQAw +FTETMBEGA1UEAwwKVGVzdGluZyBDQTAeFw0yMTAxMDEwMDA0MTVaFw0zMDEyMjkw +MDA0MTVaMBcxFTATBgNVBAMMDHZhdWx0LXNlcnZlcjCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBALuWiEN/x0ua50waYvzAVsse90LWyhgWQSL0yu6YyTJE +VvoGVF+hLAoaG30X9rnMxsbNpCMdA8PVK6dM7TciIpERP5ZA9CbO4Nu+9BBde8qR +IngzLCKVX5jHac4za/fwT46xmtpjpYlkzsgk+1Yp9aUUMhhGIFpzd0K8HZBSXmpV +MEbuTSS7k4VrkhkJ9SUM8QKs60GFVTss0ZKhf3PXXjpA9v9jPfe2fPDKu64lfC78 +4pVVpnM6V7tw1PYKRTxGh/BPv0Fi8//PtmR/PVhp5KGWJOs20pSDFHVfbug99Sw/ +RYt6xsaoX3VQ0bjI08lRlOlrNSgdrx8TDQOjPUbrqb0CAwEAAaOBvTCBujAJBgNV +HRMEAjAAMB0GA1UdDgQWBBQsX0jA3QSuYoc3XhWWtphhQR9MfzBQBgNVHSMESTBH +gBQoF6uyyeQCONA2Iuq50B/7LkOLK6EZpBcwFTETMBEGA1UEAwwKVGVzdGluZyBD +QYIUGTcQb8NrTsJxh3Ef2lGd4YFDN7owEwYDVR0lBAwwCgYIKwYBBQUHAwEwCwYD +VR0PBAQDAgWgMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATANBgkqhkiG9w0B +AQsFAAOCAQEAcBH6NihrCtx89Sx+pRrgPMANDaIv7VBxeUYL6a/Ua/iwlfB+/JwB +zVeSJ0Y7liebK7zVeX/6KwLwkjkEuDqhCSwQv0iP+UMx1dSHFX+Ccv+TeU8CU8z9 +5H3EUN/dWYZm3UrqF8602ZztsOATImZYlTNGr1um+iKsqs7dBQ2FaLBKcpeoOjk+ +HXkbmPapbjX/nA9WfGVSilW3b0+QPZtqsEVWdIDB3toQwtEbUp8yT6EPkCgCyyur +/JoYpsjGFbpV9WkjlsLAx3GlwikX96tKoKVY5WYdwOHibgV7qd2mGZxP7gHkpehN +tSoPQq9fm7Krw21LyVy5fTYhzpkvegsQDQ== +-----END CERTIFICATE----- diff --git a/go/test/endtoend/vault/vault-key.pem b/go/test/endtoend/vault/vault-key.pem new file mode 100644 index 00000000000..bc1c7905a9b --- /dev/null +++ b/go/test/endtoend/vault/vault-key.pem @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQC7lohDf8dLmudM +GmL8wFbLHvdC1soYFkEi9MrumMkyRFb6BlRfoSwKGht9F/a5zMbGzaQjHQPD1Sun +TO03IiKRET+WQPQmzuDbvvQQXXvKkSJ4MywilV+Yx2nOM2v38E+OsZraY6WJZM7I +JPtWKfWlFDIYRiBac3dCvB2QUl5qVTBG7k0ku5OFa5IZCfUlDPECrOtBhVU7LNGS +oX9z1146QPb/Yz33tnzwyruuJXwu/OKVVaZzOle7cNT2CkU8RofwT79BYvP/z7Zk +fz1YaeShliTrNtKUgxR1X27oPfUsP0WLesbGqF91UNG4yNPJUZTpazUoHa8fEw0D +oz1G66m9AgMBAAECggEAWUDASM1tN63WS0Fqw7OIGFD9eJHVyiwchdNPEsMjR4V4 +lLGaR33aBFxzo8tZGwIxublyVTqi5fRxNsLFQyw8oiVAye7Ru/1Gw4dRfM/d7H2t +lt9SKopD199ZmkChKHDwiYY7lZk/0+Vg9Z2S8GY6eHbpdt822ZKCtf/nWRm3zoM1 +THvNEIwok6Rw85AYUaoKP8RsWsVh5thZKQyptIv3J4b2EFkl/mNWSSQOZDde8CUg +N5wkZL/FSZ6Nx7Fqs6Y76DfVweDUYy/tazj7OO9VvqJk6Wq2z40KoPYVLebpNKJ4 +mw+8sj7Hq91MT/oFzvvcP7FEIqnZBdB461dwXTcEgQKBgQDedI1c5aZ4PtHSLLqr +jsxqC2zOxt5eE+b093kFc9vQ92pbtY+nc0AhM6yN90q4jD0g4Ch832rvv1TXyNTc +TcYq187mvLI29Mprt0R3N9YHUNV6Iyc1d3o13RXqUqW67TrTx3ankn40gMy6Vqbg +3Jt6tetGoH6FfSFpOXRZYyIxLQKBgQDX3/99t0f/nDRm+jVpMmaWvyvp+of+CApf +fiElrLiLm1sqmv7mgHHrZBqNvA/B8s+l4vwI2HWwQMcnNYYLP5MJWfRXpGiSUZtD +uMDBWYyqxpiKgHla243Rjr3RVI4raA2MEu0vfgZjz7Ye7QpwCOPrcQMT2jaOG5Gp +FBXecxIU0QKBgQCtWybOvih8jHf20eSmzSF/gmfIvDGOHvRc8n3dQeyLbEP2NAc+ +9xGCzkIqYAxaxO7eL9Fdfr5XF0OG5Xr8M5+6w3L5XROEwD7+slMolNq12MiD5eEo +SXNzhlcNxFpi0XyGjWpqLD8tqzHgBKcHlOOVPS+cWnY+kMT4u01wW1DKAQKBgAOS +7MrrBuEfd+qgh9PXBsXGInb8M9Yr0egk0W2rP17oUokRCdlNFRW9kYb5LxWZ7IAl +kuCenMwvNlza0P5MriWAfMAas7SAb16ep2pMDj0hjpL0b43mhqGKiG/3w2bKkTbZ +dV3M61Qpsy0t5XdXXlaeh1uDyFVv9WhkMbx+ETWRAoGBAM/k0LHRiQipcsYOVTKM +8NcNAD/rc/xWNuOyB/8QvgxkuA1kphKLVNwBPg7riPj/Sc2JL2LZ063ki2PtKxEt +MzGPUNsRBHtIk8gQVktaxkMrRPgnf2XmG7Bh/OxRxK0wiaIWpezX4hs2DH26Pyoy +1CE1W4oEFSBD/Lp5TENBWNSo +-----END PRIVATE KEY----- diff --git a/go/test/endtoend/vault/vault-setup.sh b/go/test/endtoend/vault/vault-setup.sh new file mode 100644 index 00000000000..6ad393eda44 --- /dev/null +++ b/go/test/endtoend/vault/vault-setup.sh @@ -0,0 +1,46 @@ +# We expect environment variables like the following to be set +#export VAULT_ADDR=https://test:9123 +#export VAULT=/path/to/vitess/test/bin/vault-1.6.1 +#export VAULT_CACERT=./vault-cert.pem + +# For debugging purposes +set -x +TMPFILE=/tmp/setup.sh.tmp.$RANDOM +$VAULT operator init -key-shares=1 -key-threshold=1 | grep ": " | awk '{ print $NF }' > $TMPFILE +export UNSEAL="$(head -1 $TMPFILE)" +export VAULT_TOKEN="$(tail -1 $TMPFILE)" +rm -f $TMPFILE + +# Unseal Vault +$VAULT operator unseal $UNSEAL + +# Enable secrets engine (v2); prefix will be /kv +$VAULT secrets enable -version=2 kv + +# Enable approles +$VAULT auth enable approle + +# Write a custom policy to allow credential access +$VAULT policy write dbcreds dbcreds_policy.hcl + +# Load up the db credentials (vttablet -> MySQL) secret +$VAULT kv put kv/prod/dbcreds @dbcreds_secret.json + +# Load up the vtgate credentials (app -> vttablet) secret +$VAULT kv put kv/prod/vtgatecreds @vtgatecreds_secret.json + +# Configure approle +# Keep the ttl low, so we can test a refresh +$VAULT write auth/approle/role/vitess secret_id_ttl=10m token_num_uses=0 token_ttl=30s token_max_ttl=0 secret_id_num_uses=4 policies=dbcreds +$VAULT read auth/approle/role/vitess + +# Read the role-id of the approle, we need to extract it +export ROLE_ID=$($VAULT read auth/approle/role/vitess/role-id | grep ^role_id | awk '{ print $NF }') + +# Get a secret_id for the approle +export SECRET_ID=$($VAULT write auth/approle/role/vitess/secret-id k=v | grep ^secret_id | head -1 | awk '{ print $NF }') + +# Echo it back, so the controlling process can read it from the log +echo "ROLE_ID=$ROLE_ID" +echo "SECRET_ID=$SECRET_ID" + diff --git a/go/test/endtoend/vault/vault.hcl b/go/test/endtoend/vault/vault.hcl new file mode 100644 index 00000000000..4cb4dd314b5 --- /dev/null +++ b/go/test/endtoend/vault/vault.hcl @@ -0,0 +1,21 @@ +{ + "ui": "true", + "disable_mlock": "true", + "listener": [ + { + "tcp": { + "address": "$server:$port", + "tls_disable": 0, + "tls_cert_file": "$cert", + "tls_key_file": "$key" + } + } + ], + "storage": [ + { + "inmem": {} + } + ], + "default_lease_ttl": "168h", + "max_lease_ttl": "720h" +} diff --git a/go/test/endtoend/vault/vault_server.go b/go/test/endtoend/vault/vault_server.go new file mode 100644 index 00000000000..c5edfa56a85 --- /dev/null +++ b/go/test/endtoend/vault/vault_server.go @@ -0,0 +1,169 @@ +/* +Copyright 2020 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vault + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "os/exec" + "path" + "strings" + "syscall" + "time" + + "vitess.io/vitess/go/vt/log" +) + +const ( + vaultExecutableName = "vault" + vaultDownloadSource = "https://vitess-operator.storage.googleapis.com/install/vault" + vaultDownloadSize = 132738840 + vaultDirName = "vault" + vaultConfigFileName = "vault.hcl" + vaultCertFileName = "vault-cert.pem" + vaultCAFileName = "ca.pem" + vaultKeyFileName = "vault-key.pem" + vaultSetupScript = "vault-setup.sh" +) + +// VaultServer : Basic parameters for the running the Vault server +type VaultServer struct { + address string + port1 int + port2 int + execPath string + logDir string + + proc *exec.Cmd + exit chan error +} + +// Start the Vault server in dev mode +func (vs *VaultServer) start() error { + // Download and unpack vault binary + vs.execPath = path.Join(os.Getenv("EXTRA_BIN"), vaultExecutableName) + fileStat, err := os.Stat(vs.execPath) + if err != nil || fileStat.Size() != vaultDownloadSize { + log.Warningf("Downloading Vault binary to: %v", vs.execPath) + err := downloadExecFile(vs.execPath, vaultDownloadSource) + if err != nil { + log.Error(err) + return err + } + } else { + log.Warningf("Vault binary already present at %v , not re-downloading", vs.execPath) + } + + // Create Vault log directory + vs.logDir = path.Join(os.Getenv("VTDATAROOT"), fmt.Sprintf("%s_%d", vaultDirName, vs.port1)) + if _, err := os.Stat(vs.logDir); os.IsNotExist(err) { + err := os.Mkdir(vs.logDir, 0700) + if err != nil { + log.Error(err) + return err + } + } + + hclFile := path.Join(os.Getenv("PWD"), vaultConfigFileName) + hcl, _ := ioutil.ReadFile(hclFile) + // Replace variable parts in Vault config file + hcl = bytes.Replace(hcl, []byte("$server"), []byte(vs.address), 1) + hcl = bytes.Replace(hcl, []byte("$port"), []byte(fmt.Sprintf("%d", vs.port1)), 1) + hcl = bytes.Replace(hcl, []byte("$cert"), []byte(path.Join(os.Getenv("PWD"), vaultCertFileName)), 1) + hcl = bytes.Replace(hcl, []byte("$key"), []byte(path.Join(os.Getenv("PWD"), vaultKeyFileName)), 1) + newHclFile := path.Join(vs.logDir, vaultConfigFileName) + err = ioutil.WriteFile(newHclFile, hcl, 0700) + if err != nil { + log.Error(err) + return err + } + + vs.proc = exec.Command( + vs.execPath, + "server", + fmt.Sprintf("-config=%s", newHclFile), + ) + + logFile, err := os.Create(path.Join(vs.logDir, "log.txt")) + if err != nil { + log.Error(err) + return err + } + vs.proc.Stderr = logFile + vs.proc.Stdout = logFile + + vs.proc.Env = append(vs.proc.Env, os.Environ()...) + + log.Infof("Running Vault server with command: %v", strings.Join(vs.proc.Args, " ")) + + err = vs.proc.Start() + if err != nil { + return err + } + vs.exit = make(chan error) + go func() { + if vs.proc != nil { + vs.exit <- vs.proc.Wait() + } + }() + return nil +} + +func (vs *VaultServer) stop() error { + if vs.proc == nil || vs.exit == nil { + return nil + } + // Attempt graceful shutdown with SIGTERM first + vs.proc.Process.Signal(syscall.SIGTERM) + + select { + case err := <-vs.exit: + vs.proc = nil + return err + + case <-time.After(10 * time.Second): + vs.proc.Process.Kill() + vs.proc = nil + return <-vs.exit + } +} + +// Download file from url to path; making it executable +func downloadExecFile(path string, url string) error { + resp, err := http.Get(url) + if err != nil { + return err + } + defer resp.Body.Close() + + err = ioutil.WriteFile(path, []byte(""), 0700) + if err != nil { + return err + } + out, err := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, os.ModeAppend) + if err != nil { + return err + } + defer out.Close() + + _, err = io.Copy(out, resp.Body) + return err +} diff --git a/go/test/endtoend/vault/vault_test.go b/go/test/endtoend/vault/vault_test.go new file mode 100644 index 00000000000..79a9ba24f79 --- /dev/null +++ b/go/test/endtoend/vault/vault_test.go @@ -0,0 +1,314 @@ +/* +Copyright 2020 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vault + +import ( + "bufio" + "bytes" + "context" + "fmt" + "io/ioutil" + "net" + "os" + "os/exec" + "path" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/test/endtoend/cluster" + "vitess.io/vitess/go/vt/log" +) + +var ( + createTable = `create table product (id bigint(20) primary key, name char(10), created bigint(20));` + insertTable = `insert into product (id, name, created) values(%d, '%s', unix_timestamp());` +) + +var ( + clusterInstance *cluster.LocalProcessCluster + + master *cluster.Vttablet + replica *cluster.Vttablet + + cell = "zone1" + hostname = "localhost" + keyspaceName = "ks" + shardName = "0" + dbName = "vt_ks" + mysqlUsers = []string{"vt_dba", "vt_app", "vt_appdebug", "vt_repl", "vt_filtered"} + mysqlPassword = "password" + vtgateUser = "vtgate_user" + vtgatePassword = "password123" + commonTabletArg = []string{ + "-vreplication_healthcheck_topology_refresh", "1s", + "-vreplication_healthcheck_retry_delay", "1s", + "-vreplication_retry_delay", "1s", + "-degraded_threshold", "5s", + "-lock_tables_timeout", "5s", + "-watch_replication_stream", + // Frequently reload schema, generating some tablet traffic, + // so we can speed up token refresh + "-queryserver-config-schema-reload-time", "5", + "-serving_state_grace_period", "1s"} + vaultTabletArg = []string{ + "-db-credentials-server", "vault", + "-db-credentials-vault-timeout", "3s", + "-db-credentials-vault-path", "kv/prod/dbcreds", + // This is overriden by our env VAULT_ADDR + "-db-credentials-vault-addr", "https://127.0.0.1:8200", + // This is overriden by our env VAULT_CACERT + "-db-credentials-vault-tls-ca", "/path/to/ca.pem", + // This is provided by our env VAULT_ROLEID + //"-db-credentials-vault-roleid", "34644576-9ffc-8bb5-d046-4a0e41194e15", + // Contents of this file provided by our env VAULT_SECRETID + //"-db-credentials-vault-secretidfile", "/path/to/file/containing/secret_id", + // Make this small, so we can get a renewal + "-db-credentials-vault-ttl", "21s"} + vaultVTGateArg = []string{ + "-mysql_auth_server_impl", "vault", + "-mysql_auth_vault_timeout", "3s", + "-mysql_auth_vault_path", "kv/prod/vtgatecreds", + // This is overriden by our env VAULT_ADDR + "-mysql_auth_vault_addr", "https://127.0.0.1:8200", + // This is overriden by our env VAULT_CACERT + "-mysql_auth_vault_tls_ca", "/path/to/ca.pem", + // This is provided by our env VAULT_ROLEID + //"-mysql_auth_vault_roleid", "34644576-9ffc-8bb5-d046-4a0e41194e15", + // Contents of this file provided by our env VAULT_SECRETID + //"-mysql_auth_vault_role_secretidfile", "/path/to/file/containing/secret_id", + // Make this small, so we can get a renewal + "-mysql_auth_vault_ttl", "21s"} + mysqlctlArg = []string{ + "-db_dba_password", mysqlPassword} + vttabletLogFileName = "vttablet.INFO" + tokenRenewalString = "Vault client status: token renewed" +) + +func TestVaultAuth(t *testing.T) { + defer cluster.PanicHandler(nil) + + // Instantiate Vitess Cluster objects and start topo + initializeClusterEarly(t) + defer clusterInstance.Teardown() + + // start Vault server + vs := startVaultServer(t, master) + defer vs.stop() + + // Wait for Vault server to come up + for i := 0; i < 60; i++ { + time.Sleep(250 * time.Millisecond) + ln, err := net.Listen("tcp", fmt.Sprintf("%s:%d", hostname, vs.port1)) + if err != nil { + // Vault is now up, we can continue + break + } + ln.Close() + } + + roleID, secretID := setupVaultServer(t, vs) + require.NotEmpty(t, roleID) + require.NotEmpty(t, secretID) + + // Passing via environment, easier than trying to modify + // vtgate/vttablet flags within our test machinery + os.Setenv("VAULT_ROLEID", roleID) + os.Setenv("VAULT_SECRETID", secretID) + + // Bring up rest of the Vitess cluster + initializeClusterLate(t) + + // Create a table + _, err := master.VttabletProcess.QueryTablet(createTable, keyspaceName, true) + require.NoError(t, err) + + // This tests the vtgate Vault auth & indirectly vttablet Vault auth too + insertRow(t, 1, "prd-1") + insertRow(t, 2, "prd-2") + + cluster.VerifyRowsInTabletForTable(t, replica, keyspaceName, 2, "product") + + // Sleep for a while; giving enough time for a token renewal + // and it making it into the (asynchronous) log + time.Sleep(30 * time.Second) + // Check the log for the Vault token renewal message + // If we don't see it, that is a test failure + logContents, _ := ioutil.ReadFile(path.Join(clusterInstance.TmpDirectory, vttabletLogFileName)) + require.True(t, bytes.Contains(logContents, []byte(tokenRenewalString))) +} + +func startVaultServer(t *testing.T, masterTablet *cluster.Vttablet) *VaultServer { + vs := &VaultServer{ + address: hostname, + port1: clusterInstance.GetAndReservePort(), + port2: clusterInstance.GetAndReservePort(), + } + err := vs.start() + require.NoError(t, err) + + return vs +} + +// Setup everything we need in the Vault server +func setupVaultServer(t *testing.T, vs *VaultServer) (string, string) { + // The setup script uses these environment variables + // We also reuse VAULT_ADDR and VAULT_CACERT later on + os.Setenv("VAULT", vs.execPath) + os.Setenv("VAULT_ADDR", fmt.Sprintf("https://%s:%d", vs.address, vs.port1)) + os.Setenv("VAULT_CACERT", path.Join(os.Getenv("PWD"), vaultCAFileName)) + setup := exec.Command( + "/bin/bash", + path.Join(os.Getenv("PWD"), vaultSetupScript), + ) + + logFilePath := path.Join(vs.logDir, "log_setup.txt") + logFile, _ := os.Create(logFilePath) + setup.Stderr = logFile + setup.Stdout = logFile + + setup.Env = append(setup.Env, os.Environ()...) + log.Infof("Running Vault setup command: %v", strings.Join(setup.Args, " ")) + err := setup.Start() + if err != nil { + log.Errorf("Error during Vault setup: %v", err) + } + + setup.Wait() + var secretID, roleID string + file, err := os.Open(logFilePath) + if err != nil { + log.Error(err) + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + if strings.HasPrefix(scanner.Text(), "ROLE_ID=") { + roleID = strings.Split(scanner.Text(), "=")[1] + } else if strings.HasPrefix(scanner.Text(), "SECRET_ID=") { + secretID = strings.Split(scanner.Text(), "=")[1] + } + } + if err := scanner.Err(); err != nil { + log.Error(err) + } + + return roleID, secretID +} + +// Setup cluster object and start topo +// We need this before vault, because we re-use the port reservation code +func initializeClusterEarly(t *testing.T) { + clusterInstance = cluster.NewCluster(cell, hostname) + + // Start topo server + err := clusterInstance.StartTopo() + require.NoError(t, err) +} + +func initializeClusterLate(t *testing.T) { + // Start keyspace + keyspace := &cluster.Keyspace{ + Name: keyspaceName, + } + clusterInstance.Keyspaces = append(clusterInstance.Keyspaces, *keyspace) + shard := &cluster.Shard{ + Name: shardName, + } + + master = clusterInstance.NewVttabletInstance("replica", 0, "") + // We don't really need the replica to test this feature + // but keeping it in to excercise the vt_repl user/password path + replica = clusterInstance.NewVttabletInstance("replica", 0, "") + + shard.Vttablets = []*cluster.Vttablet{master, replica} + + clusterInstance.VtTabletExtraArgs = append(clusterInstance.VtTabletExtraArgs, commonTabletArg...) + clusterInstance.VtTabletExtraArgs = append(clusterInstance.VtTabletExtraArgs, vaultTabletArg...) + clusterInstance.VtGateExtraArgs = append(clusterInstance.VtGateExtraArgs, vaultVTGateArg...) + + err := clusterInstance.SetupCluster(keyspace, []cluster.Shard{*shard}) + require.NoError(t, err) + + // Start MySQL + var mysqlCtlProcessList []*exec.Cmd + for _, shard := range clusterInstance.Keyspaces[0].Shards { + for _, tablet := range shard.Vttablets { + proc, err := tablet.MysqlctlProcess.StartProcess() + require.NoError(t, err) + mysqlCtlProcessList = append(mysqlCtlProcessList, proc) + } + } + + // Wait for MySQL startup + for _, proc := range mysqlCtlProcessList { + err = proc.Wait() + require.NoError(t, err) + } + + for _, tablet := range []*cluster.Vttablet{master, replica} { + for _, user := range mysqlUsers { + query := fmt.Sprintf("ALTER USER '%s'@'%s' IDENTIFIED BY '%s';", user, hostname, mysqlPassword) + _, err = tablet.VttabletProcess.QueryTablet(query, keyspace.Name, false) + // Reset after the first ALTER, or we lock ourselves out. + tablet.VttabletProcess.DbPassword = mysqlPassword + if err != nil { + query = fmt.Sprintf("ALTER USER '%s'@'%%' IDENTIFIED BY '%s';", user, mysqlPassword) + _, err = tablet.VttabletProcess.QueryTablet(query, keyspace.Name, false) + require.NoError(t, err) + } + } + query := fmt.Sprintf("create database %s;", dbName) + _, err = tablet.VttabletProcess.QueryTablet(query, keyspace.Name, false) + require.NoError(t, err) + + tablet.VttabletProcess.EnableSemiSync = true + err = tablet.VttabletProcess.Setup() + require.NoError(t, err) + + // Modify mysqlctl password too, or teardown will be locked out + tablet.MysqlctlProcess.ExtraArgs = append(tablet.MysqlctlProcess.ExtraArgs, mysqlctlArg...) + } + + err = clusterInstance.VtctlclientProcess.InitShardMaster(keyspaceName, shard.Name, cell, master.TabletUID) + require.NoError(t, err) + + // Start vtgate + err = clusterInstance.StartVtgate() + require.NoError(t, err) +} + +func insertRow(t *testing.T, id int, productName string) { + ctx := context.Background() + vtParams := mysql.ConnParams{ + Host: clusterInstance.Hostname, + Port: clusterInstance.VtgateMySQLPort, + Uname: vtgateUser, + Pass: vtgatePassword, + } + conn, err := mysql.Connect(ctx, &vtParams) + require.NoError(t, err) + defer conn.Close() + + insertSmt := fmt.Sprintf(insertTable, id, productName) + _, err = conn.ExecuteFetch(insertSmt, 1000, true) + require.NoError(t, err) +} diff --git a/go/test/endtoend/vault/vtgatecreds_secret.json b/go/test/endtoend/vault/vtgatecreds_secret.json new file mode 100644 index 00000000000..568609419f0 --- /dev/null +++ b/go/test/endtoend/vault/vtgatecreds_secret.json @@ -0,0 +1,7 @@ +{ + "vtgate_user": [ + { + "Password": "password123" + } + ] +} diff --git a/go/vt/dbconfigs/credentials.go b/go/vt/dbconfigs/credentials.go index 026c9bafd6b..760fc13d4d4 100644 --- a/go/vt/dbconfigs/credentials.go +++ b/go/vt/dbconfigs/credentials.go @@ -28,8 +28,12 @@ import ( "io/ioutil" "os" "os/signal" + "strings" "sync" "syscall" + "time" + + vaultapi "github.com/aquarapid/vaultlib" "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/vt/log" @@ -37,11 +41,22 @@ import ( var ( // generic flags - dbCredentialsServer = flag.String("db-credentials-server", "file", "db credentials server type (use 'file' for the file implementation)") + dbCredentialsServer = flag.String("db-credentials-server", "file", "db credentials server type ('file' - file implementation; 'vault' - HashiCorp Vault implementation)") // 'file' implementation flags dbCredentialsFile = flag.String("db-credentials-file", "", "db credentials file; send SIGHUP to reload this file") + // 'vault' implementation flags + vaultAddr = flag.String("db-credentials-vault-addr", "", "URL to Vault server") + vaultTimeout = flag.Duration("db-credentials-vault-timeout", 10*time.Second, "Timeout for vault API operations") + vaultCACert = flag.String("db-credentials-vault-tls-ca", "", "Path to CA PEM for validating Vault server certificate") + vaultPath = flag.String("db-credentials-vault-path", "", "Vault path to credentials JSON blob, e.g.: secret/data/prod/dbcreds") + vaultCacheTTL = flag.Duration("db-credentials-vault-ttl", 30*time.Minute, "How long to cache DB credentials from the Vault server") + vaultTokenFile = flag.String("db-credentials-vault-tokenfile", "", "Path to file containing Vault auth token; token can also be passed using VAULT_TOKEN environment variable") + vaultRoleID = flag.String("db-credentials-vault-roleid", "", "Vault AppRole id; can also be passed using VAULT_ROLEID environment variable") + vaultRoleSecretIDFile = flag.String("db-credentials-vault-role-secretidfile", "", "Path to file containing Vault AppRole secret_id; can also be passed using VAULT_SECRETID environment variable") + vaultRoleMountPoint = flag.String("db-credentials-vault-role-mountpoint", "approle", "Vault AppRole mountpoint; can also be passed using VAULT_MOUNTPOINT environment variable") + // ErrUnknownUser is returned by credential server when the // user doesn't exist ErrUnknownUser = errors.New("unknown user") @@ -79,6 +94,18 @@ type FileCredentialsServer struct { dbCredentials map[string][]string } +// VaultCredentialsServer implements CredentialsServer using +// a Vault backend from HashiCorp. +type VaultCredentialsServer struct { + mu sync.Mutex + dbCredsCache map[string][]string + vaultCacheExpireTicker *time.Ticker + vaultClient *vaultapi.Client + // We use a separate valid flag to allow invalidating the cache + // without destroying it, in case Vault is temp down. + cacheValid bool +} + // GetUserAndPassword is part of the CredentialsServer interface func (fcs *FileCredentialsServer) GetUserAndPassword(user string) (string, string, error) { fcs.mu.Lock() @@ -111,6 +138,124 @@ func (fcs *FileCredentialsServer) GetUserAndPassword(user string) (string, strin return user, passwd[0], nil } +// GetUserAndPassword for Vault implementation +func (vcs *VaultCredentialsServer) GetUserAndPassword(user string) (string, string, error) { + vcs.mu.Lock() + defer vcs.mu.Unlock() + + if vcs.vaultCacheExpireTicker == nil { + vcs.vaultCacheExpireTicker = time.NewTicker(*vaultCacheTTL) + go func() { + for range vcs.vaultCacheExpireTicker.C { + if vcs, ok := AllCredentialsServers["vault"].(*VaultCredentialsServer); ok { + vcs.cacheValid = false + } + } + }() + } + + if vcs.cacheValid && vcs.dbCredsCache != nil { + if vcs.dbCredsCache[user] == nil { + log.Errorf("Vault cache is valid, but user %s unknown in cache, will retry", user) + return "", "", ErrUnknownUser + } + return user, vcs.dbCredsCache[user][0], nil + } + + if *vaultAddr == "" { + return "", "", errors.New("No Vault server specified") + } + + token, err := readFromFile(*vaultTokenFile) + if err != nil { + return "", "", errors.New("No Vault token in provided filename") + } + secretID, err := readFromFile(*vaultRoleSecretIDFile) + if err != nil { + return "", "", errors.New("No Vault secret_id in provided filename") + } + + // From here on, errors might be transient, so we use ErrUnknownUser + // for everything, so we get retries + if vcs.vaultClient == nil { + config := vaultapi.NewConfig() + + // All these can be overriden by environment + // so we need to check if they have been set by NewConfig + if config.Address == "" { + config.Address = *vaultAddr + } + if config.Timeout == (0 * time.Second) { + config.Timeout = *vaultTimeout + } + if config.CACert == "" { + config.CACert = *vaultCACert + } + if config.Token == "" { + config.Token = token + } + if config.AppRoleCredentials.RoleID == "" { + config.AppRoleCredentials.RoleID = *vaultRoleID + } + if config.AppRoleCredentials.SecretID == "" { + config.AppRoleCredentials.SecretID = secretID + } + if config.AppRoleCredentials.MountPoint == "" { + config.AppRoleCredentials.MountPoint = *vaultRoleMountPoint + } + + if config.CACert != "" { + // If we provide a CA, ensure we actually use it + config.InsecureSSL = false + } + + var err error + vcs.vaultClient, err = vaultapi.NewClient(config) + if err != nil || vcs.vaultClient == nil { + log.Errorf("Error in vault client initialization, will retry: %v", err) + vcs.vaultClient = nil + return "", "", ErrUnknownUser + } + } + + secret, err := vcs.vaultClient.GetSecret(*vaultPath) + if err != nil { + log.Errorf("Error in Vault server params: %v", err) + return "", "", ErrUnknownUser + } + + if secret.JSONSecret == nil { + log.Errorf("Empty DB credentials retrieved from Vault server") + return "", "", ErrUnknownUser + } + + dbCreds := make(map[string][]string) + if err = json.Unmarshal(secret.JSONSecret, &dbCreds); err != nil { + log.Errorf("Error unmarshaling DB credentials from Vault server") + return "", "", ErrUnknownUser + } + if dbCreds[user] == nil { + log.Warningf("Vault lookup for user not found: %v\n", user) + return "", "", ErrUnknownUser + } + log.Infof("Vault client status: %s", vcs.vaultClient.GetStatus()) + + vcs.dbCredsCache = dbCreds + vcs.cacheValid = true + return user, dbCreds[user][0], nil +} + +func readFromFile(filePath string) (string, error) { + if filePath == "" { + return "", nil + } + fileBytes, err := ioutil.ReadFile(filePath) + if err != nil { + return "", err + } + return strings.TrimSpace(string(fileBytes)), nil +} + // WithCredentials returns a copy of the provided ConnParams that we can use // to connect, after going through the CredentialsServer. func withCredentials(cp *mysql.ConnParams) (*mysql.ConnParams, error) { @@ -122,6 +267,8 @@ func withCredentials(cp *mysql.ConnParams) (*mysql.ConnParams, error) { result.Pass = passwd case ErrUnknownUser: // we just use what we have, and will fail later anyway + // except if the actual password is empty, in which case + // things will just "work" err = nil } return &result, err @@ -129,6 +276,7 @@ func withCredentials(cp *mysql.ConnParams) (*mysql.ConnParams, error) { func init() { AllCredentialsServers["file"] = &FileCredentialsServer{} + AllCredentialsServers["vault"] = &VaultCredentialsServer{} sigChan := make(chan os.Signal, 1) signal.Notify(sigChan, syscall.SIGHUP) @@ -139,6 +287,11 @@ func init() { fcs.dbCredentials = nil fcs.mu.Unlock() } + if vcs, ok := AllCredentialsServers["vault"].(*VaultCredentialsServer); ok { + vcs.mu.Lock() + vcs.dbCredsCache = nil + vcs.mu.Unlock() + } } }() } diff --git a/go/vt/vtgate/plugin_mysql_server.go b/go/vt/vtgate/plugin_mysql_server.go index 08a460534d8..60b5121d236 100644 --- a/go/vt/vtgate/plugin_mysql_server.go +++ b/go/vt/vtgate/plugin_mysql_server.go @@ -52,7 +52,7 @@ var ( mysqlServerBindAddress = flag.String("mysql_server_bind_address", "", "Binds on this address when listening to MySQL binary protocol. Useful to restrict listening to 'localhost' only for instance.") mysqlServerSocketPath = flag.String("mysql_server_socket_path", "", "This option specifies the Unix socket file to use when listening for local connections. By default it will be empty and it won't listen to a unix socket") mysqlTCPVersion = flag.String("mysql_tcp_version", "tcp", "Select tcp, tcp4, or tcp6 to control the socket type.") - mysqlAuthServerImpl = flag.String("mysql_auth_server_impl", "static", "Which auth server implementation to use.") + mysqlAuthServerImpl = flag.String("mysql_auth_server_impl", "static", "Which auth server implementation to use. Options: none, ldap, clientcert, static, vault.") mysqlAllowClearTextWithoutTLS = flag.Bool("mysql_allow_clear_text_without_tls", false, "If set, the server will allow the use of a clear text password over non-SSL connections.") mysqlServerVersion = flag.String("mysql_server_version", mysql.DefaultServerVersion, "MySQL server version to advertise.") mysqlProxyProtocol = flag.Bool("proxy_protocol", false, "Enable HAProxy PROXY protocol on MySQL listener socket") diff --git a/go/vt/vttablet/tabletmanager/tm_init.go b/go/vt/vttablet/tabletmanager/tm_init.go index 375b5c46b88..c45991f4cb6 100644 --- a/go/vt/vttablet/tabletmanager/tm_init.go +++ b/go/vt/vttablet/tabletmanager/tm_init.go @@ -511,7 +511,11 @@ func (tm *TabletManager) checkMastership(ctx context.Context, si *topo.ShardInfo } func (tm *TabletManager) checkMysql(ctx context.Context) error { - if appConfig, _ := tm.DBConfigs.AppWithDB().MysqlParams(); appConfig.Host != "" { + appConfig, err := tm.DBConfigs.AppWithDB().MysqlParams() + if err != nil { + return err + } + if appConfig.Host != "" { tm.tmState.UpdateTablet(func(tablet *topodatapb.Tablet) { tablet.MysqlHostname = appConfig.Host tablet.MysqlPort = int32(appConfig.Port) diff --git a/test/config.json b/test/config.json index a7a48e23a1b..691ff4370a3 100644 --- a/test/config.json +++ b/test/config.json @@ -616,6 +616,15 @@ "RetryMax": 0, "Tags": [] }, + "vault": { + "File": "unused.go", + "Args": ["vitess.io/vitess/go/test/endtoend/vault"], + "Command": [], + "Manual": false, + "Shard": 23, + "RetryMax": 0, + "Tags": [] + }, "vreplication_v2": { "File": "unused.go", "Args": ["vitess.io/vitess/go/test/endtoend/vreplication", "-run", "TestBasicV2Workflows"],