From 4a8c33cca36b6e188d5c41f45e5c9e5a32e58c01 Mon Sep 17 00:00:00 2001 From: Chris Hoffman Date: Fri, 15 Sep 2017 00:21:35 -0400 Subject: [PATCH] Disable the `sys/raw` endpoint by default (#3329) * disable raw endpoint by default * adding docs * config option raw -> raw_storage_endpoint * docs updates * adding listing on raw endpoint * reworking tests for enabled raw endpoints * root protecting base raw endpoint --- command/server.go | 1 + command/server/config.go | 23 +++++-- command/server/config_test.go | 10 +++ .../server/test-fixtures/config-dir/baz.hcl | 1 + command/server/test-fixtures/config.hcl | 1 + command/server/test-fixtures/config.hcl.json | 3 +- command/server/test-fixtures/config2.hcl.json | 1 + vault/core.go | 7 ++ vault/logical_system.go | 67 +++++++++++++------ vault/logical_system_test.go | 42 ++++++------ vault/seal_testing.go | 2 +- vault/testing.go | 27 +++++++- website/source/api/system/raw.html.md | 39 +++++++++++ .../source/docs/configuration/index.html.md | 4 ++ 14 files changed, 179 insertions(+), 49 deletions(-) diff --git a/command/server.go b/command/server.go index e5101fd124da..cb549f4ed9b2 100644 --- a/command/server.go +++ b/command/server.go @@ -260,6 +260,7 @@ func (c *ServerCommand) Run(args []string) int { ClusterName: config.ClusterName, CacheSize: config.CacheSize, PluginDirectory: config.PluginDirectory, + EnableRaw: config.EnableRawEndpoint, } if dev { coreConfig.DevToken = devRootTokenID diff --git a/command/server/config.go b/command/server/config.go index d1f7c23e627d..7c20a371ffde 100644 --- a/command/server/config.go +++ b/command/server/config.go @@ -46,13 +46,17 @@ type Config struct { ClusterCipherSuites string `hcl:"cluster_cipher_suites"` PluginDirectory string `hcl:"plugin_directory"` + + EnableRawEndpoint bool `hcl:"-"` + EnableRawEndpointRaw interface{} `hcl:"raw_storage_endpoint"` } // DevConfig is a Config that is used for dev mode of Vault. func DevConfig(ha, transactional bool) *Config { ret := &Config{ - DisableCache: false, - DisableMlock: true, + DisableCache: false, + DisableMlock: true, + EnableRawEndpoint: true, Storage: &Storage{ Type: "inmem", @@ -288,6 +292,11 @@ func (c *Config) Merge(c2 *Config) *Config { result.EnableUI = c2.EnableUI } + result.EnableRawEndpoint = c.EnableRawEndpoint + if c2.EnableRawEndpoint { + result.EnableRawEndpoint = c2.EnableRawEndpoint + } + result.PluginDirectory = c.PluginDirectory if c2.PluginDirectory != "" { result.PluginDirectory = c2.PluginDirectory @@ -306,9 +315,8 @@ func LoadConfig(path string, logger log.Logger) (*Config, error) { if fi.IsDir() { return LoadConfigDir(path, logger) - } else { - return LoadConfigFile(path, logger) } + return LoadConfigFile(path, logger) } // LoadConfigFile loads the configuration from the given file. @@ -363,6 +371,12 @@ func ParseConfig(d string, logger log.Logger) (*Config, error) { } } + if result.EnableRawEndpointRaw != nil { + if result.EnableRawEndpoint, err = parseutil.ParseBool(result.EnableRawEndpointRaw); err != nil { + return nil, err + } + } + list, ok := obj.Node.(*ast.ObjectList) if !ok { return nil, fmt.Errorf("error parsing: file doesn't contain a root object") @@ -385,6 +399,7 @@ func ParseConfig(d string, logger log.Logger) (*Config, error) { "cluster_name", "cluster_cipher_suites", "plugin_directory", + "raw_storage_endpoint", } if err := checkHCLKeys(list, valid); err != nil { return nil, err diff --git a/command/server/config_test.go b/command/server/config_test.go index 49cf93a515bb..8e0efe70406b 100644 --- a/command/server/config_test.go +++ b/command/server/config_test.go @@ -62,6 +62,9 @@ func TestLoadConfigFile(t *testing.T) { EnableUI: true, EnableUIRaw: true, + EnableRawEndpoint: true, + EnableRawEndpointRaw: true, + MaxLeaseTTL: 10 * time.Hour, MaxLeaseTTLRaw: "10h", DefaultLeaseTTL: 10 * time.Hour, @@ -129,6 +132,9 @@ func TestLoadConfigFile_json(t *testing.T) { DisableMlockRaw: interface{}(nil), EnableUI: true, EnableUIRaw: true, + + EnableRawEndpoint: true, + EnableRawEndpointRaw: true, } if !reflect.DeepEqual(config, expected) { t.Fatalf("expected \n\n%#v\n\n to be \n\n%#v\n\n", config, expected) @@ -178,6 +184,8 @@ func TestLoadConfigFile_json2(t *testing.T) { EnableUI: true, + EnableRawEndpoint: true, + Telemetry: &Telemetry{ StatsiteAddr: "foo", StatsdAddr: "bar", @@ -232,6 +240,8 @@ func TestLoadConfigDir(t *testing.T) { EnableUI: true, + EnableRawEndpoint: true, + Telemetry: &Telemetry{ StatsiteAddr: "qux", StatsdAddr: "baz", diff --git a/command/server/test-fixtures/config-dir/baz.hcl b/command/server/test-fixtures/config-dir/baz.hcl index 7d8864094aef..bae5bdb4cbad 100644 --- a/command/server/test-fixtures/config-dir/baz.hcl +++ b/command/server/test-fixtures/config-dir/baz.hcl @@ -4,5 +4,6 @@ telemetry { disable_hostname = true } ui=true +raw_storage_endpoint=true default_lease_ttl = "10h" cluster_name = "testcluster" diff --git a/command/server/test-fixtures/config.hcl b/command/server/test-fixtures/config.hcl index b7826d8a8348..f4866d8cb0c0 100644 --- a/command/server/test-fixtures/config.hcl +++ b/command/server/test-fixtures/config.hcl @@ -28,3 +28,4 @@ telemetry { max_lease_ttl = "10h" default_lease_ttl = "10h" cluster_name = "testcluster" +raw_storage_endpoint = true \ No newline at end of file diff --git a/command/server/test-fixtures/config.hcl.json b/command/server/test-fixtures/config.hcl.json index 5723eb2b8630..fc1fda099cf5 100644 --- a/command/server/test-fixtures/config.hcl.json +++ b/command/server/test-fixtures/config.hcl.json @@ -17,5 +17,6 @@ "max_lease_ttl": "10h", "default_lease_ttl": "10h", "cluster_name":"testcluster", - "ui":true + "ui":true, + "raw_storage_endpoint":true } diff --git a/command/server/test-fixtures/config2.hcl.json b/command/server/test-fixtures/config2.hcl.json index 5279d637954a..e1eb73e5e2dc 100644 --- a/command/server/test-fixtures/config2.hcl.json +++ b/command/server/test-fixtures/config2.hcl.json @@ -1,5 +1,6 @@ { "ui":true, + "raw_storage_endpoint":true, "listener":[ { "tcp":{ diff --git a/vault/core.go b/vault/core.go index 3d169abb6255..d0472a467e21 100644 --- a/vault/core.go +++ b/vault/core.go @@ -344,6 +344,9 @@ type Core struct { // uiEnabled indicates whether Vault Web UI is enabled or not uiEnabled bool + // rawEnabled indicates whether the Raw endpoint is enabled + rawEnabled bool + // pluginDirectory is the location vault will look for plugin binaries pluginDirectory string @@ -402,6 +405,9 @@ type CoreConfig struct { EnableUI bool `json:"ui" structs:"ui" mapstructure:"ui"` + // Enable the raw endpoint + EnableRaw bool `json:"enable_raw" structs:"enable_raw" mapstructure:"enable_raw"` + PluginDirectory string `json:"plugin_directory" structs:"plugin_directory" mapstructure:"plugin_directory"` ReloadFuncs *map[string][]reload.ReloadFunc @@ -462,6 +468,7 @@ func NewCore(conf *CoreConfig) (*Core, error) { clusterListenerShutdownSuccessCh: make(chan struct{}), clusterPeerClusterAddrsCache: cache.New(3*heartbeatInterval, time.Second), enableMlock: !conf.DisableMlock, + rawEnabled: conf.EnableRaw, } if conf.ClusterCipherSuites != "" { diff --git a/vault/logical_system.go b/vault/logical_system.go index b6dd9a4c1b92..1593a1f2f9c4 100644 --- a/vault/logical_system.go +++ b/vault/logical_system.go @@ -22,7 +22,7 @@ var ( // protectedPaths cannot be accessed via the raw APIs. // This is both for security and to prevent disrupting Vault. protectedPaths = []string{ - "core", + keyringPath, } replicationPaths = func(b *SystemBackend) []*framework.Path { @@ -59,6 +59,7 @@ func NewSystemBackend(core *Core) *SystemBackend { "remount", "audit", "audit/*", + "raw", "raw/*", "replication/primary/secondary-token", "replication/reindex", @@ -652,25 +653,6 @@ func NewSystemBackend(core *Core) *SystemBackend { HelpDescription: strings.TrimSpace(sysHelp["audit"][1]), }, - &framework.Path{ - Pattern: "raw/(?P.+)", - - Fields: map[string]*framework.FieldSchema{ - "path": &framework.FieldSchema{ - Type: framework.TypeString, - }, - "value": &framework.FieldSchema{ - Type: framework.TypeString, - }, - }, - - Callbacks: map[logical.Operation]framework.OperationFunc{ - logical.ReadOperation: b.handleRawRead, - logical.UpdateOperation: b.handleRawWrite, - logical.DeleteOperation: b.handleRawDelete, - }, - }, - &framework.Path{ Pattern: "key-status$", @@ -871,6 +853,28 @@ func NewSystemBackend(core *Core) *SystemBackend { b.Backend.Paths = append(b.Backend.Paths, replicationPaths(b)...) + if core.rawEnabled { + b.Backend.Paths = append(b.Backend.Paths, &framework.Path{ + Pattern: "(raw/?$|raw/(?P.+))", + + Fields: map[string]*framework.FieldSchema{ + "path": &framework.FieldSchema{ + Type: framework.TypeString, + }, + "value": &framework.FieldSchema{ + Type: framework.TypeString, + }, + }, + + Callbacks: map[logical.Operation]framework.OperationFunc{ + logical.ReadOperation: b.handleRawRead, + logical.UpdateOperation: b.handleRawWrite, + logical.DeleteOperation: b.handleRawDelete, + logical.ListOperation: b.handleRawList, + }, + }) + } + b.Backend.Invalidate = b.invalidate return b @@ -2143,6 +2147,29 @@ func (b *SystemBackend) handleRawDelete( return nil, nil } +// handleRawList is used to list directly from the barrier +func (b *SystemBackend) handleRawList( + req *logical.Request, data *framework.FieldData) (*logical.Response, error) { + path := data.Get("path").(string) + if path != "" && !strings.HasSuffix(path, "/") { + path = path + "/" + } + + // Prevent access of protected paths + for _, p := range protectedPaths { + if strings.HasPrefix(path, p) { + err := fmt.Sprintf("cannot list '%s'", path) + return logical.ErrorResponse(err), logical.ErrInvalidRequest + } + } + + keys, err := b.Core.barrier.List(path) + if err != nil { + return handleError(err) + } + return logical.ListResponse(keys), nil +} + // handleKeyStatus returns status information about the backend key func (b *SystemBackend) handleKeyStatus( req *logical.Request, data *framework.FieldData) (*logical.Response, error) { diff --git a/vault/logical_system_test.go b/vault/logical_system_test.go index ecc5362eca34..613e4ea2c139 100644 --- a/vault/logical_system_test.go +++ b/vault/logical_system_test.go @@ -27,6 +27,7 @@ func TestSystemBackend_RootPaths(t *testing.T) { "remount", "audit", "audit/*", + "raw", "raw/*", "replication/primary/secondary-token", "replication/reindex", @@ -1447,7 +1448,7 @@ func TestSystemBackend_disableAudit(t *testing.T) { } func TestSystemBackend_rawRead_Protected(t *testing.T) { - b := testSystemBackend(t) + b := testSystemBackendRaw(t) req := logical.TestRequest(t, logical.ReadOperation, "raw/"+keyringPath) _, err := b.HandleRequest(req) @@ -1457,7 +1458,7 @@ func TestSystemBackend_rawRead_Protected(t *testing.T) { } func TestSystemBackend_rawWrite_Protected(t *testing.T) { - b := testSystemBackend(t) + b := testSystemBackendRaw(t) req := logical.TestRequest(t, logical.UpdateOperation, "raw/"+keyringPath) _, err := b.HandleRequest(req) @@ -1467,7 +1468,7 @@ func TestSystemBackend_rawWrite_Protected(t *testing.T) { } func TestSystemBackend_rawReadWrite(t *testing.T) { - c, b, _ := testCoreSystemBackend(t) + c, b, _ := testCoreSystemBackendRaw(t) req := logical.TestRequest(t, logical.UpdateOperation, "raw/sys/policy/test") req.Data["value"] = `path "secret/" { policy = "read" }` @@ -1503,7 +1504,7 @@ func TestSystemBackend_rawReadWrite(t *testing.T) { } func TestSystemBackend_rawDelete_Protected(t *testing.T) { - b := testSystemBackend(t) + b := testSystemBackendRaw(t) req := logical.TestRequest(t, logical.DeleteOperation, "raw/"+keyringPath) _, err := b.HandleRequest(req) @@ -1513,7 +1514,7 @@ func TestSystemBackend_rawDelete_Protected(t *testing.T) { } func TestSystemBackend_rawDelete(t *testing.T) { - c, b, _ := testCoreSystemBackend(t) + c, b, _ := testCoreSystemBackendRaw(t) // set the policy! p := &Policy{Name: "test"} @@ -1589,25 +1590,25 @@ func TestSystemBackend_rotate(t *testing.T) { func testSystemBackend(t *testing.T) logical.Backend { c, _, _ := TestCoreUnsealed(t) - bc := &logical.BackendConfig{ - Logger: c.logger, - System: logical.StaticSystemView{ - DefaultLeaseTTLVal: time.Hour * 24, - MaxLeaseTTLVal: time.Hour * 24 * 32, - }, - } - - b := NewSystemBackend(c) - err := b.Backend.Setup(bc) - if err != nil { - t.Fatal(err) - } + return testSystemBackendInternal(t, c) +} - return b +func testSystemBackendRaw(t *testing.T) logical.Backend { + c, _, _ := TestCoreUnsealedRaw(t) + return testSystemBackendInternal(t, c) } func testCoreSystemBackend(t *testing.T) (*Core, logical.Backend, string) { c, _, root := TestCoreUnsealed(t) + return c, testSystemBackendInternal(t, c), root +} + +func testCoreSystemBackendRaw(t *testing.T) (*Core, logical.Backend, string) { + c, _, root := TestCoreUnsealedRaw(t) + return c, testSystemBackendInternal(t, c), root +} + +func testSystemBackendInternal(t *testing.T, c *Core) logical.Backend { bc := &logical.BackendConfig{ Logger: c.logger, System: logical.StaticSystemView{ @@ -1621,7 +1622,8 @@ func testCoreSystemBackend(t *testing.T) (*Core, logical.Backend, string) { if err != nil { t.Fatal(err) } - return c, b, root + + return b } func TestSystemBackend_PluginCatalog_CRUD(t *testing.T) { diff --git a/vault/seal_testing.go b/vault/seal_testing.go index f74b140e0471..27271cf4aecb 100644 --- a/vault/seal_testing.go +++ b/vault/seal_testing.go @@ -107,7 +107,7 @@ func (d *TestSeal) SetRecoveryKey(key []byte) error { func testCoreUnsealedWithConfigs(t *testing.T, barrierConf, recoveryConf *SealConfig) (*Core, [][]byte, [][]byte, string) { seal := &TestSeal{} - core := TestCoreWithSeal(t, seal) + core := TestCoreWithSeal(t, seal, false) result, err := core.Initialize(&InitParams{ BarrierConfig: barrierConf, RecoveryConfig: recoveryConf, diff --git a/vault/testing.go b/vault/testing.go index 83e11d8f9506..d8078a11ecdb 100644 --- a/vault/testing.go +++ b/vault/testing.go @@ -85,18 +85,24 @@ oOyBJU/HMVvBfv4g+OVFLVgSwwm6owwsouZ0+D/LasbuHqYyqYqdyPJQYzWA2Y+F // TestCore returns a pure in-memory, uninitialized core for testing. func TestCore(t testing.T) *Core { - return TestCoreWithSeal(t, nil) + return TestCoreWithSeal(t, nil, false) +} + +// TestCoreRaw returns a pure in-memory, uninitialized core for testing. The raw +// storage endpoints are enabled with this core. +func TestCoreRaw(t testing.T) *Core { + return TestCoreWithSeal(t, nil, true) } // TestCoreNewSeal returns a pure in-memory, uninitialized core with // the new seal configuration. func TestCoreNewSeal(t testing.T) *Core { - return TestCoreWithSeal(t, &TestSeal{}) + return TestCoreWithSeal(t, &TestSeal{}, false) } // TestCoreWithSeal returns a pure in-memory, uninitialized core with the // specified seal for testing. -func TestCoreWithSeal(t testing.T, testSeal Seal) *Core { +func TestCoreWithSeal(t testing.T, testSeal Seal, enableRaw bool) *Core { logger := logformat.NewVaultLogger(log.LevelTrace) physicalBackend, err := physInmem.NewInmem(nil, logger) if err != nil { @@ -105,6 +111,10 @@ func TestCoreWithSeal(t testing.T, testSeal Seal) *Core { conf := testCoreConfig(t, physicalBackend, logger) + if enableRaw { + conf.EnableRaw = true + } + if testSeal != nil { conf.Seal = testSeal } @@ -198,6 +208,17 @@ func TestCoreUnseal(core *Core, key []byte) (bool, error) { // initialized and unsealed. func TestCoreUnsealed(t testing.T) (*Core, [][]byte, string) { core := TestCore(t) + return testCoreUnsealed(t, core) +} + +// TestCoreUnsealedRaw returns a pure in-memory core that is already +// initialized, unsealed, and with raw endpoints enabled. +func TestCoreUnsealedRaw(t testing.T) (*Core, [][]byte, string) { + core := TestCoreRaw(t) + return testCoreUnsealed(t, core) +} + +func testCoreUnsealed(t testing.T, core *Core) (*Core, [][]byte, string) { keys, token := TestCoreInit(t, core) for _, key := range keys { if _, err := TestCoreUnseal(core, TestKeyCopy(key)); err != nil { diff --git a/website/source/api/system/raw.html.md b/website/source/api/system/raw.html.md index 041c3589d5c5..7963dbb50e38 100644 --- a/website/source/api/system/raw.html.md +++ b/website/source/api/system/raw.html.md @@ -10,6 +10,10 @@ description: |- The `/sys/raw` endpoint is access the raw underlying store in Vault. +This endpont is off by default. See the +[Vault configuration documentation](/docs/configuration/index.html) to +enable. + ## Read Raw This endpoint reads the value of the key at the given path. This is the raw path @@ -76,6 +80,41 @@ $ curl \ https://vault.rocks/v1/sys/raw/secret/foo ``` +## List Raw + +This endpoint returns a list keys for a given path prefix. + +**This endpoint requires 'sudo' capability.** + +| Method | Path | Produces | +| :------- | :--------------------------- | :--------------------- | +| `LIST` | `/sys/raw/:prefix` | `200 application/json` | +| `GET` | `/sys/raw/:prefix?list=true` | `200 application/json` | + + +### Sample Request + +``` +$ curl \ + --header "X-Vault-Token: ..." \ + --request LIST \ + https://vault.rocks/v1/sys/raw/logical +``` + +### Sample Response + +```json +{ + "data":{ + "keys":[ + "abcd-1234...", + "efgh-1234...", + "ijkl-1234..." + ] + } +} +``` + ## Delete Raw This endpoint deletes the key with given path. This is the raw path in the diff --git a/website/source/docs/configuration/index.html.md b/website/source/docs/configuration/index.html.md index 6d7df9186aa8..a4da60ffcf35 100644 --- a/website/source/docs/configuration/index.html.md +++ b/website/source/docs/configuration/index.html.md @@ -100,6 +100,10 @@ to specify where the configuration is. duration for tokens and secrets. This is specified using a label suffix like `"30s"` or `"1h"`. +- `raw_storage_endpoint` `(bool: false)` – Enables the `sys/raw` endpoint which + allows the decryption/encryption of raw data into and out of the security + barrier. This is a highly priveleged endpoint. + - `ui` `(bool: false, Enterprise-only)` – Enables the built-in web UI, which is available on all listeners (address + port) at the `/ui` path. Browsers accessing the standard Vault API address will automatically redirect there. This can also