diff --git a/nomad/search_endpoint.go b/nomad/search_endpoint.go index bad903dc078..2f6fd293743 100644 --- a/nomad/search_endpoint.go +++ b/nomad/search_endpoint.go @@ -17,7 +17,9 @@ import ( const ( // truncateLimit is the maximum number of matches that will be returned for a - // prefix for a specific context + // prefix for a specific context. + // + // Does not apply to fuzzy searching. truncateLimit = 20 ) @@ -95,9 +97,13 @@ func (s *Search) getPrefixMatches(iter memdb.ResultIterator, prefix string) ([]s } func (s *Search) getFuzzyMatches(iter memdb.ResultIterator, text string) (map[structs.Context][]structs.FuzzyMatch, map[structs.Context]bool) { + fmt.Println("getFuzzyMatches enter, text:", text, "iter:", iter) + limitQuery := s.srv.config.SearchConfig.LimitQuery limitResults := s.srv.config.SearchConfig.LimitResults + fmt.Println("limitQuery:", limitQuery, "limitResults:", limitResults) + unsorted := make(map[structs.Context][]fuzzyMatch) truncations := make(map[structs.Context]bool) @@ -146,6 +152,7 @@ func (s *Search) getFuzzyMatches(iter memdb.ResultIterator, text string) (map[st for i := 0; i < limitQuery; i++ { raw := iter.Next() + fmt.Println(" i:", i, "raw==nil", raw == nil) if raw == nil { break } @@ -232,6 +239,8 @@ func (*Search) fuzzyMatchesJob(j *structs.Job, text string) map[structs.Context] ns := j.Namespace job := j.ID + fmt.Println("fuzzyMatchesJob, jobNS:", j.Namespace, "id:", j.ID, "text:", text) + // job.name if idx := strings.Index(j.Name, text); idx >= 0 { sm[structs.Jobs] = append(sm[structs.Jobs], score(job, ns, idx)) @@ -371,12 +380,15 @@ func getResourceIter(context structs.Context, aclObj *acl.ACL, namespace, prefix if aclObj == nil { return iter, nil } - return memdb.NewFilterIterator(iter, namespaceFilter(aclObj)), nil + return memdb.NewFilterIterator(iter, nsCapFilter(aclObj)), nil default: return getEnterpriseResourceIter(context, aclObj, namespace, prefix, ws, state) } } +// wildcard is a helper for determining if namespace is '*', used to determine +// if objects from every namespace should be considered when iterating, and that +// additional ACL checks will be necessary. func wildcard(namespace string) bool { return namespace == structs.AllNamespacesSentinel } @@ -386,53 +398,73 @@ func getFuzzyResourceIterator(context structs.Context, aclObj *acl.ACL, namespac case structs.Jobs: if wildcard(namespace) { iter, err := state.Jobs(ws) - return namespaceFilterIterator(iter, err, aclObj) + return nsCapIterFilter(iter, err, aclObj) } return state.JobsByNamespace(ws, namespace) case structs.Allocs: if wildcard(namespace) { iter, err := state.Allocs(ws) - return namespaceFilterIterator(iter, err, aclObj) + return nsCapIterFilter(iter, err, aclObj) } return state.AllocsByNamespace(ws, namespace) case structs.Nodes: + if wildcard(namespace) { + iter, err := state.Nodes(ws) + return nsCapIterFilter(iter, err, aclObj) + } return state.Nodes(ws) case structs.Plugins: + if wildcard(namespace) { + iter, err := state.CSIPlugins(ws) + return nsCapIterFilter(iter, err, aclObj) + } return state.CSIPlugins(ws) case structs.Namespaces: iter, err := state.Namespaces(ws) - return namespaceFilterIterator(iter, err, aclObj) + return nsCapIterFilter(iter, err, aclObj) default: return getEnterpriseFuzzyResourceIter(context, aclObj, namespace, ws, state) } } -func namespaceFilterIterator(iter memdb.ResultIterator, err error, aclObj *acl.ACL) (memdb.ResultIterator, error) { +// nsCapIterFilter wraps an iterator with a filter for removing items that the token +// does not have permission to read (whether missing the capability or in the +// wrong namespace). +func nsCapIterFilter(iter memdb.ResultIterator, err error, aclObj *acl.ACL) (memdb.ResultIterator, error) { if err != nil { return nil, err } if aclObj == nil { return iter, nil } - return memdb.NewFilterIterator(iter, namespaceFilter(aclObj)), nil + return memdb.NewFilterIterator(iter, nsCapFilter(aclObj)), nil } -// namespaceFilter wraps an iterator with a filter for removing items that belong -// to a namespace the ACL cannot access. -func namespaceFilter(aclObj *acl.ACL) memdb.FilterFunc { +// nsCapFilter produces a memdb.FilterFunc for removing objects not accessible +// by aclObj during a table scan. +func nsCapFilter(aclObj *acl.ACL) memdb.FilterFunc { return func(v interface{}) bool { switch t := v.(type) { case *structs.Job: - return !aclObj.AllowNamespace(t.Namespace) + return !aclObj.AllowNsOp(t.Namespace, acl.NamespaceCapabilityReadJob) + case *structs.Allocation: - return !aclObj.AllowNamespace(t.Namespace) + return !aclObj.AllowNsOp(t.Namespace, acl.NamespaceCapabilityReadJob) + case *structs.Namespace: return !aclObj.AllowNamespace(t.Name) + + case *structs.Node: + return !aclObj.AllowNodeRead() + + case *structs.CSIPlugin: + return !aclObj.AllowPluginRead() + default: return false } @@ -510,7 +542,7 @@ func (s *Search) PrefixSearch(args *structs.SearchRequest, reply *structs.Search run: func(ws memdb.WatchSet, state *state.StateStore) error { iters := make(map[structs.Context]memdb.ResultIterator) - contexts := expandSearchContexts(aclObj, namespace, args.Context) + contexts := filteredSearchContexts(aclObj, namespace, args.Context) for _, ctx := range contexts { iter, err := getResourceIter(ctx, aclObj, namespace, roundUUIDDownIfOdd(args.Prefix, args.Context), ws, state) @@ -592,7 +624,7 @@ func (s *Search) FuzzySearch(args *structs.FuzzySearchRequest, reply *structs.Fu namespace := args.RequestNamespace() context := args.Context - if !sufficientSearchPerms(aclObj, namespace, context) { + if !sufficientFuzzySearchPerms(aclObj, namespace, context) { return structs.ErrPermissionDenied } @@ -607,11 +639,17 @@ func (s *Search) FuzzySearch(args *structs.FuzzySearchRequest, reply *structs.Fu fuzzyIters := make(map[structs.Context]memdb.ResultIterator) prefixIters := make(map[structs.Context]memdb.ResultIterator) - contexts := expandSearchContexts(aclObj, namespace, context) - for _, ctx := range contexts { + prefixContexts := filteredSearchContexts(aclObj, namespace, context) + fuzzyContexts := filteredFuzzySearchContexts(aclObj, namespace, context) + + fmt.Println("prefixContexts:", prefixContexts) + fmt.Println("fuzzyContexts:", fuzzyContexts) + + // Gather the iterators used for prefix searching from those allowable contexts + for _, ctx := range prefixContexts { switch ctx { - // types that use UUID prefix searching + // only apply on the types that use UUID prefix searching case structs.Evals, structs.Deployments, structs.ScalingPolicies, structs.Volumes: iter, err := getResourceIter(ctx, aclObj, namespace, roundUUIDDownIfOdd(args.Prefix, args.Context), ws, state) if err != nil { @@ -621,8 +659,15 @@ func (s *Search) FuzzySearch(args *structs.FuzzySearchRequest, reply *structs.Fu } else { prefixIters[ctx] = iter } + } + } - // types that use fuzzy searching + // Gather the iterators used for fuzzy searching from those allowable contexts + for _, ctx := range fuzzyContexts { + switch ctx { + // skip the types that use UUID prefix searching + case structs.Evals, structs.Deployments, structs.ScalingPolicies, structs.Volumes: + continue default: iter, err := getFuzzyResourceIterator(ctx, aclObj, namespace, ws, state) if err != nil { @@ -632,6 +677,9 @@ func (s *Search) FuzzySearch(args *structs.FuzzySearchRequest, reply *structs.Fu } } + fmt.Println("prefixIters:", prefixIters) + fmt.Println("fuzzyIters:", fuzzyIters) + // Set prefix matches of the given text for ctx, iter := range prefixIters { res, isTrunc := s.getPrefixMatches(iter, args.Text) @@ -644,8 +692,10 @@ func (s *Search) FuzzySearch(args *structs.FuzzySearchRequest, reply *structs.Fu } // Set fuzzy matches of the given text - for _, iter := range fuzzyIters { + for kind, iter := range fuzzyIters { + fmt.Println("\n\nwill get fuzzy matches for kind:", kind, "text:", args.Text, "iter:", iter) matches, truncations := s.getFuzzyMatches(iter, args.Text) + fmt.Println(" -> matches:", matches, "truncations:", truncations) for ctx := range matches { reply.Matches[ctx] = matches[ctx] } @@ -657,7 +707,7 @@ func (s *Search) FuzzySearch(args *structs.FuzzySearchRequest, reply *structs.Fu // Set the index for the context. If the context has been specified, // it will be used as the index of the response. Otherwise, the maximum // index from all the resources will be used. - for _, ctx := range contexts { + for _, ctx := range fuzzyContexts { index, err := state.Index(contextToIndex(ctx)) if err != nil { return err @@ -675,6 +725,8 @@ func (s *Search) FuzzySearch(args *structs.FuzzySearchRequest, reply *structs.Fu return s.srv.blockingRPC(&opts) } +// expandContext returns either allContexts if context is 'all', or a one +// element slice with context by itself. func expandContext(context structs.Context) []structs.Context { switch context { case structs.All: @@ -685,3 +737,28 @@ func expandContext(context structs.Context) []structs.Context { return []structs.Context{context} } } + +// sufficientFuzzySearchPerms returns true if the searched namespace is the wildcard +// namespace, indicating we should bypass the preflight ACL checks otherwise performed +// by sufficientSearchPerms. This is to support fuzzy searching multiple namespaces +// with tokens that have permission for more than one namespace. The actual ACL +// validation will be performed while scanning objects instead, where we have finally +// have a concrete namespace to work with. +func sufficientFuzzySearchPerms(aclObj *acl.ACL, namespace string, context structs.Context) bool { + if wildcard(namespace) { + return true + } + return sufficientSearchPerms(aclObj, namespace, context) +} + +// filterFuzzySearchContexts returns every context asked for if the searched namespace +// is the wildcard namespace, indicating we should bypass ACL checks otherwise +// performed by filterSearchContexts. Instead we will rely on iterator filters to +// perform the ACL validation while scanning objects, where we have a concrete +// namespace to work with. +func filteredFuzzySearchContexts(aclObj *acl.ACL, namespace string, context structs.Context) []structs.Context { + if wildcard(namespace) { + return expandContext(context) + } + return filteredSearchContexts(aclObj, namespace, context) +} diff --git a/nomad/search_endpoint_oss.go b/nomad/search_endpoint_oss.go index 581191e2c8b..a724518c296 100644 --- a/nomad/search_endpoint_oss.go +++ b/nomad/search_endpoint_oss.go @@ -86,11 +86,11 @@ func sufficientSearchPerms(aclObj *acl.ACL, namespace string, context structs.Co return true } -// expandSearchContexts returns the expanded set of contexts of context, filtered down +// filteredSearchContexts returns the expanded set of contexts, filtered down // to the subset of contexts the aclObj is valid for. // // If aclObj is nil, no contexts are filtered out. -func expandSearchContexts(aclObj *acl.ACL, namespace string, context structs.Context) []structs.Context { +func filteredSearchContexts(aclObj *acl.ACL, namespace string, context structs.Context) []structs.Context { desired := expandContext(context) // If ACLs aren't enabled return all contexts diff --git a/nomad/search_endpoint_test.go b/nomad/search_endpoint_test.go index 58be6413a68..c0c1ccfd056 100644 --- a/nomad/search_endpoint_test.go +++ b/nomad/search_endpoint_test.go @@ -6,6 +6,7 @@ import ( "strings" "testing" + "github.com/hashicorp/go-hclog" msgpackrpc "github.com/hashicorp/net-rpc-msgpackrpc" "github.com/hashicorp/nomad/acl" "github.com/hashicorp/nomad/helper/uuid" @@ -1441,6 +1442,7 @@ func TestSearch_FuzzySearch_Namespace_ACL(t *testing.T) { s, root, cleanup := TestACLServer(t, func(c *Config) { c.NumSchedulers = 0 + c.Logger.SetLevel(hclog.Warn) }) defer cleanup() @@ -1449,6 +1451,7 @@ func TestSearch_FuzzySearch_Namespace_ACL(t *testing.T) { fsmState := s.fsm.State() ns := mock.Namespace() + ns.Name = "team-job-app" require.NoError(t, fsmState.UpsertNamespaces(500, []*structs.Namespace{ns})) job1 := mock.Job() @@ -1458,7 +1461,9 @@ func TestSearch_FuzzySearch_Namespace_ACL(t *testing.T) { job2.Namespace = ns.Name require.NoError(t, fsmState.UpsertJob(structs.MsgTypeTestSetup, 504, job2)) - require.NoError(t, fsmState.UpsertNode(structs.MsgTypeTestSetup, 1001, mock.Node())) + node := mock.Node() + node.Name = "run-jobs" + require.NoError(t, fsmState.UpsertNode(structs.MsgTypeTestSetup, 1001, node)) req := &structs.FuzzySearchRequest{ Text: "set-text-in-test", @@ -1499,7 +1504,7 @@ func TestSearch_FuzzySearch_Namespace_ACL(t *testing.T) { // Try with a node:read token and expect success due to All context { validToken := mock.CreatePolicyAndToken(t, fsmState, 1007, "test-valid", mock.NodePolicy(acl.PolicyRead)) - req.Text = "foo" + req.Text = "job" req.Context = structs.All req.AuthToken = validToken.SecretID var resp structs.FuzzySearchResponse @@ -1533,19 +1538,212 @@ func TestSearch_FuzzySearch_Namespace_ACL(t *testing.T) { // Try with a management token { + req.Text = "job" req.Context = structs.All req.AuthToken = root.SecretID - req.Namespace = structs.DefaultNamespace - var resp structs.SearchResponse - require.NoError(t, msgpackrpc.CallWithCodec(codec, "Search.PrefixSearch", req, &resp)) + req.Namespace = job1.Namespace + var resp structs.FuzzySearchResponse + require.NoError(t, msgpackrpc.CallWithCodec(codec, "Search.FuzzySearch", req, &resp)) require.Equal(t, uint64(1001), resp.Index) require.Len(t, resp.Matches[structs.Jobs], 1) - require.Equal(t, job1.ID, resp.Matches[structs.Jobs][0]) + require.Equal(t, job1.ID, resp.Matches[structs.Jobs][0].ID) require.Len(t, resp.Matches[structs.Nodes], 1) - require.Len(t, resp.Matches[structs.Namespaces], 2) + require.Len(t, resp.Matches[structs.Namespaces], 1) // matches "team-job-app" } } +func TestSearch_FuzzySearch_MultiNamespace_ACL(t *testing.T) { + t.Parallel() + + s, root, cleanupS := TestACLServer(t, func(c *Config) { + c.NumSchedulers = 0 + c.Logger.SetLevel(hclog.Warn) + }) + defer cleanupS() + + codec := rpcClient(t, s) + testutil.WaitForLeader(t, s.RPC) + fsmState := s.fsm.State() + + require.NoError(t, fsmState.UpsertNamespaces(500, []*structs.Namespace{{ + Name: "teamA", + Description: "first namespace", + CreateIndex: 100, + ModifyIndex: 200, + }, { + Name: "teamB", + Description: "second namespace", + CreateIndex: 101, + ModifyIndex: 201, + }, { + Name: "teamC", + Description: "third namespace", + CreateIndex: 102, + ModifyIndex: 202, + }})) + + // Upsert 3 jobs each in separate namespace + job1 := mock.Job() + job1.Name = "teamA-job1" + job1.ID = "job1" + job1.Namespace = "teamA" + require.NoError(t, fsmState.UpsertJob(structs.MsgTypeTestSetup, 502, job1)) + + job2 := mock.Job() + job2.Name = "teamB-job2" + job2.ID = "job2" + job2.Namespace = "teamB" + require.NoError(t, fsmState.UpsertJob(structs.MsgTypeTestSetup, 503, job2)) + + job3 := mock.Job() + job3.Name = "teamC-job3" + job3.ID = "job3" + job3.Namespace = "teamC" + require.NoError(t, fsmState.UpsertJob(structs.MsgTypeTestSetup, 504, job3)) + + // Upsert a node + node := mock.Node() + node.Name = "node-for-teams" + require.NoError(t, fsmState.UpsertNode(structs.MsgTypeTestSetup, 1001, node)) + + // Upsert a node that will not be matched + node2 := mock.Node() + node2.Name = "node-for-ops" + require.NoError(t, fsmState.UpsertNode(structs.MsgTypeTestSetup, 1002, node2)) + + // Create parameterized requests + request := func(text, namespace, token string, context structs.Context) *structs.FuzzySearchRequest { + return &structs.FuzzySearchRequest{ + Text: text, + Context: context, + QueryOptions: structs.QueryOptions{ + Region: "global", + Namespace: namespace, + AuthToken: token, + }, + } + } + + t.Run("without a token expect failure", func(t *testing.T) { + var resp structs.FuzzySearchResponse + req := request("anything", job1.Namespace, "", structs.Jobs) + err := msgpackrpc.CallWithCodec(codec, "Search.FuzzySearch", req, &resp) + require.EqualError(t, err, structs.ErrPermissionDenied.Error()) + }) + + t.Run("with an invalid token expect failure", func(t *testing.T) { + invalidToken := mock.CreatePolicyAndToken(t, fsmState, 1003, "test-invalid", + mock.NamespacePolicy(structs.DefaultNamespace, "", []string{acl.NamespaceCapabilityListJobs})) + req := request("anything", job1.Namespace, invalidToken.SecretID, structs.Jobs) + var resp structs.FuzzySearchResponse + err := msgpackrpc.CallWithCodec(codec, "Search.FuzzySearch", req, &resp) + require.EqualError(t, err, structs.ErrPermissionDenied.Error()) + }) + + t.Run("with node:read token search namespaces expect failure", func(t *testing.T) { + validToken := mock.CreatePolicyAndToken(t, fsmState, 1005, "test-invalid2", mock.NodePolicy(acl.PolicyRead)) + req := request("team", job1.Namespace, validToken.SecretID, structs.Namespaces) + var resp structs.FuzzySearchResponse + err := msgpackrpc.CallWithCodec(codec, "Search.FuzzySearch", req, &resp) + require.EqualError(t, err, structs.ErrPermissionDenied.Error()) + }) + + t.Run("with node:read token search all expect success", func(t *testing.T) { + validToken := mock.CreatePolicyAndToken(t, fsmState, 1007, "test-valid", mock.NodePolicy(acl.PolicyRead)) + req := request("team", job1.Namespace, validToken.SecretID, structs.All) + var resp structs.FuzzySearchResponse + require.NoError(t, msgpackrpc.CallWithCodec(codec, "Search.FuzzySearch", req, &resp)) + require.Equal(t, uint64(1002), resp.Index) // the latest nodes index + require.Len(t, resp.Matches[structs.Nodes], 1) + + // Jobs filtered out since token only has access to node:read + require.Len(t, resp.Matches[structs.Jobs], 0) + }) + + t.Run("with a teamB/job:read token search all expect 1 job", func(t *testing.T) { + token := mock.CreatePolicyAndToken(t, fsmState, 1009, "test-valid2", + mock.NamespacePolicy(job2.Namespace, "", []string{acl.NamespaceCapabilityReadJob})) + req := request("team", job2.Namespace, token.SecretID, structs.All) + var resp structs.FuzzySearchResponse + require.NoError(t, msgpackrpc.CallWithCodec(codec, "Search.FuzzySearch", req, &resp)) + require.Len(t, resp.Matches[structs.Jobs], 1) // YOU ARE HERE + require.Equal(t, job2.ID, resp.Matches[structs.Jobs][0].ID) + + // Index of job - not node - because node context is filtered out + require.Equal(t, uint64(504), resp.Index) + + // Nodes filtered out since token only has access to namespace:read-job + require.Len(t, resp.Matches[structs.Nodes], 0) + }) + + // Using a token that can read jobs in 2 namespaces, we should get job results from + // both those namespaces (using wildcard namespace in the query) but not the + // third (and from no other contexts). + t.Run("with a multi-ns job:read token search all expect 2 jobs", func(t *testing.T) { + policyB := mock.NamespacePolicy(job2.Namespace, "", []string{acl.NamespaceCapabilityReadJob}) + mock.CreatePolicy(t, fsmState, uint64(1009), "policyB", policyB) + + policyC := mock.NamespacePolicy(job3.Namespace, "", []string{acl.NamespaceCapabilityReadJob}) + mock.CreatePolicy(t, fsmState, uint64(1010), "policyC", policyC) + + token := mock.CreateToken(t, fsmState, uint64(1011), []string{"policyB", "policyC"}) + req := request("team", structs.AllNamespacesSentinel, token.SecretID, structs.Jobs) + var resp structs.FuzzySearchResponse + require.NoError(t, msgpackrpc.CallWithCodec(codec, "Search.FuzzySearch", req, &resp)) + require.Len(t, resp.Matches[structs.Jobs], 2) + require.Equal(t, job2.ID, resp.Matches[structs.Jobs][0].ID) + require.Equal(t, job3.ID, resp.Matches[structs.Jobs][1].ID) + + // Index of job - not node - because node context is filtered out + require.Equal(t, uint64(504), resp.Index) + }) + + // Using a management token, we should get job results from all three namespaces + // (using wildcard namespace in the query). + t.Run("with a management token search all expect 3 jobs", func(t *testing.T) { + req := request("team", structs.AllNamespacesSentinel, root.SecretID, structs.Jobs) + var resp structs.FuzzySearchResponse + require.NoError(t, msgpackrpc.CallWithCodec(codec, "Search.FuzzySearch", req, &resp)) + require.Len(t, resp.Matches[structs.Jobs], 3) + require.Equal(t, job1.ID, resp.Matches[structs.Jobs][0].ID) + require.Equal(t, job2.ID, resp.Matches[structs.Jobs][1].ID) + require.Equal(t, job3.ID, resp.Matches[structs.Jobs][2].ID) + + // Index of job - not node - because node context is filtered out + require.Equal(t, uint64(504), resp.Index) + }) + + // Using a token that can read nodes, we should get our 1 matching node when + // searching the nodes context. + t.Run("with node:read token read nodes", func(t *testing.T) { + policy := mock.NodePolicy("read") + mock.CreatePolicy(t, fsmState, uint64(1012), "node-read-policy", policy) + + token := mock.CreateToken(t, fsmState, uint64(1013), []string{"node-read-policy"}) + req := request("team", structs.AllNamespacesSentinel, token.SecretID, structs.Nodes) + var resp structs.FuzzySearchResponse + require.NoError(t, msgpackrpc.CallWithCodec(codec, "Search.FuzzySearch", req, &resp)) + require.Len(t, resp.Matches[structs.Nodes], 1) + require.Equal(t, "node-for-teams", resp.Matches[structs.Nodes][0].ID) + }) + + // Using a token that cannot read nodes, we should get no matching nodes when + // searching the nodes context. + t.Run("with a job:read token read nodes", func(t *testing.T) { + policy := mock.AgentPolicy("read") + mock.CreatePolicy(t, fsmState, uint64(1014), "agent-read-policy", policy) + + token := mock.CreateToken(t, fsmState, uint64(1015), []string{"agent-read-policy"}) + req := request("team", structs.AllNamespacesSentinel, token.SecretID, structs.Nodes) + var resp structs.FuzzySearchResponse + require.NoError(t, msgpackrpc.CallWithCodec(codec, "Search.FuzzySearch", req, &resp)) + require.Empty(t, resp.Matches[structs.Nodes]) + }) + + // todo YOU ARE HERE, add tests for everything in nsCapFilter + +} + func TestSearch_FuzzySearch_Job(t *testing.T) { t.Parallel()