diff --git a/nomad/deployment_endpoint.go b/nomad/deployment_endpoint.go index 70f685d4a28..84fcb42467b 100644 --- a/nomad/deployment_endpoint.go +++ b/nomad/deployment_endpoint.go @@ -10,33 +10,10 @@ import ( memdb "github.com/hashicorp/go-memdb" "github.com/hashicorp/nomad/acl" "github.com/hashicorp/nomad/nomad/state" + "github.com/hashicorp/nomad/nomad/state/paginator" "github.com/hashicorp/nomad/nomad/structs" ) -// DeploymentPaginationIterator is a wrapper over a go-memdb iterator that -// implements the paginator Iterator interface. -type DeploymentPaginationIterator struct { - iter memdb.ResultIterator - byCreateIndex bool -} - -func (it DeploymentPaginationIterator) Next() (string, interface{}) { - raw := it.iter.Next() - if raw == nil { - return "", nil - } - - d := raw.(*structs.Deployment) - token := d.ID - - // prefix the pagination token by CreateIndex to keep it properly sorted. - if it.byCreateIndex { - token = fmt.Sprintf("%v-%v", d.CreateIndex, d.ID) - } - - return token, d -} - // Deployment endpoint is used for manipulating deployments type Deployment struct { srv *Server @@ -433,26 +410,34 @@ func (d *Deployment) List(args *structs.DeploymentListRequest, reply *structs.De // Capture all the deployments var err error var iter memdb.ResultIterator - var deploymentIter DeploymentPaginationIterator + var opts paginator.StructsTokenizerOptions if prefix := args.QueryOptions.Prefix; prefix != "" { iter, err = store.DeploymentsByIDPrefix(ws, namespace, prefix) - deploymentIter.byCreateIndex = false + opts = paginator.StructsTokenizerOptions{ + WithID: true, + } } else if namespace != structs.AllNamespacesSentinel { iter, err = store.DeploymentsByNamespaceOrdered(ws, namespace, args.Ascending) - deploymentIter.byCreateIndex = true + opts = paginator.StructsTokenizerOptions{ + WithCreateIndex: true, + WithID: true, + } } else { iter, err = store.Deployments(ws, args.Ascending) - deploymentIter.byCreateIndex = true + opts = paginator.StructsTokenizerOptions{ + WithCreateIndex: true, + WithID: true, + } } if err != nil { return err } - deploymentIter.iter = iter + tokenizer := paginator.NewStructsTokenizer(iter, opts) var deploys []*structs.Deployment - paginator, err := state.NewPaginator(deploymentIter, args.QueryOptions, + paginator, err := paginator.NewPaginator(iter, tokenizer, nil, args.QueryOptions, func(raw interface{}) error { deploy := raw.(*structs.Deployment) deploys = append(deploys, deploy) diff --git a/nomad/deployment_endpoint_test.go b/nomad/deployment_endpoint_test.go index e91bc28a8a7..110471d06e2 100644 --- a/nomad/deployment_endpoint_test.go +++ b/nomad/deployment_endpoint_test.go @@ -1312,7 +1312,7 @@ func TestDeploymentEndpoint_List_Pagination(t *testing.T) { { name: "test01 size-2 page-1 default NS", pageSize: 2, - expectedNextToken: "1003-aaaaaaaa-3350-4b4b-d185-0e1992ed43e9", + expectedNextToken: "1003.aaaaaaaa-3350-4b4b-d185-0e1992ed43e9", expectedIDs: []string{ "aaaa1111-3350-4b4b-d185-0e1992ed43e9", "aaaaaa22-3350-4b4b-d185-0e1992ed43e9", @@ -1331,8 +1331,8 @@ func TestDeploymentEndpoint_List_Pagination(t *testing.T) { { name: "test03 size-2 page-2 default NS", pageSize: 2, - nextToken: "1003-aaaaaaaa-3350-4b4b-d185-0e1992ed43e9", - expectedNextToken: "1005-aaaaaacc-3350-4b4b-d185-0e1992ed43e9", + nextToken: "1003.aaaaaaaa-3350-4b4b-d185-0e1992ed43e9", + expectedNextToken: "1005.aaaaaacc-3350-4b4b-d185-0e1992ed43e9", expectedIDs: []string{ "aaaaaaaa-3350-4b4b-d185-0e1992ed43e9", "aaaaaabb-3350-4b4b-d185-0e1992ed43e9", @@ -1353,8 +1353,8 @@ func TestDeploymentEndpoint_List_Pagination(t *testing.T) { name: "test05 size-2 page-2 all namespaces", namespace: "*", pageSize: 2, - nextToken: "1002-aaaaaa33-3350-4b4b-d185-0e1992ed43e9", - expectedNextToken: "1004-aaaaaabb-3350-4b4b-d185-0e1992ed43e9", + nextToken: "1002.aaaaaa33-3350-4b4b-d185-0e1992ed43e9", + expectedNextToken: "1004.aaaaaabb-3350-4b4b-d185-0e1992ed43e9", expectedIDs: []string{ "aaaaaa33-3350-4b4b-d185-0e1992ed43e9", "aaaaaaaa-3350-4b4b-d185-0e1992ed43e9", @@ -1382,7 +1382,7 @@ func TestDeploymentEndpoint_List_Pagination(t *testing.T) { namespace: "*", filter: `ID matches "^a+[123]"`, pageSize: 2, - expectedNextToken: "1002-aaaaaa33-3350-4b4b-d185-0e1992ed43e9", + expectedNextToken: "1002.aaaaaa33-3350-4b4b-d185-0e1992ed43e9", expectedIDs: []string{ "aaaa1111-3350-4b4b-d185-0e1992ed43e9", "aaaaaa22-3350-4b4b-d185-0e1992ed43e9", @@ -1415,8 +1415,8 @@ func TestDeploymentEndpoint_List_Pagination(t *testing.T) { { name: "test13 non-lexicographic order", pageSize: 1, - nextToken: "1007-00000111-3350-4b4b-d185-0e1992ed43e9", - expectedNextToken: "1009-bbbb1111-3350-4b4b-d185-0e1992ed43e9", + nextToken: "1007.00000111-3350-4b4b-d185-0e1992ed43e9", + expectedNextToken: "1009.bbbb1111-3350-4b4b-d185-0e1992ed43e9", expectedIDs: []string{ "00000111-3350-4b4b-d185-0e1992ed43e9", }, @@ -1424,7 +1424,7 @@ func TestDeploymentEndpoint_List_Pagination(t *testing.T) { { name: "test14 missing index", pageSize: 1, - nextToken: "1008-e9522802-0cd8-4b1d-9c9e-ab3d97938371", + nextToken: "1008.e9522802-0cd8-4b1d-9c9e-ab3d97938371", expectedIDs: []string{ "bbbb1111-3350-4b4b-d185-0e1992ed43e9", }, diff --git a/nomad/eval_endpoint.go b/nomad/eval_endpoint.go index 0b6b26f598a..1c52130c699 100644 --- a/nomad/eval_endpoint.go +++ b/nomad/eval_endpoint.go @@ -12,6 +12,7 @@ import ( "github.com/hashicorp/nomad/acl" "github.com/hashicorp/nomad/nomad/state" + "github.com/hashicorp/nomad/nomad/state/paginator" "github.com/hashicorp/nomad/nomad/structs" "github.com/hashicorp/nomad/scheduler" ) @@ -21,30 +22,6 @@ const ( DefaultDequeueTimeout = time.Second ) -// EvalPaginationIterator is a wrapper over a go-memdb iterator that implements -// the paginator Iterator interface. -type EvalPaginationIterator struct { - iter memdb.ResultIterator - byCreateIndex bool -} - -func (it EvalPaginationIterator) Next() (string, interface{}) { - raw := it.iter.Next() - if raw == nil { - return "", nil - } - - eval := raw.(*structs.Evaluation) - token := eval.ID - - // prefix the pagination token by CreateIndex to keep it properly sorted. - if it.byCreateIndex { - token = fmt.Sprintf("%v-%v", eval.CreateIndex, eval.ID) - } - - return token, eval -} - // Eval endpoint is used for eval interactions type Eval struct { srv *Server @@ -438,17 +415,25 @@ func (e *Eval) List(args *structs.EvalListRequest, reply *structs.EvalListRespon // Scan all the evaluations var err error var iter memdb.ResultIterator - var evalIter EvalPaginationIterator + var opts paginator.StructsTokenizerOptions if prefix := args.QueryOptions.Prefix; prefix != "" { iter, err = store.EvalsByIDPrefix(ws, namespace, prefix) - evalIter.byCreateIndex = false + opts = paginator.StructsTokenizerOptions{ + WithID: true, + } } else if namespace != structs.AllNamespacesSentinel { iter, err = store.EvalsByNamespaceOrdered(ws, namespace, args.Ascending) - evalIter.byCreateIndex = true + opts = paginator.StructsTokenizerOptions{ + WithCreateIndex: true, + WithID: true, + } } else { iter, err = store.Evals(ws, args.Ascending) - evalIter.byCreateIndex = true + opts = paginator.StructsTokenizerOptions{ + WithCreateIndex: true, + WithID: true, + } } if err != nil { return err @@ -460,10 +445,11 @@ func (e *Eval) List(args *structs.EvalListRequest, reply *structs.EvalListRespon } return false }) - evalIter.iter = iter + + tokenizer := paginator.NewStructsTokenizer(iter, opts) var evals []*structs.Evaluation - paginator, err := state.NewPaginator(evalIter, args.QueryOptions, + paginator, err := paginator.NewPaginator(iter, tokenizer, nil, args.QueryOptions, func(raw interface{}) error { eval := raw.(*structs.Evaluation) evals = append(evals, eval) diff --git a/nomad/eval_endpoint_test.go b/nomad/eval_endpoint_test.go index 92463394e7e..12dc256e3f7 100644 --- a/nomad/eval_endpoint_test.go +++ b/nomad/eval_endpoint_test.go @@ -1084,7 +1084,7 @@ func TestEvalEndpoint_List_PaginationFiltering(t *testing.T) { "aaaa1111-3350-4b4b-d185-0e1992ed43e9", "aaaaaa22-3350-4b4b-d185-0e1992ed43e9", }, - expectedNextToken: "1003-aaaaaaaa-3350-4b4b-d185-0e1992ed43e9", // next one in default namespace + expectedNextToken: "1003.aaaaaaaa-3350-4b4b-d185-0e1992ed43e9", // next one in default namespace }, { name: "test02 size-2 page-1 default NS with prefix", @@ -1099,8 +1099,8 @@ func TestEvalEndpoint_List_PaginationFiltering(t *testing.T) { { name: "test03 size-2 page-2 default NS", pageSize: 2, - nextToken: "1003-aaaaaaaa-3350-4b4b-d185-0e1992ed43e9", - expectedNextToken: "1005-aaaaaacc-3350-4b4b-d185-0e1992ed43e9", + nextToken: "1003.aaaaaaaa-3350-4b4b-d185-0e1992ed43e9", + expectedNextToken: "1005.aaaaaacc-3350-4b4b-d185-0e1992ed43e9", expectedIDs: []string{ "aaaaaaaa-3350-4b4b-d185-0e1992ed43e9", "aaaaaabb-3350-4b4b-d185-0e1992ed43e9", @@ -1123,7 +1123,7 @@ func TestEvalEndpoint_List_PaginationFiltering(t *testing.T) { filterJobID: "example", filterStatus: "pending", // aaaaaaaa, bb, and cc are filtered by status - expectedNextToken: "1006-aaaaaadd-3350-4b4b-d185-0e1992ed43e9", + expectedNextToken: "1006.aaaaaadd-3350-4b4b-d185-0e1992ed43e9", expectedIDs: []string{ "aaaa1111-3350-4b4b-d185-0e1992ed43e9", "aaaaaa22-3350-4b4b-d185-0e1992ed43e9", @@ -1159,7 +1159,7 @@ func TestEvalEndpoint_List_PaginationFiltering(t *testing.T) { pageSize: 3, // reads off the end filterJobID: "example", filterStatus: "pending", - nextToken: "1003-aaaaaaaa-3350-4b4b-d185-0e1992ed43e9", + nextToken: "1003.aaaaaaaa-3350-4b4b-d185-0e1992ed43e9", expectedNextToken: "", expectedIDs: []string{ "aaaaaadd-3350-4b4b-d185-0e1992ed43e9", @@ -1183,8 +1183,8 @@ func TestEvalEndpoint_List_PaginationFiltering(t *testing.T) { name: "test10 size-2 page-2 all namespaces", namespace: "*", pageSize: 2, - nextToken: "1002-aaaaaa33-3350-4b4b-d185-0e1992ed43e9", - expectedNextToken: "1004-aaaaaabb-3350-4b4b-d185-0e1992ed43e9", + nextToken: "1002.aaaaaa33-3350-4b4b-d185-0e1992ed43e9", + expectedNextToken: "1004.aaaaaabb-3350-4b4b-d185-0e1992ed43e9", expectedIDs: []string{ "aaaaaa33-3350-4b4b-d185-0e1992ed43e9", "aaaaaaaa-3350-4b4b-d185-0e1992ed43e9", @@ -1228,7 +1228,7 @@ func TestEvalEndpoint_List_PaginationFiltering(t *testing.T) { name: "test16 go-bexpr filter with pagination", filter: `JobID == "example"`, pageSize: 2, - expectedNextToken: "1003-aaaaaaaa-3350-4b4b-d185-0e1992ed43e9", + expectedNextToken: "1003.aaaaaaaa-3350-4b4b-d185-0e1992ed43e9", expectedIDs: []string{ "aaaa1111-3350-4b4b-d185-0e1992ed43e9", "aaaaaa22-3350-4b4b-d185-0e1992ed43e9", @@ -1267,8 +1267,8 @@ func TestEvalEndpoint_List_PaginationFiltering(t *testing.T) { { name: "test22 non-lexicographic order", pageSize: 1, - nextToken: "1009-00000111-3350-4b4b-d185-0e1992ed43e9", - expectedNextToken: "1010-00000222-3350-4b4b-d185-0e1992ed43e9", + nextToken: "1009.00000111-3350-4b4b-d185-0e1992ed43e9", + expectedNextToken: "1010.00000222-3350-4b4b-d185-0e1992ed43e9", expectedIDs: []string{ "00000111-3350-4b4b-d185-0e1992ed43e9", }, @@ -1276,8 +1276,8 @@ func TestEvalEndpoint_List_PaginationFiltering(t *testing.T) { { name: "test23 same index", pageSize: 1, - nextToken: "1010-00000222-3350-4b4b-d185-0e1992ed43e9", - expectedNextToken: "1010-00000333-3350-4b4b-d185-0e1992ed43e9", + nextToken: "1010.00000222-3350-4b4b-d185-0e1992ed43e9", + expectedNextToken: "1010.00000333-3350-4b4b-d185-0e1992ed43e9", expectedIDs: []string{ "00000222-3350-4b4b-d185-0e1992ed43e9", }, @@ -1285,7 +1285,7 @@ func TestEvalEndpoint_List_PaginationFiltering(t *testing.T) { { name: "test24 missing index", pageSize: 1, - nextToken: "1011-e9522802-0cd8-4b1d-9c9e-ab3d97938371", + nextToken: "1011.e9522802-0cd8-4b1d-9c9e-ab3d97938371", expectedIDs: []string{ "bbbb1111-3350-4b4b-d185-0e1992ed43e9", }, diff --git a/nomad/state/paginator/filter.go b/nomad/state/paginator/filter.go new file mode 100644 index 00000000000..e4ed1f250f0 --- /dev/null +++ b/nomad/state/paginator/filter.go @@ -0,0 +1,41 @@ +package paginator + +// Filter is the interface that must be implemented to skip values when using +// the Paginator. +type Filter interface { + // Evaluate returns true if the element should be added to the page. + Evaluate(interface{}) (bool, error) +} + +// GenericFilter wraps a function that can be used to provide simple or in +// scope filtering. +type GenericFilter struct { + Allow func(interface{}) (bool, error) +} + +func (f GenericFilter) Evaluate(raw interface{}) (bool, error) { + return f.Allow(raw) +} + +// NamespaceFilter skips elements with a namespace value that is not in the +// allowable set. +type NamespaceFilter struct { + AllowableNamespaces map[string]bool +} + +func (f NamespaceFilter) Evaluate(raw interface{}) (bool, error) { + if raw == nil { + return false, nil + } + + item, _ := raw.(NamespaceGetter) + namespace := item.GetNamespace() + + if f.AllowableNamespaces == nil { + return true, nil + } + if f.AllowableNamespaces[namespace] { + return true, nil + } + return false, nil +} diff --git a/nomad/state/filter_test.go b/nomad/state/paginator/filter_test.go similarity index 61% rename from nomad/state/filter_test.go rename to nomad/state/paginator/filter_test.go index 2fa1b02ad3e..5ce3ebb3e1a 100644 --- a/nomad/state/filter_test.go +++ b/nomad/state/paginator/filter_test.go @@ -1,15 +1,111 @@ -package state +package paginator import ( "testing" "time" "github.com/hashicorp/go-bexpr" - memdb "github.com/hashicorp/go-memdb" "github.com/hashicorp/nomad/helper/uuid" + "github.com/hashicorp/nomad/nomad/state" "github.com/hashicorp/nomad/nomad/structs" + "github.com/stretchr/testify/require" ) +func TestGenericFilter(t *testing.T) { + t.Parallel() + ids := []string{"0", "1", "2", "3", "4", "5", "6", "7", "8", "9"} + + filters := []Filter{GenericFilter{ + Allow: func(raw interface{}) (bool, error) { + result := raw.(*mockObject) + return result.id > "5", nil + }, + }} + iter := newTestIterator(ids) + tokenizer := testTokenizer{} + opts := structs.QueryOptions{ + PerPage: 3, + Ascending: true, + } + results := []string{} + paginator, err := NewPaginator(iter, tokenizer, filters, opts, + func(raw interface{}) error { + result := raw.(*mockObject) + results = append(results, result.id) + return nil + }, + ) + require.NoError(t, err) + + nextToken, err := paginator.Page() + require.NoError(t, err) + + expected := []string{"6", "7", "8"} + require.Equal(t, "9", nextToken) + require.Equal(t, expected, results) +} + +func TestNamespaceFilter(t *testing.T) { + t.Parallel() + + mocks := []*mockObject{ + {namespace: "default"}, + {namespace: "dev"}, + {namespace: "qa"}, + {namespace: "region-1"}, + } + + cases := []struct { + name string + allowable map[string]bool + expected []string + }{ + { + name: "nil map", + expected: []string{"default", "dev", "qa", "region-1"}, + }, + { + name: "allow default", + allowable: map[string]bool{"default": true}, + expected: []string{"default"}, + }, + { + name: "allow multiple", + allowable: map[string]bool{"default": true, "dev": false, "qa": true}, + expected: []string{"default", "qa"}, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + filters := []Filter{NamespaceFilter{ + AllowableNamespaces: tc.allowable, + }} + iter := newTestIteratorWithMocks(mocks) + tokenizer := testTokenizer{} + opts := structs.QueryOptions{ + PerPage: int32(len(mocks)), + Ascending: true, + } + + results := []string{} + paginator, err := NewPaginator(iter, tokenizer, filters, opts, + func(raw interface{}) error { + result := raw.(*mockObject) + results = append(results, result.namespace) + return nil + }, + ) + require.NoError(t, err) + + nextToken, err := paginator.Page() + require.NoError(t, err) + require.Equal(t, "", nextToken) + require.Equal(t, tc.expected, results) + }) + } +} + func BenchmarkEvalListFilter(b *testing.B) { const evalCount = 100_000 @@ -76,9 +172,10 @@ func BenchmarkEvalListFilter(b *testing.B) { for i := 0; i < b.N; i++ { iter, _ := state.EvalsByNamespace(nil, structs.DefaultNamespace) - evalIter := evalPaginationIterator{iter} + tokenizer := NewStructsTokenizer(iter, StructsTokenizerOptions{WithID: true}) + var evals []*structs.Evaluation - paginator, err := NewPaginator(evalIter, opts, func(raw interface{}) error { + paginator, err := NewPaginator(iter, tokenizer, nil, opts, func(raw interface{}) error { eval := raw.(*structs.Evaluation) evals = append(evals, eval) return nil @@ -100,9 +197,10 @@ func BenchmarkEvalListFilter(b *testing.B) { for i := 0; i < b.N; i++ { iter, _ := state.Evals(nil, false) - evalIter := evalPaginationIterator{iter} + tokenizer := NewStructsTokenizer(iter, StructsTokenizerOptions{WithID: true}) + var evals []*structs.Evaluation - paginator, err := NewPaginator(evalIter, opts, func(raw interface{}) error { + paginator, err := NewPaginator(iter, tokenizer, nil, opts, func(raw interface{}) error { eval := raw.(*structs.Evaluation) evals = append(evals, eval) return nil @@ -137,9 +235,10 @@ func BenchmarkEvalListFilter(b *testing.B) { for i := 0; i < b.N; i++ { iter, _ := state.EvalsByNamespace(nil, structs.DefaultNamespace) - evalIter := evalPaginationIterator{iter} + tokenizer := NewStructsTokenizer(iter, StructsTokenizerOptions{WithID: true}) + var evals []*structs.Evaluation - paginator, err := NewPaginator(evalIter, opts, func(raw interface{}) error { + paginator, err := NewPaginator(iter, tokenizer, nil, opts, func(raw interface{}) error { eval := raw.(*structs.Evaluation) evals = append(evals, eval) return nil @@ -175,9 +274,10 @@ func BenchmarkEvalListFilter(b *testing.B) { for i := 0; i < b.N; i++ { iter, _ := state.Evals(nil, false) - evalIter := evalPaginationIterator{iter} + tokenizer := NewStructsTokenizer(iter, StructsTokenizerOptions{WithID: true}) + var evals []*structs.Evaluation - paginator, err := NewPaginator(evalIter, opts, func(raw interface{}) error { + paginator, err := NewPaginator(iter, tokenizer, nil, opts, func(raw interface{}) error { eval := raw.(*structs.Evaluation) evals = append(evals, eval) return nil @@ -193,12 +293,12 @@ func BenchmarkEvalListFilter(b *testing.B) { // ----------------- // BENCHMARK HELPER FUNCTIONS -func setupPopulatedState(b *testing.B, evalCount int) *StateStore { +func setupPopulatedState(b *testing.B, evalCount int) *state.StateStore { evals := generateEvals(evalCount) index := uint64(0) var err error - state := TestStateStore(b) + state := state.TestStateStore(b) for _, eval := range evals { index++ err = state.UpsertEvals( @@ -235,17 +335,3 @@ func generateEval(i int, ns string) *structs.Evaluation { ModifyTime: now, } } - -type evalPaginationIterator struct { - iter memdb.ResultIterator -} - -func (it evalPaginationIterator) Next() (string, interface{}) { - raw := it.iter.Next() - if raw == nil { - return "", nil - } - - eval := raw.(*structs.Evaluation) - return eval.ID, eval -} diff --git a/nomad/state/paginator.go b/nomad/state/paginator/paginator.go similarity index 65% rename from nomad/state/paginator.go rename to nomad/state/paginator/paginator.go index 607ff8cde07..2d3f2370f8d 100644 --- a/nomad/state/paginator.go +++ b/nomad/state/paginator/paginator.go @@ -1,4 +1,4 @@ -package state +package paginator import ( "fmt" @@ -7,20 +7,19 @@ import ( "github.com/hashicorp/nomad/nomad/structs" ) -// Iterator is the interface that must be implemented to use the Paginator. +// Iterator is the interface that must be implemented to suply data to the +// Paginator. type Iterator interface { - // Next returns the next element to be considered for pagination along with - // a token string used to uniquely identify elements in the iteration. + // Next returns the next element to be considered for pagination. // The page will end if nil is returned. - // Tokens should have a stable order and the order must match the paginator - // ascending property. - Next() (string, interface{}) + Next() interface{} } -// Paginator is an iterator over a memdb.ResultIterator that returns -// only the expected number of pages. +// Paginator wraps an iterator and returns only the expected number of pages. type Paginator struct { iter Iterator + tokenizer Tokenizer + filters []Filter perPage int32 itemCount int32 seekingToken string @@ -29,17 +28,16 @@ type Paginator struct { nextTokenFound bool pageErr error - // filterEvaluator is used to filter results using go-bexpr. It's nil if - // no filter expression is defined. - filterEvaluator *bexpr.Evaluator - // appendFunc is the function the caller should use to append raw // entries to the results set. The object is guaranteed to be // non-nil. appendFunc func(interface{}) error } -func NewPaginator(iter Iterator, opts structs.QueryOptions, appendFunc func(interface{}) error) (*Paginator, error) { +// NewPaginator returns a new Paginator. +func NewPaginator(iter Iterator, tokenizer Tokenizer, filters []Filter, + opts structs.QueryOptions, appendFunc func(interface{}) error) (*Paginator, error) { + var evaluator *bexpr.Evaluator var err error @@ -48,21 +46,23 @@ func NewPaginator(iter Iterator, opts structs.QueryOptions, appendFunc func(inte if err != nil { return nil, fmt.Errorf("failed to read filter expression: %v", err) } + filters = append(filters, evaluator) } return &Paginator{ - iter: iter, - perPage: opts.PerPage, - seekingToken: opts.NextToken, - ascending: opts.Ascending, - nextTokenFound: opts.NextToken == "", - filterEvaluator: evaluator, - appendFunc: appendFunc, + iter: iter, + tokenizer: tokenizer, + filters: filters, + perPage: opts.PerPage, + seekingToken: opts.NextToken, + ascending: opts.Ascending, + nextTokenFound: opts.NextToken == "", + appendFunc: appendFunc, }, nil } // Page populates a page by running the append function -// over all results. Returns the next token +// over all results. Returns the next token. func (p *Paginator) Page() (string, error) { DONE: for { @@ -84,11 +84,12 @@ DONE: } func (p *Paginator) next() (interface{}, paginatorState) { - token, raw := p.iter.Next() + raw := p.iter.Next() if raw == nil { p.nextToken = "" return nil, paginatorComplete } + token := p.tokenizer.GetToken(raw) // have we found the token we're seeking (if any)? p.nextToken = token @@ -104,14 +105,14 @@ func (p *Paginator) next() (interface{}, paginatorState) { return nil, paginatorSkip } - // apply filter if defined - if p.filterEvaluator != nil { - match, err := p.filterEvaluator.Evaluate(raw) + // apply filters if defined + for _, f := range p.filters { + allow, err := f.Evaluate(raw) if err != nil { p.pageErr = err return nil, paginatorComplete } - if !match { + if !allow { return nil, paginatorSkip } } diff --git a/nomad/state/paginator_test.go b/nomad/state/paginator/paginator_test.go similarity index 74% rename from nomad/state/paginator_test.go rename to nomad/state/paginator/paginator_test.go index 0d6f07fdac9..d5b164b2389 100644 --- a/nomad/state/paginator_test.go +++ b/nomad/state/paginator/paginator_test.go @@ -1,4 +1,4 @@ -package state +package paginator import ( "errors" @@ -58,14 +58,15 @@ func TestPaginator(t *testing.T) { t.Run(tc.name, func(t *testing.T) { iter := newTestIterator(ids) - results := []string{} + tokenizer := testTokenizer{} + opts := structs.QueryOptions{ + PerPage: tc.perPage, + NextToken: tc.nextToken, + Ascending: true, + } - paginator, err := NewPaginator(iter, - structs.QueryOptions{ - PerPage: tc.perPage, - NextToken: tc.nextToken, - Ascending: true, - }, + results := []string{} + paginator, err := NewPaginator(iter, tokenizer, nil, opts, func(raw interface{}) error { if tc.expectedError != "" { return errors.New(tc.expectedError) @@ -94,27 +95,32 @@ func TestPaginator(t *testing.T) { // helpers for pagination tests -// implements memdb.ResultIterator interface +// implements Iterator interface type testResultIterator struct { results chan interface{} } -func (i testResultIterator) Next() (string, interface{}) { +func (i testResultIterator) Next() interface{} { select { case raw := <-i.results: if raw == nil { - return "", nil + return nil } m := raw.(*mockObject) - return m.id, m + return m default: - return "", nil + return nil } } type mockObject struct { - id string + id string + namespace string +} + +func (m *mockObject) GetNamespace() string { + return m.namespace } func newTestIterator(ids []string) testResultIterator { @@ -124,3 +130,18 @@ func newTestIterator(ids []string) testResultIterator { } return iter } + +func newTestIteratorWithMocks(mocks []*mockObject) testResultIterator { + iter := testResultIterator{results: make(chan interface{}, 20)} + for _, m := range mocks { + iter.results <- m + } + return iter +} + +// implements Tokenizer interface +type testTokenizer struct{} + +func (t testTokenizer) GetToken(raw interface{}) string { + return raw.(*mockObject).id +} diff --git a/nomad/state/paginator/tokenizer.go b/nomad/state/paginator/tokenizer.go new file mode 100644 index 00000000000..a2109cf953f --- /dev/null +++ b/nomad/state/paginator/tokenizer.go @@ -0,0 +1,84 @@ +package paginator + +import ( + "fmt" + "strings" +) + +// Tokenizer is the interface that must be implemented to provide pagination +// tokens to the Paginator. +type Tokenizer interface { + // GetToken returns the pagination token for the given element. + GetToken(interface{}) string +} + +// IDGetter is the interface that must be implemented by structs that need to +// have their ID as part of the pagination token. +type IDGetter interface { + GetID() string +} + +// NamespaceGetter is the interface that must be implemented by structs that +// need to have their Namespace as part of the pagination token. +type NamespaceGetter interface { + GetNamespace() string +} + +// CreateIndexGetter is the interface that must be implemented by structs that +// need to have their CreateIndex as part of the pagination token. +type CreateIndexGetter interface { + GetCreateIndex() uint64 +} + +// StructsTokenizerOptions is the configuration provided to a StructsTokenizer. +type StructsTokenizerOptions struct { + WithCreateIndex bool + WithNamespace bool + WithID bool +} + +// StructsTokenizer is an pagination token generator that can create different +// formats of pagination tokens based on common fields found in the structs +// package. +type StructsTokenizer struct { + iter Iterator + opts StructsTokenizerOptions +} + +// NewStructsTokenizer returns a new StructsTokenizer. +func NewStructsTokenizer(it Iterator, opts StructsTokenizerOptions) StructsTokenizer { + return StructsTokenizer{ + iter: it, + opts: opts, + } +} + +func (it StructsTokenizer) GetToken(raw interface{}) string { + if raw == nil { + return "" + } + + parts := []string{} + + if it.opts.WithCreateIndex { + token := raw.(CreateIndexGetter).GetCreateIndex() + parts = append(parts, fmt.Sprintf("%v", token)) + } + + if it.opts.WithNamespace { + token := raw.(NamespaceGetter).GetNamespace() + parts = append(parts, token) + } + + if it.opts.WithID { + token := raw.(IDGetter).GetID() + parts = append(parts, token) + } + + // Use a character that is not part of validNamespaceName as separator to + // avoid accidentally generating collisions. + // For example, namespace `a` and job `b-c` would collide with namespace + // `a-b` and job `c` into the same token `a-b-c`, since `-` is an allowed + // character in namespace. + return strings.Join(parts, ".") +} diff --git a/nomad/state/paginator/tokenizer_test.go b/nomad/state/paginator/tokenizer_test.go new file mode 100644 index 00000000000..c74fe8a67fd --- /dev/null +++ b/nomad/state/paginator/tokenizer_test.go @@ -0,0 +1,67 @@ +package paginator + +import ( + "fmt" + "testing" + + "github.com/hashicorp/nomad/nomad/mock" + "github.com/stretchr/testify/require" +) + +func TestStructsTokenizer(t *testing.T) { + j := mock.Job() + + cases := []struct { + name string + opts StructsTokenizerOptions + expected string + }{ + { + name: "ID", + opts: StructsTokenizerOptions{ + WithID: true, + }, + expected: fmt.Sprintf("%v", j.ID), + }, + { + name: "Namespace.ID", + opts: StructsTokenizerOptions{ + WithNamespace: true, + WithID: true, + }, + expected: fmt.Sprintf("%v.%v", j.Namespace, j.ID), + }, + { + name: "CreateIndex.Namespace.ID", + opts: StructsTokenizerOptions{ + WithCreateIndex: true, + WithNamespace: true, + WithID: true, + }, + expected: fmt.Sprintf("%v.%v.%v", j.CreateIndex, j.Namespace, j.ID), + }, + { + name: "CreateIndex.ID", + opts: StructsTokenizerOptions{ + WithCreateIndex: true, + WithID: true, + }, + expected: fmt.Sprintf("%v.%v", j.CreateIndex, j.ID), + }, + { + name: "CreateIndex.Namespace", + opts: StructsTokenizerOptions{ + WithCreateIndex: true, + WithNamespace: true, + }, + expected: fmt.Sprintf("%v.%v", j.CreateIndex, j.Namespace), + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + tokenizer := StructsTokenizer{opts: tc.opts} + require.Equal(t, tc.expected, tokenizer.GetToken(j)) + }) + } +} diff --git a/nomad/structs/structs.go b/nomad/structs/structs.go index 739340105df..f0b110b20c9 100644 --- a/nomad/structs/structs.go +++ b/nomad/structs/structs.go @@ -9057,6 +9057,15 @@ func (d *Deployment) GetID() string { return d.ID } +// GetCreateIndex implements the CreateIndexGetter interface, required for +// pagination. +func (d *Deployment) GetCreateIndex() uint64 { + if d == nil { + return 0 + } + return d.CreateIndex +} + // HasPlacedCanaries returns whether the deployment has placed canaries func (d *Deployment) HasPlacedCanaries() bool { if d == nil || len(d.TaskGroups) == 0 { @@ -10548,6 +10557,23 @@ type Evaluation struct { ModifyTime int64 } +// GetID implements the IDGetter interface, required for pagination. +func (e *Evaluation) GetID() string { + if e == nil { + return "" + } + return e.ID +} + +// GetCreateIndex implements the CreateIndexGetter interface, required for +// pagination. +func (e *Evaluation) GetCreateIndex() uint64 { + if e == nil { + return 0 + } + return e.CreateIndex +} + // TerminalStatus returns if the current status is terminal and // will no longer transition. func (e *Evaluation) TerminalStatus() bool {