Skip to content

Commit

Permalink
api: split paginator components
Browse files Browse the repository at this point in the history
As more endpoints started using the API response paginator, new patterns
started to emerge that required either more flexible configuration or a
lot of re-used code.

This commit splits the paginator into three logical components:

- The Paginator is reponsible for splitting the values of an iterator
  into chunks of the requested size.
- The Tokenizer returns a page token that is used to uniquely identify
  elements of the iterator.
- The Filter is used to decide if a given element should be added to the
  page or skipped altogether.

The Paginator implementation has returned, more or less, to its initial
clean design and implementation.

Additional logic and flow customization can be provided by implementing
the Tokenizer and Filter interfaces.

One Tokenizer implementation is provided (StructsTokenizer) which can be
configured to emit tokens using common fields found in structs, such as
ID, Namespace, and CreateIndex.

The go-bexpr Evaluator implements the Filter interface, so it can be
used directly. Another implementation provided is the NamespaceFilter,
which can be used to apply filtering logic based on the namespaces the
ACL token used for the request is allowed to access.
  • Loading branch information
lgfa29 committed Mar 3, 2022
1 parent ad99a45 commit 1a5d07c
Show file tree
Hide file tree
Showing 11 changed files with 446 additions and 149 deletions.
45 changes: 15 additions & 30 deletions nomad/deployment_endpoint.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,33 +10,10 @@ import (
memdb "github.com/hashicorp/go-memdb"
"github.com/hashicorp/nomad/acl"
"github.com/hashicorp/nomad/nomad/state"
"github.com/hashicorp/nomad/nomad/state/paginator"
"github.com/hashicorp/nomad/nomad/structs"
)

// DeploymentPaginationIterator is a wrapper over a go-memdb iterator that
// implements the paginator Iterator interface.
type DeploymentPaginationIterator struct {
iter memdb.ResultIterator
byCreateIndex bool
}

func (it DeploymentPaginationIterator) Next() (string, interface{}) {
raw := it.iter.Next()
if raw == nil {
return "", nil
}

d := raw.(*structs.Deployment)
token := d.ID

// prefix the pagination token by CreateIndex to keep it properly sorted.
if it.byCreateIndex {
token = fmt.Sprintf("%v-%v", d.CreateIndex, d.ID)
}

return token, d
}

// Deployment endpoint is used for manipulating deployments
type Deployment struct {
srv *Server
Expand Down Expand Up @@ -433,26 +410,34 @@ func (d *Deployment) List(args *structs.DeploymentListRequest, reply *structs.De
// Capture all the deployments
var err error
var iter memdb.ResultIterator
var deploymentIter DeploymentPaginationIterator
var opts paginator.StructsTokenizerOptions

if prefix := args.QueryOptions.Prefix; prefix != "" {
iter, err = store.DeploymentsByIDPrefix(ws, namespace, prefix)
deploymentIter.byCreateIndex = false
opts = paginator.StructsTokenizerOptions{
WithID: true,
}
} else if namespace != structs.AllNamespacesSentinel {
iter, err = store.DeploymentsByNamespaceOrdered(ws, namespace, args.Ascending)
deploymentIter.byCreateIndex = true
opts = paginator.StructsTokenizerOptions{
WithCreateIndex: true,
WithID: true,
}
} else {
iter, err = store.Deployments(ws, args.Ascending)
deploymentIter.byCreateIndex = true
opts = paginator.StructsTokenizerOptions{
WithCreateIndex: true,
WithID: true,
}
}
if err != nil {
return err
}

deploymentIter.iter = iter
tokenizer := paginator.NewStructsTokenizer(iter, opts)

var deploys []*structs.Deployment
paginator, err := state.NewPaginator(deploymentIter, args.QueryOptions,
paginator, err := paginator.NewPaginator(iter, tokenizer, nil, args.QueryOptions,
func(raw interface{}) error {
deploy := raw.(*structs.Deployment)
deploys = append(deploys, deploy)
Expand Down
18 changes: 9 additions & 9 deletions nomad/deployment_endpoint_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1312,7 +1312,7 @@ func TestDeploymentEndpoint_List_Pagination(t *testing.T) {
{
name: "test01 size-2 page-1 default NS",
pageSize: 2,
expectedNextToken: "1003-aaaaaaaa-3350-4b4b-d185-0e1992ed43e9",
expectedNextToken: "1003.aaaaaaaa-3350-4b4b-d185-0e1992ed43e9",
expectedIDs: []string{
"aaaa1111-3350-4b4b-d185-0e1992ed43e9",
"aaaaaa22-3350-4b4b-d185-0e1992ed43e9",
Expand All @@ -1331,8 +1331,8 @@ func TestDeploymentEndpoint_List_Pagination(t *testing.T) {
{
name: "test03 size-2 page-2 default NS",
pageSize: 2,
nextToken: "1003-aaaaaaaa-3350-4b4b-d185-0e1992ed43e9",
expectedNextToken: "1005-aaaaaacc-3350-4b4b-d185-0e1992ed43e9",
nextToken: "1003.aaaaaaaa-3350-4b4b-d185-0e1992ed43e9",
expectedNextToken: "1005.aaaaaacc-3350-4b4b-d185-0e1992ed43e9",
expectedIDs: []string{
"aaaaaaaa-3350-4b4b-d185-0e1992ed43e9",
"aaaaaabb-3350-4b4b-d185-0e1992ed43e9",
Expand All @@ -1353,8 +1353,8 @@ func TestDeploymentEndpoint_List_Pagination(t *testing.T) {
name: "test05 size-2 page-2 all namespaces",
namespace: "*",
pageSize: 2,
nextToken: "1002-aaaaaa33-3350-4b4b-d185-0e1992ed43e9",
expectedNextToken: "1004-aaaaaabb-3350-4b4b-d185-0e1992ed43e9",
nextToken: "1002.aaaaaa33-3350-4b4b-d185-0e1992ed43e9",
expectedNextToken: "1004.aaaaaabb-3350-4b4b-d185-0e1992ed43e9",
expectedIDs: []string{
"aaaaaa33-3350-4b4b-d185-0e1992ed43e9",
"aaaaaaaa-3350-4b4b-d185-0e1992ed43e9",
Expand Down Expand Up @@ -1382,7 +1382,7 @@ func TestDeploymentEndpoint_List_Pagination(t *testing.T) {
namespace: "*",
filter: `ID matches "^a+[123]"`,
pageSize: 2,
expectedNextToken: "1002-aaaaaa33-3350-4b4b-d185-0e1992ed43e9",
expectedNextToken: "1002.aaaaaa33-3350-4b4b-d185-0e1992ed43e9",
expectedIDs: []string{
"aaaa1111-3350-4b4b-d185-0e1992ed43e9",
"aaaaaa22-3350-4b4b-d185-0e1992ed43e9",
Expand Down Expand Up @@ -1415,16 +1415,16 @@ func TestDeploymentEndpoint_List_Pagination(t *testing.T) {
{
name: "test13 non-lexicographic order",
pageSize: 1,
nextToken: "1007-00000111-3350-4b4b-d185-0e1992ed43e9",
expectedNextToken: "1009-bbbb1111-3350-4b4b-d185-0e1992ed43e9",
nextToken: "1007.00000111-3350-4b4b-d185-0e1992ed43e9",
expectedNextToken: "1009.bbbb1111-3350-4b4b-d185-0e1992ed43e9",
expectedIDs: []string{
"00000111-3350-4b4b-d185-0e1992ed43e9",
},
},
{
name: "test14 missing index",
pageSize: 1,
nextToken: "1008-e9522802-0cd8-4b1d-9c9e-ab3d97938371",
nextToken: "1008.e9522802-0cd8-4b1d-9c9e-ab3d97938371",
expectedIDs: []string{
"bbbb1111-3350-4b4b-d185-0e1992ed43e9",
},
Expand Down
46 changes: 16 additions & 30 deletions nomad/eval_endpoint.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (

"github.com/hashicorp/nomad/acl"
"github.com/hashicorp/nomad/nomad/state"
"github.com/hashicorp/nomad/nomad/state/paginator"
"github.com/hashicorp/nomad/nomad/structs"
"github.com/hashicorp/nomad/scheduler"
)
Expand All @@ -21,30 +22,6 @@ const (
DefaultDequeueTimeout = time.Second
)

// EvalPaginationIterator is a wrapper over a go-memdb iterator that implements
// the paginator Iterator interface.
type EvalPaginationIterator struct {
iter memdb.ResultIterator
byCreateIndex bool
}

func (it EvalPaginationIterator) Next() (string, interface{}) {
raw := it.iter.Next()
if raw == nil {
return "", nil
}

eval := raw.(*structs.Evaluation)
token := eval.ID

// prefix the pagination token by CreateIndex to keep it properly sorted.
if it.byCreateIndex {
token = fmt.Sprintf("%v-%v", eval.CreateIndex, eval.ID)
}

return token, eval
}

// Eval endpoint is used for eval interactions
type Eval struct {
srv *Server
Expand Down Expand Up @@ -438,17 +415,25 @@ func (e *Eval) List(args *structs.EvalListRequest, reply *structs.EvalListRespon
// Scan all the evaluations
var err error
var iter memdb.ResultIterator
var evalIter EvalPaginationIterator
var opts paginator.StructsTokenizerOptions

if prefix := args.QueryOptions.Prefix; prefix != "" {
iter, err = store.EvalsByIDPrefix(ws, namespace, prefix)
evalIter.byCreateIndex = false
opts = paginator.StructsTokenizerOptions{
WithID: true,
}
} else if namespace != structs.AllNamespacesSentinel {
iter, err = store.EvalsByNamespaceOrdered(ws, namespace, args.Ascending)
evalIter.byCreateIndex = true
opts = paginator.StructsTokenizerOptions{
WithCreateIndex: true,
WithID: true,
}
} else {
iter, err = store.Evals(ws, args.Ascending)
evalIter.byCreateIndex = true
opts = paginator.StructsTokenizerOptions{
WithCreateIndex: true,
WithID: true,
}
}
if err != nil {
return err
Expand All @@ -460,10 +445,11 @@ func (e *Eval) List(args *structs.EvalListRequest, reply *structs.EvalListRespon
}
return false
})
evalIter.iter = iter

tokenizer := paginator.NewStructsTokenizer(iter, opts)

var evals []*structs.Evaluation
paginator, err := state.NewPaginator(evalIter, args.QueryOptions,
paginator, err := paginator.NewPaginator(iter, tokenizer, nil, args.QueryOptions,
func(raw interface{}) error {
eval := raw.(*structs.Evaluation)
evals = append(evals, eval)
Expand Down
26 changes: 13 additions & 13 deletions nomad/eval_endpoint_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1084,7 +1084,7 @@ func TestEvalEndpoint_List_PaginationFiltering(t *testing.T) {
"aaaa1111-3350-4b4b-d185-0e1992ed43e9",
"aaaaaa22-3350-4b4b-d185-0e1992ed43e9",
},
expectedNextToken: "1003-aaaaaaaa-3350-4b4b-d185-0e1992ed43e9", // next one in default namespace
expectedNextToken: "1003.aaaaaaaa-3350-4b4b-d185-0e1992ed43e9", // next one in default namespace
},
{
name: "test02 size-2 page-1 default NS with prefix",
Expand All @@ -1099,8 +1099,8 @@ func TestEvalEndpoint_List_PaginationFiltering(t *testing.T) {
{
name: "test03 size-2 page-2 default NS",
pageSize: 2,
nextToken: "1003-aaaaaaaa-3350-4b4b-d185-0e1992ed43e9",
expectedNextToken: "1005-aaaaaacc-3350-4b4b-d185-0e1992ed43e9",
nextToken: "1003.aaaaaaaa-3350-4b4b-d185-0e1992ed43e9",
expectedNextToken: "1005.aaaaaacc-3350-4b4b-d185-0e1992ed43e9",
expectedIDs: []string{
"aaaaaaaa-3350-4b4b-d185-0e1992ed43e9",
"aaaaaabb-3350-4b4b-d185-0e1992ed43e9",
Expand All @@ -1123,7 +1123,7 @@ func TestEvalEndpoint_List_PaginationFiltering(t *testing.T) {
filterJobID: "example",
filterStatus: "pending",
// aaaaaaaa, bb, and cc are filtered by status
expectedNextToken: "1006-aaaaaadd-3350-4b4b-d185-0e1992ed43e9",
expectedNextToken: "1006.aaaaaadd-3350-4b4b-d185-0e1992ed43e9",
expectedIDs: []string{
"aaaa1111-3350-4b4b-d185-0e1992ed43e9",
"aaaaaa22-3350-4b4b-d185-0e1992ed43e9",
Expand Down Expand Up @@ -1159,7 +1159,7 @@ func TestEvalEndpoint_List_PaginationFiltering(t *testing.T) {
pageSize: 3, // reads off the end
filterJobID: "example",
filterStatus: "pending",
nextToken: "1003-aaaaaaaa-3350-4b4b-d185-0e1992ed43e9",
nextToken: "1003.aaaaaaaa-3350-4b4b-d185-0e1992ed43e9",
expectedNextToken: "",
expectedIDs: []string{
"aaaaaadd-3350-4b4b-d185-0e1992ed43e9",
Expand All @@ -1183,8 +1183,8 @@ func TestEvalEndpoint_List_PaginationFiltering(t *testing.T) {
name: "test10 size-2 page-2 all namespaces",
namespace: "*",
pageSize: 2,
nextToken: "1002-aaaaaa33-3350-4b4b-d185-0e1992ed43e9",
expectedNextToken: "1004-aaaaaabb-3350-4b4b-d185-0e1992ed43e9",
nextToken: "1002.aaaaaa33-3350-4b4b-d185-0e1992ed43e9",
expectedNextToken: "1004.aaaaaabb-3350-4b4b-d185-0e1992ed43e9",
expectedIDs: []string{
"aaaaaa33-3350-4b4b-d185-0e1992ed43e9",
"aaaaaaaa-3350-4b4b-d185-0e1992ed43e9",
Expand Down Expand Up @@ -1228,7 +1228,7 @@ func TestEvalEndpoint_List_PaginationFiltering(t *testing.T) {
name: "test16 go-bexpr filter with pagination",
filter: `JobID == "example"`,
pageSize: 2,
expectedNextToken: "1003-aaaaaaaa-3350-4b4b-d185-0e1992ed43e9",
expectedNextToken: "1003.aaaaaaaa-3350-4b4b-d185-0e1992ed43e9",
expectedIDs: []string{
"aaaa1111-3350-4b4b-d185-0e1992ed43e9",
"aaaaaa22-3350-4b4b-d185-0e1992ed43e9",
Expand Down Expand Up @@ -1267,25 +1267,25 @@ func TestEvalEndpoint_List_PaginationFiltering(t *testing.T) {
{
name: "test22 non-lexicographic order",
pageSize: 1,
nextToken: "1009-00000111-3350-4b4b-d185-0e1992ed43e9",
expectedNextToken: "1010-00000222-3350-4b4b-d185-0e1992ed43e9",
nextToken: "1009.00000111-3350-4b4b-d185-0e1992ed43e9",
expectedNextToken: "1010.00000222-3350-4b4b-d185-0e1992ed43e9",
expectedIDs: []string{
"00000111-3350-4b4b-d185-0e1992ed43e9",
},
},
{
name: "test23 same index",
pageSize: 1,
nextToken: "1010-00000222-3350-4b4b-d185-0e1992ed43e9",
expectedNextToken: "1010-00000333-3350-4b4b-d185-0e1992ed43e9",
nextToken: "1010.00000222-3350-4b4b-d185-0e1992ed43e9",
expectedNextToken: "1010.00000333-3350-4b4b-d185-0e1992ed43e9",
expectedIDs: []string{
"00000222-3350-4b4b-d185-0e1992ed43e9",
},
},
{
name: "test24 missing index",
pageSize: 1,
nextToken: "1011-e9522802-0cd8-4b1d-9c9e-ab3d97938371",
nextToken: "1011.e9522802-0cd8-4b1d-9c9e-ab3d97938371",
expectedIDs: []string{
"bbbb1111-3350-4b4b-d185-0e1992ed43e9",
},
Expand Down
41 changes: 41 additions & 0 deletions nomad/state/paginator/filter.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
package paginator

// Filter is the interface that must be implemented to skip values when using
// the Paginator.
type Filter interface {
// Evaluate returns true if the element should be added to the page.
Evaluate(interface{}) (bool, error)
}

// GenericFilter wraps a function that can be used to provide simple or in
// scope filtering.
type GenericFilter struct {
Allow func(interface{}) (bool, error)
}

func (f GenericFilter) Evaluate(raw interface{}) (bool, error) {
return f.Allow(raw)
}

// NamespaceFilter skips elements with a namespace value that is not in the
// allowable set.
type NamespaceFilter struct {
AllowableNamespaces map[string]bool
}

func (f NamespaceFilter) Evaluate(raw interface{}) (bool, error) {
if raw == nil {
return false, nil
}

item, _ := raw.(NamespaceGetter)
namespace := item.GetNamespace()

if f.AllowableNamespaces == nil {
return true, nil
}
if f.AllowableNamespaces[namespace] {
return true, nil
}
return false, nil
}
Loading

0 comments on commit 1a5d07c

Please sign in to comment.