diff --git a/CHANGELOG.md b/CHANGELOG.md index d3f48b19d9..e5d8be821d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,6 +19,8 @@ For details about compatibility between different releases, see the **Commitment ### Fixed +- Enforce default page limit on AS and NS List RPCs if a value is not provided in the request. + ### Security ## [3.33.0] - unreleased diff --git a/cmd/ttn-lw-stack/commands/start.go b/cmd/ttn-lw-stack/commands/start.go index 940dcfa3ad..ec3eaf7b09 100644 --- a/cmd/ttn-lw-stack/commands/start.go +++ b/cmd/ttn-lw-stack/commands/start.go @@ -79,6 +79,16 @@ func NewNetworkServerDownlinkTaskRedis(conf *Config) *redis.Client { return redis.New(conf.Redis.WithNamespace("ns", "tasks")) } +// NewNetworkServerMACSettingsProfileRegistryRedis instantiates a new redis client +// with the Network Server MAC Settings Profile Registry namespace. +func NewNetworkServerMACSettingsProfileRegistryRedis(conf *Config) *redis.Client { + redis.SetPaginationDefaults(redis.PaginationDefaults{ + DefaultLimit: conf.NS.Pagination.DefaultLimit, + }) + + return redis.New(conf.Redis.WithNamespace("ns", "mac-settings-profiles")) +} + // NewIdentityServerTelemetryTaskRedis instantiates a new redis client // with the Identity Server Telemetry Task namespace. func NewIdentityServerTelemetryTaskRedis(conf *Config) *redis.Client { @@ -91,6 +101,36 @@ func NewApplicationServerDeviceRegistryRedis(conf *Config) *redis.Client { return NewComponentDeviceRegistryRedis(conf, "as") } +// NewApplicationServerPubSubRegistryRedis instantiates a new redis client +// with the Application Server PubSub Registry namespace. +func NewApplicationServerPubSubRegistryRedis(conf *Config) *redis.Client { + redis.SetPaginationDefaults(redis.PaginationDefaults{ + DefaultLimit: conf.AS.Pagination.DefaultLimit, + }) + + return redis.New(config.Redis.WithNamespace("as", "io", "pubsub")) +} + +// NewApplicationServerPackagesRegistryRedis instantiates a new redis client +// with the Application Server Packages Registry namespace. +func NewApplicationServerPackagesRegistryRedis(conf *Config) *redis.Client { + redis.SetPaginationDefaults(redis.PaginationDefaults{ + DefaultLimit: conf.AS.Pagination.DefaultLimit, + }) + + return redis.New(config.Redis.WithNamespace("as", "io", "applicationpackages")) +} + +// NewApplicationServerWebhookRegistryRedis instantiates a new redis client +// with the Application Server Webhook Registry namespace. +func NewApplicationServerWebhookRegistryRedis(conf *Config) *redis.Client { + redis.SetPaginationDefaults(redis.PaginationDefaults{ + DefaultLimit: conf.AS.Pagination.DefaultLimit, + }) + + return redis.New(config.Redis.WithNamespace("as", "io", "webhooks")) +} + // NewJoinServerDeviceRegistryRedis instantiates a new redis client // with the Join Server Device Registry namespace. func NewJoinServerDeviceRegistryRedis(conf *Config) *redis.Client { @@ -343,7 +383,7 @@ var startCommand = &cobra.Command{ Redis: redis.New(config.Cache.Redis.WithNamespace("ns", "scheduled-downlinks")), } macSettingsProfiles := &nsredis.MACSettingsProfileRegistry{ - Redis: redis.New(config.Redis.WithNamespace("ns", "mac-settings-profiles")), + Redis: NewNetworkServerMACSettingsProfileRegistryRedis(config), LockTTL: defaultLockTTL, } if err := macSettingsProfiles.Init(ctx); err != nil { @@ -379,7 +419,7 @@ var startCommand = &cobra.Command{ Redis: redis.New(config.Cache.Redis.WithNamespace("as", "traffic")), } pubsubRegistry := &asiopsredis.PubSubRegistry{ - Redis: redis.New(config.Redis.WithNamespace("as", "io", "pubsub")), + Redis: NewApplicationServerPubSubRegistryRedis(config), LockTTL: defaultLockTTL, } if err := pubsubRegistry.Init(ctx); err != nil { @@ -388,7 +428,7 @@ var startCommand = &cobra.Command{ config.AS.PubSub.Registry = pubsubRegistry applicationPackagesRegistry, err := asioapredis.NewApplicationPackagesRegistry( ctx, - redis.New(config.Redis.WithNamespace("as", "io", "applicationpackages")), + NewApplicationServerPackagesRegistryRedis(config), defaultLockTTL, ) if err != nil { @@ -397,7 +437,7 @@ var startCommand = &cobra.Command{ config.AS.Packages.Registry = applicationPackagesRegistry if config.AS.Webhooks.Target != "" { webhookRegistry := &asiowebredis.WebhookRegistry{ - Redis: redis.New(config.Redis.WithNamespace("as", "io", "webhooks")), + Redis: NewApplicationServerWebhookRegistryRedis(config), LockTTL: defaultLockTTL, } if err := webhookRegistry.Init(ctx); err != nil { diff --git a/pkg/applicationserver/config.go b/pkg/applicationserver/config.go index 97f3b3e0db..54b46ca378 100644 --- a/pkg/applicationserver/config.go +++ b/pkg/applicationserver/config.go @@ -105,6 +105,11 @@ type DownlinksConfig struct { ConfirmationConfig ConfirmationConfig `name:"confirmation" description:"Configuration for confirmed downlink"` } +// PaginationConfig represents the configuration for pagination. +type PaginationConfig struct { + DefaultLimit int64 `name:"default-limit" description:"Default limit for pagination"` +} + // Config represents the ApplicationServer configuration. type Config struct { LinkMode string `name:"link-mode" description:"Deprecated - mode to link applications to their Network Server (all, explicit)"` @@ -123,6 +128,7 @@ type Config struct { DeviceKEKLabel string `name:"device-kek-label" description:"Label of KEK used to encrypt device keys at rest"` DeviceLastSeen LastSeenConfig `name:"device-last-seen" description:"End Device last seen batch update configuration"` Downlinks DownlinksConfig `name:"downlinks" description:"Downlink configuration"` + Pagination PaginationConfig `name:"pagination" description:"Pagination configuration"` } func (c Config) toProto() *ttnpb.AsConfiguration { diff --git a/pkg/applicationserver/io/packages/grpc_test.go b/pkg/applicationserver/io/packages/grpc_test.go index cb0cc6a158..5069cd7fdb 100644 --- a/pkg/applicationserver/io/packages/grpc_test.go +++ b/pkg/applicationserver/io/packages/grpc_test.go @@ -30,6 +30,7 @@ import ( "go.thethings.network/lorawan-stack/v3/pkg/config" "go.thethings.network/lorawan-stack/v3/pkg/errors" mockis "go.thethings.network/lorawan-stack/v3/pkg/identityserver/mock" + ttnredis "go.thethings.network/lorawan-stack/v3/pkg/redis" "go.thethings.network/lorawan-stack/v3/pkg/rpcmetadata" "go.thethings.network/lorawan-stack/v3/pkg/ttnpb" "go.thethings.network/lorawan-stack/v3/pkg/unique" @@ -371,6 +372,8 @@ func TestAssociations(t *testing.T) { t.Run("Pagination", func(t *testing.T) { a := assertions.New(t) + ttnredis.SetPaginationDefaults(ttnredis.PaginationDefaults{DefaultLimit: 10}) + for i := 1; i < 21; i++ { association := &ttnpb.ApplicationPackageAssociation{ Ids: &ttnpb.ApplicationPackageAssociationIdentifiers{ @@ -424,8 +427,8 @@ func TestAssociations(t *testing.T) { limit: 0, page: 0, portLow: 1, - portHigh: 20, - length: 20, + portHigh: 10, + length: 10, }, } { t.Run(fmt.Sprintf("limit:%v_page:%v", tc.limit, tc.page), diff --git a/pkg/applicationserver/io/pubsub/redis/registry.go b/pkg/applicationserver/io/pubsub/redis/registry.go index ef718fbf4a..01134e6c0f 100644 --- a/pkg/applicationserver/io/pubsub/redis/registry.go +++ b/pkg/applicationserver/io/pubsub/redis/registry.go @@ -16,6 +16,7 @@ package redis import ( "context" + "runtime/trace" "time" "github.com/redis/go-redis/v9" @@ -132,19 +133,54 @@ func (r PubSubRegistry) Range(ctx context.Context, paths []string, f func(contex func (r PubSubRegistry) List(ctx context.Context, ids *ttnpb.ApplicationIdentifiers, paths []string) ([]*ttnpb.ApplicationPubSub, error) { var pbs []*ttnpb.ApplicationPubSub appUID := unique.ID(ctx, ids) - err := ttnredis.FindProtos(ctx, r.Redis, r.appKey(appUID), r.makeUIDKeyFunc(appUID)).Range(func() (proto.Message, func() (bool, error)) { - pb := &ttnpb.ApplicationPubSub{} - return pb, func() (bool, error) { - pb, err := applyPubSubFieldMask(nil, pb, appendImplicitPubSubGetPaths(paths...)...) + uidKey := r.appKey(appUID) + + opts := []ttnredis.FindProtosOption{} + limit, offset := ttnredis.PaginationLimitAndOffsetFromContext(ctx) + if limit != 0 { + opts = append(opts, + ttnredis.FindProtosSorted(true), + ttnredis.FindProtosWithOffsetAndCount(offset, limit), + ) + } + + rangeProtos := func(c redis.Cmdable) error { + return ttnredis.FindProtos(ctx, c, uidKey, r.makeUIDKeyFunc(appUID), opts...).Range( + func() (proto.Message, func() (bool, error)) { + pb := &ttnpb.ApplicationPubSub{} + return pb, func() (bool, error) { + pb, err := applyPubSubFieldMask(nil, pb, appendImplicitPubSubGetPaths(paths...)...) + if err != nil { + return false, err + } + pbs = append(pbs, pb) + return true, nil + } + }) + } + + defer trace.StartRegion(ctx, "list pubsub by application id").End() + + var err error + if limit != 0 { + var lockerID string + lockerID, err = ttnredis.GenerateLockerID() + if err != nil { + return nil, err + } + err = ttnredis.LockedWatch(ctx, r.Redis, uidKey, lockerID, r.LockTTL, func(tx *redis.Tx) (err error) { + total, err := tx.SCard(ctx, uidKey).Result() if err != nil { - return false, err + return err } - pbs = append(pbs, pb) - return true, nil - } - }) + ttnredis.SetPaginationTotal(ctx, total) + return rangeProtos(tx) + }) + } else { + err = rangeProtos(r.Redis) + } if err != nil { - return nil, err + return nil, ttnredis.ConvertError(err) } return pbs, nil } @@ -283,3 +319,13 @@ func (r PubSubRegistry) Set(ctx context.Context, ids *ttnpb.ApplicationPubSubIde } return pb, nil } + +// WithPagination returns a new context with pagination parameters. +func (PubSubRegistry) WithPagination( + ctx context.Context, + limit uint32, + page uint32, + total *int64, +) context.Context { + return ttnredis.NewContextWithPagination(ctx, int64(limit), int64(page), total) +} diff --git a/pkg/applicationserver/io/pubsub/redis/registry_test.go b/pkg/applicationserver/io/pubsub/redis/registry_test.go new file mode 100644 index 0000000000..68e829e8a4 --- /dev/null +++ b/pkg/applicationserver/io/pubsub/redis/registry_test.go @@ -0,0 +1,281 @@ +// Copyright © 2024 The Things Network Foundation, The Things Industries B.V. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package redis implements a Redis-backed PubSub registry. +package redis + +import ( + "fmt" + "testing" + + "go.thethings.network/lorawan-stack/v3/pkg/errors" + ttnredis "go.thethings.network/lorawan-stack/v3/pkg/redis" + "go.thethings.network/lorawan-stack/v3/pkg/ttnpb" + "go.thethings.network/lorawan-stack/v3/pkg/util/test" + "go.thethings.network/lorawan-stack/v3/pkg/util/test/assertions/should" +) + +var Timeout = 10 * test.Delay + +func TestPubSubRegistry(t *testing.T) { + t.Parallel() + a, ctx := test.New(t) + cl, flush := test.NewRedis(ctx, "redis_test") + t.Cleanup(func() { + flush() + cl.Close() + }) + + ids := &ttnpb.ApplicationPubSubIdentifiers{ + ApplicationIds: &ttnpb.ApplicationIdentifiers{ + ApplicationId: "myapp-00", + }, + PubSubId: "pubsub-00", + } + ids1 := &ttnpb.ApplicationPubSubIdentifiers{ + ApplicationIds: &ttnpb.ApplicationIdentifiers{ + ApplicationId: "myapp-01", + }, + PubSubId: "pubsub-01", + } + ids2 := &ttnpb.ApplicationPubSubIdentifiers{ + ApplicationIds: &ttnpb.ApplicationIdentifiers{ + ApplicationId: "myapp-02", + }, + PubSubId: "pubsub-02", + } + provider := &ttnpb.ApplicationPubSub_Nats{ + Nats: &ttnpb.ApplicationPubSub_NATSProvider{ + ServerUrl: "nats://localhost", + }, + } + + registry := &PubSubRegistry{ + Redis: cl, + LockTTL: test.Delay << 10, + } + if err := registry.Init(ctx); !a.So(err, should.BeNil) { + t.FailNow() + } + + paths := []string{"ids", "provider", "format", "base_topic"} + format := "json" + format2 := "xml" + baseTopic := "app1.ps1" + baseTopic2 := "app1.ps2" + + createPubSubFunc := func(ps *ttnpb.ApplicationPubSub, + ) (*ttnpb.ApplicationPubSub, []string, error) { // nolint: unparam + a.So(ps, should.BeNil) + return &ttnpb.ApplicationPubSub{ + Ids: ids1, + Provider: provider, + Format: format, + }, paths, nil + } + updatePubSubFunc := func(ps *ttnpb.ApplicationPubSub, + ) (*ttnpb.ApplicationPubSub, []string, error) { // nolint: unparam + a.So(ps, should.NotBeNil) + return &ttnpb.ApplicationPubSub{ + Ids: ids1, + Provider: provider, + Format: format, + BaseTopic: baseTopic, + }, paths, nil + } + updateFieldMaskPubSubFunc := func(ps *ttnpb.ApplicationPubSub, + ) (*ttnpb.ApplicationPubSub, []string, error) { // nolint: unparam + a.So(ps, should.NotBeNil) + return &ttnpb.ApplicationPubSub{ + Ids: ids1, + Provider: provider, + Format: format2, + BaseTopic: baseTopic2, + }, []string{"ids", "base_topic"}, nil + } + deletePubSubFunc := func(ps *ttnpb.ApplicationPubSub, + ) (*ttnpb.ApplicationPubSub, []string, error) { // nolint: unparam + a.So(ps, should.NotBeNil) + return nil, nil, nil + } + listPubSubFunc := func(ps *ttnpb.ApplicationPubSub, + ) (*ttnpb.ApplicationPubSub, []string, error) { // nolint: unparam + a.So(ps, should.BeNil) + return &ttnpb.ApplicationPubSub{ + Ids: ids2, + Provider: provider, + Format: format, + }, paths, nil + } + + t.Run("GetNonExisting", func(t *testing.T) { + t.Parallel() + a, ctx := test.New(t) + pubsub, err := registry.Get(ctx, ids, []string{"ids"}) + a.So(pubsub, should.BeNil) + a.So(errors.IsNotFound(err), should.BeTrue) + }) + + t.Run("CreateReadUpdateDelete", func(t *testing.T) { + t.Parallel() + a, ctx := test.New(t) + pubsub, err := registry.Set(ctx, ids1, paths, createPubSubFunc) + a.So(err, should.BeNil) + a.So(pubsub, should.NotBeNil) + a.So(pubsub.Ids, should.Resemble, ids1) + a.So(pubsub.Format, should.NotBeNil) + a.So(pubsub.Format, should.Equal, format) + + retrieved, err := registry.Get(ctx, ids1, paths) + a.So(err, should.BeNil) + a.So(retrieved, should.NotBeNil) + a.So(retrieved.Ids, should.Resemble, ids1) + a.So(retrieved.Format, should.NotBeNil) + a.So(retrieved.Format, should.Equal, format) + + updated, err := registry.Set(ctx, ids1, paths, updatePubSubFunc) + a.So(err, should.BeNil) + a.So(updated, should.NotBeNil) + a.So(updated.Ids, should.Resemble, ids1) + a.So(updated.Format, should.NotBeNil) + a.So(updated.Format, should.Equal, format) + a.So(updated.BaseTopic, should.NotBeNil) + a.So(updated.BaseTopic, should.Equal, baseTopic) + + updated2, err := registry.Set(ctx, ids1, paths, updateFieldMaskPubSubFunc) + a.So(err, should.BeNil) + a.So(updated2, should.NotBeNil) + a.So(updated2.Ids, should.Resemble, ids1) + a.So(updated2.Format, should.NotBeNil) + a.So(updated2.Format, should.Equal, format) + a.So(updated2.BaseTopic, should.NotBeNil) + a.So(updated2.BaseTopic, should.Equal, baseTopic2) + + deleted, err := registry.Set(ctx, ids1, []string{"ids"}, deletePubSubFunc) + a.So(err, should.BeNil) + a.So(deleted, should.BeNil) + }) + + t.Run("List", func(t *testing.T) { + t.Parallel() + a, ctx := test.New(t) + pubsub, err := registry.Set(ctx, ids2, paths, listPubSubFunc) + a.So(err, should.BeNil) + a.So(pubsub, should.NotBeNil) + a.So(pubsub.Ids, should.Resemble, ids2) + a.So(pubsub.Format, should.NotBeNil) + a.So(pubsub.Format, should.Equal, format) + + pubsubs, err := registry.List(ctx, ids2.ApplicationIds, paths) + a.So(err, should.BeNil) + a.So(pubsubs, should.HaveLength, 1) + a.So(pubsubs[0], should.NotBeNil) + a.So(pubsubs[0].Ids, should.Resemble, ids2) + a.So(pubsubs[0].Format, should.NotBeNil) + a.So(pubsubs[0].Format, should.Equal, format) + }) + + t.Run("Pagination", func(t *testing.T) { + t.Parallel() + a, ctx := test.New(t) + + ttnredis.SetPaginationDefaults(ttnredis.PaginationDefaults{DefaultLimit: 10}) + + for i := 1; i < 21; i++ { + ids3 := &ttnpb.ApplicationPubSubIdentifiers{ + ApplicationIds: &ttnpb.ApplicationIdentifiers{ + ApplicationId: "myapp-pagination", + }, + PubSubId: fmt.Sprintf("pubsub-%02d", i), + } + + pubsub, err := registry.Set( + ctx, + ids3, + paths, + func(ps *ttnpb.ApplicationPubSub) (*ttnpb.ApplicationPubSub, []string, error) { + a.So(ps, should.BeNil) + return &ttnpb.ApplicationPubSub{ + Ids: ids3, + Provider: provider, + Format: format, + }, paths, nil + }, + ) + a.So(err, should.BeNil) + a.So(pubsub, should.NotBeNil) + } + + for _, tc := range []struct { + limit uint32 + page uint32 + idLow string + idHigh string + length int + }{ + { + limit: 10, + page: 0, + idLow: "pubsub-01", + idHigh: "pubsub-10", + length: 10, + }, + { + limit: 10, + page: 1, + idLow: "pubsub-01", + idHigh: "pubsub-10", + length: 10, + }, + { + limit: 10, + page: 2, + idLow: "pubsub-11", + idHigh: "pubsub-20", + length: 10, + }, + { + limit: 10, + page: 3, + length: 0, + }, + { + limit: 0, + page: 0, + idLow: "pubsub-01", + idHigh: "pubsub-10", + length: 10, + }, + } { + t.Run(fmt.Sprintf("limit:%v_page:%v", tc.limit, tc.page), + func(t *testing.T) { + t.Parallel() + var total int64 + paginationCtx := registry.WithPagination(ctx, tc.limit, tc.page, &total) + + pubsubs, err := registry.List(paginationCtx, &ttnpb.ApplicationIdentifiers{ + ApplicationId: "myapp-pagination", + }, + paths, + ) + a.So(err, should.BeNil) + a.So(pubsubs, should.HaveLength, tc.length) + a.So(total, should.Equal, 20) + for _, pubsub := range pubsubs { + a.So(pubsub.Ids.PubSubId, should.BeBetweenOrEqual, tc.idLow, tc.idHigh) + } + }) + } + }) +} diff --git a/pkg/applicationserver/io/pubsub/registry.go b/pkg/applicationserver/io/pubsub/registry.go index b4a2a5b5c6..21a25b7af5 100644 --- a/pkg/applicationserver/io/pubsub/registry.go +++ b/pkg/applicationserver/io/pubsub/registry.go @@ -30,4 +30,6 @@ type Registry interface { List(ctx context.Context, ids *ttnpb.ApplicationIdentifiers, paths []string) ([]*ttnpb.ApplicationPubSub, error) // Set creates, updates or deletes the pub/sub integration by its identifiers. Set(ctx context.Context, ids *ttnpb.ApplicationPubSubIdentifiers, paths []string, f func(*ttnpb.ApplicationPubSub) (*ttnpb.ApplicationPubSub, []string, error)) (*ttnpb.ApplicationPubSub, error) + // WithPagination returns a new context with pagination parameters. + WithPagination(ctx context.Context, limit uint32, page uint32, total *int64) context.Context } diff --git a/pkg/applicationserver/io/web/redis/registry.go b/pkg/applicationserver/io/web/redis/registry.go index 3ecd739818..937ed2b01a 100644 --- a/pkg/applicationserver/io/web/redis/registry.go +++ b/pkg/applicationserver/io/web/redis/registry.go @@ -17,6 +17,7 @@ package redis import ( "context" "regexp" + "runtime/trace" "strings" "time" @@ -99,19 +100,55 @@ func (r WebhookRegistry) Get(ctx context.Context, ids *ttnpb.ApplicationWebhookI func (r WebhookRegistry) List(ctx context.Context, ids *ttnpb.ApplicationIdentifiers, paths []string) ([]*ttnpb.ApplicationWebhook, error) { var pbs []*ttnpb.ApplicationWebhook appUID := unique.ID(ctx, ids) - err := ttnredis.FindProtos(ctx, r.Redis, r.appKey(appUID), r.makeIDKeyFunc(appUID)).Range(func() (proto.Message, func() (bool, error)) { - pb := &ttnpb.ApplicationWebhook{} - return pb, func() (bool, error) { - pb, err := applyWebhookFieldMask(nil, pb, appendImplicitWebhookGetPaths(paths...)...) + uidKey := r.appKey(appUID) + + opts := []ttnredis.FindProtosOption{} + limit, offset := ttnredis.PaginationLimitAndOffsetFromContext(ctx) + if limit != 0 { + opts = append(opts, + ttnredis.FindProtosSorted(true), + ttnredis.FindProtosWithOffsetAndCount(offset, limit), + ) + } + + rangeProtos := func(c redis.Cmdable) error { + return ttnredis.FindProtos(ctx, c, uidKey, r.makeIDKeyFunc(appUID), opts...).Range( + func() (proto.Message, func() (bool, error)) { + pb := &ttnpb.ApplicationWebhook{} + return pb, func() (bool, error) { + pb, err := applyWebhookFieldMask(nil, pb, appendImplicitWebhookGetPaths(paths...)...) + if err != nil { + return false, err + } + pbs = append(pbs, pb) + return true, nil + } + }) + } + + defer trace.StartRegion(ctx, "list webhooks by application id").End() + + var err error + if limit != 0 { + var lockerID string + lockerID, err = ttnredis.GenerateLockerID() + if err != nil { + return nil, err + } + err = ttnredis.LockedWatch(ctx, r.Redis, uidKey, lockerID, r.LockTTL, func(tx *redis.Tx) (err error) { + total, err := tx.SCard(ctx, uidKey).Result() if err != nil { - return false, err + return err } - pbs = append(pbs, pb) - return true, nil - } - }) + ttnredis.SetPaginationTotal(ctx, total) + return rangeProtos(tx) + }) + } else { + err = rangeProtos(r.Redis) + } + if err != nil { - return nil, err + return nil, ttnredis.ConvertError(err) } return pbs, nil } @@ -267,3 +304,13 @@ func (r WebhookRegistry) Range(ctx context.Context, paths []string, f func(conte return true, nil }) } + +// WithPagination returns a new context with pagination parameters. +func (WebhookRegistry) WithPagination( + ctx context.Context, + limit uint32, + page uint32, + total *int64, +) context.Context { + return ttnredis.NewContextWithPagination(ctx, int64(limit), int64(page), total) +} diff --git a/pkg/applicationserver/io/web/redis/registry_test.go b/pkg/applicationserver/io/web/redis/registry_test.go new file mode 100644 index 0000000000..696a141e33 --- /dev/null +++ b/pkg/applicationserver/io/web/redis/registry_test.go @@ -0,0 +1,274 @@ +// Copyright © 2024 The Things Network Foundation, The Things Industries B.V. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package redis implements a Redis-backed Webhook registry. +package redis + +import ( + "fmt" + "testing" + + "go.thethings.network/lorawan-stack/v3/pkg/errors" + ttnredis "go.thethings.network/lorawan-stack/v3/pkg/redis" + "go.thethings.network/lorawan-stack/v3/pkg/ttnpb" + "go.thethings.network/lorawan-stack/v3/pkg/util/test" + "go.thethings.network/lorawan-stack/v3/pkg/util/test/assertions/should" +) + +var Timeout = 10 * test.Delay + +func TestWebhookRegistry(t *testing.T) { + t.Parallel() + a, ctx := test.New(t) + cl, flush := test.NewRedis(ctx, "redis_test") + t.Cleanup(func() { + flush() + cl.Close() + }) + + ids := &ttnpb.ApplicationWebhookIdentifiers{ + ApplicationIds: &ttnpb.ApplicationIdentifiers{ + ApplicationId: "myapp-00", + }, + WebhookId: "webhook-00", + } + ids1 := &ttnpb.ApplicationWebhookIdentifiers{ + ApplicationIds: &ttnpb.ApplicationIdentifiers{ + ApplicationId: "myapp-01", + }, + WebhookId: "webhook-01", + } + ids2 := &ttnpb.ApplicationWebhookIdentifiers{ + ApplicationIds: &ttnpb.ApplicationIdentifiers{ + ApplicationId: "myapp-02", + }, + WebhookId: "webhook-02", + } + + registry := &WebhookRegistry{ + Redis: cl, + LockTTL: test.Delay << 10, + } + if err := registry.Init(ctx); !a.So(err, should.BeNil) { + t.FailNow() + } + + paths := []string{"ids", "base_url", "format"} + format := "json" + format2 := "xml" + baseURL := "http://localhost/test1" + baseURL2 := "http://localhost/test2" + + createWebhookFunc := func(ps *ttnpb.ApplicationWebhook, + ) (*ttnpb.ApplicationWebhook, []string, error) { // nolint: unparam + a.So(ps, should.BeNil) + return &ttnpb.ApplicationWebhook{ + Ids: ids1, + Format: format, + BaseUrl: baseURL, + }, paths, nil + } + updateWebhookFunc := func(ps *ttnpb.ApplicationWebhook, + ) (*ttnpb.ApplicationWebhook, []string, error) { // nolint: unparam + a.So(ps, should.NotBeNil) + return &ttnpb.ApplicationWebhook{ + Ids: ids1, + Format: format, + BaseUrl: baseURL, + }, paths, nil + } + updateFieldMaskWebhookFunc := func(ps *ttnpb.ApplicationWebhook, + ) (*ttnpb.ApplicationWebhook, []string, error) { // nolint: unparam + a.So(ps, should.NotBeNil) + return &ttnpb.ApplicationWebhook{ + Ids: ids1, + Format: format2, + BaseUrl: baseURL2, + }, []string{"ids", "base_url"}, nil + } + deleteWebhookFunc := func(ps *ttnpb.ApplicationWebhook, + ) (*ttnpb.ApplicationWebhook, []string, error) { // nolint: unparam + a.So(ps, should.NotBeNil) + return nil, nil, nil + } + listWebhookFunc := func(ps *ttnpb.ApplicationWebhook, + ) (*ttnpb.ApplicationWebhook, []string, error) { // nolint: unparam + a.So(ps, should.BeNil) + return &ttnpb.ApplicationWebhook{ + Ids: ids2, + Format: format, + BaseUrl: baseURL, + }, paths, nil + } + + t.Run("GetNonExisting", func(t *testing.T) { + t.Parallel() + a, ctx := test.New(t) + webhook, err := registry.Get(ctx, ids, []string{"ids"}) + a.So(webhook, should.BeNil) + a.So(errors.IsNotFound(err), should.BeTrue) + }) + + t.Run("CreateReadUpdateDelete", func(t *testing.T) { + t.Parallel() + a, ctx := test.New(t) + webhook, err := registry.Set(ctx, ids1, paths, createWebhookFunc) + a.So(err, should.BeNil) + a.So(webhook, should.NotBeNil) + a.So(webhook.Ids, should.Resemble, ids1) + a.So(webhook.Format, should.NotBeNil) + a.So(webhook.Format, should.Equal, format) + + retrieved, err := registry.Get(ctx, ids1, paths) + a.So(err, should.BeNil) + a.So(retrieved, should.NotBeNil) + a.So(retrieved.Ids, should.Resemble, ids1) + a.So(retrieved.Format, should.NotBeNil) + a.So(retrieved.Format, should.Equal, format) + + updated, err := registry.Set(ctx, ids1, paths, updateWebhookFunc) + a.So(err, should.BeNil) + a.So(updated, should.NotBeNil) + a.So(updated.Ids, should.Resemble, ids1) + a.So(updated.Format, should.NotBeNil) + a.So(updated.Format, should.Equal, format) + a.So(updated.BaseUrl, should.NotBeNil) + a.So(updated.BaseUrl, should.Equal, baseURL) + + updated2, err := registry.Set(ctx, ids1, paths, updateFieldMaskWebhookFunc) + a.So(err, should.BeNil) + a.So(updated2, should.NotBeNil) + a.So(updated2.Ids, should.Resemble, ids1) + a.So(updated2.Format, should.NotBeNil) + a.So(updated2.Format, should.Equal, format) + a.So(updated2.BaseUrl, should.NotBeNil) + a.So(updated2.BaseUrl, should.Equal, baseURL2) + + deleted, err := registry.Set(ctx, ids1, []string{"ids"}, deleteWebhookFunc) + a.So(err, should.BeNil) + a.So(deleted, should.BeNil) + }) + + t.Run("List", func(t *testing.T) { + t.Parallel() + a, ctx := test.New(t) + webhook, err := registry.Set(ctx, ids2, paths, listWebhookFunc) + a.So(err, should.BeNil) + a.So(webhook, should.NotBeNil) + a.So(webhook.Ids, should.Resemble, ids2) + a.So(webhook.Format, should.NotBeNil) + a.So(webhook.Format, should.Equal, format) + + webhooks, err := registry.List(ctx, ids2.ApplicationIds, paths) + a.So(err, should.BeNil) + a.So(webhooks, should.HaveLength, 1) + a.So(webhooks[0], should.NotBeNil) + a.So(webhooks[0].Ids, should.Resemble, ids2) + a.So(webhooks[0].Format, should.NotBeNil) + a.So(webhooks[0].Format, should.Equal, format) + }) + + t.Run("Pagination", func(t *testing.T) { + t.Parallel() + a, ctx := test.New(t) + + ttnredis.SetPaginationDefaults(ttnredis.PaginationDefaults{DefaultLimit: 10}) + + for i := 1; i < 21; i++ { + ids3 := &ttnpb.ApplicationWebhookIdentifiers{ + ApplicationIds: &ttnpb.ApplicationIdentifiers{ + ApplicationId: "myapp-pagination", + }, + WebhookId: fmt.Sprintf("webhook-%02d", i), + } + + webhook, err := registry.Set( + ctx, + ids3, + paths, + func(ps *ttnpb.ApplicationWebhook) (*ttnpb.ApplicationWebhook, []string, error) { + a.So(ps, should.BeNil) + return &ttnpb.ApplicationWebhook{ + Ids: ids3, + Format: format, + BaseUrl: baseURL, + }, paths, nil + }, + ) + a.So(err, should.BeNil) + a.So(webhook, should.NotBeNil) + } + + for _, tc := range []struct { + limit uint32 + page uint32 + idLow string + idHigh string + length int + }{ + { + limit: 10, + page: 0, + idLow: "webhook-01", + idHigh: "webhook-10", + length: 10, + }, + { + limit: 10, + page: 1, + idLow: "webhook-01", + idHigh: "webhook-10", + length: 10, + }, + { + limit: 10, + page: 2, + idLow: "webhook-11", + idHigh: "webhook-20", + length: 10, + }, + { + limit: 10, + page: 3, + length: 0, + }, + { + limit: 0, + page: 0, + idLow: "webhook-01", + idHigh: "webhook-10", + length: 10, + }, + } { + t.Run(fmt.Sprintf("limit:%v_page:%v", tc.limit, tc.page), + func(t *testing.T) { + t.Parallel() + var total int64 + paginationCtx := registry.WithPagination(ctx, tc.limit, tc.page, &total) + + webhooks, err := registry.List(paginationCtx, &ttnpb.ApplicationIdentifiers{ + ApplicationId: "myapp-pagination", + }, + paths, + ) + a.So(err, should.BeNil) + a.So(webhooks, should.HaveLength, tc.length) + a.So(total, should.Equal, 20) + for _, webhook := range webhooks { + a.So(webhook.Ids.WebhookId, should.BeBetweenOrEqual, tc.idLow, tc.idHigh) + } + }) + } + }) +} diff --git a/pkg/applicationserver/io/web/registry.go b/pkg/applicationserver/io/web/registry.go index df7f43a3f1..7e1823cce9 100644 --- a/pkg/applicationserver/io/web/registry.go +++ b/pkg/applicationserver/io/web/registry.go @@ -30,4 +30,6 @@ type WebhookRegistry interface { Set(ctx context.Context, ids *ttnpb.ApplicationWebhookIdentifiers, paths []string, f func(*ttnpb.ApplicationWebhook) (*ttnpb.ApplicationWebhook, []string, error)) (*ttnpb.ApplicationWebhook, error) // Range ranges over the webhooks and calls the callback function, until false is returned. Range(ctx context.Context, paths []string, f func(context.Context, *ttnpb.ApplicationIdentifiers, *ttnpb.ApplicationWebhook) bool) error + // WithPagination returns a new context with pagination parameters. + WithPagination(ctx context.Context, limit uint32, page uint32, total *int64) context.Context } diff --git a/pkg/networkserver/config.go b/pkg/networkserver/config.go index d2de27dc1e..9dfa2b0062 100644 --- a/pkg/networkserver/config.go +++ b/pkg/networkserver/config.go @@ -131,6 +131,11 @@ type InteropConfig struct { ID *types.EUI64 `name:"id" description:"NSID of this Network Server (EUI)"` } +// PaginationConfig represents the configuration for pagination. +type PaginationConfig struct { + DefaultLimit int64 `name:"default-limit" description:"Default limit for pagination"` +} + // Config represents the NetworkServer configuration. type Config struct { ApplicationUplinkQueue ApplicationUplinkQueueConfig `name:"application-uplink-queue"` @@ -149,6 +154,7 @@ type Config struct { DeviceKEKLabel string `name:"device-kek-label" description:"Label of KEK used to encrypt device keys at rest"` // nolint: lll DownlinkQueueCapacity int `name:"downlink-queue-capacity" description:"Maximum downlink queue size per-session"` // nolint: lll MACSettingsProfileRegistry MACSettingsProfileRegistry `name:"-"` + Pagination PaginationConfig `name:"pagination" description:"Pagination configuration"` } // DefaultConfig is the default Network Server configuration. diff --git a/pkg/networkserver/grpc_mac_settings_profile.go b/pkg/networkserver/grpc_mac_settings_profile.go index 759f8131d7..c91fb4aad4 100644 --- a/pkg/networkserver/grpc_mac_settings_profile.go +++ b/pkg/networkserver/grpc_mac_settings_profile.go @@ -31,6 +31,10 @@ var ( errMACSettingsProfileNotFound = errors.DefineNotFound("mac_settings_profile_not_found", "MAC settings profile not found") // nolint: lll ) +func setTotalHeader(ctx context.Context, total int64) { + grpc.SetHeader(ctx, metadata.Pairs("x-total-count", strconv.FormatInt(total, 10))) // nolint: errcheck +} + // NsMACSettingsProfileRegistry implements the MAC settings profile registry grpc service. type NsMACSettingsProfileRegistry struct { ttnpb.UnimplementedNsMACSettingsProfileRegistryServer @@ -161,13 +165,17 @@ func (m *NsMACSettingsProfileRegistry) List(ctx context.Context, req *ttnpb.List if req.FieldMask != nil { paths = req.FieldMask.GetPaths() } + + var total int64 + ctx = m.registry.WithPagination(ctx, req.Limit, req.Page, &total) + profiles, err := m.registry.List(ctx, req.ApplicationIds, paths) if err != nil { logRegistryRPCError(ctx, err, "Failed to list MAC settings profiles") return nil, err } - grpc.SetHeader(ctx, metadata.Pairs("x-total-count", strconv.FormatInt(int64(len(profiles)), 10))) // nolint: errcheck + setTotalHeader(ctx, total) return &ttnpb.ListMACSettingsProfilesResponse{ MacSettingsProfiles: profiles, }, nil diff --git a/pkg/networkserver/grpc_mac_settings_profile_test.go b/pkg/networkserver/grpc_mac_settings_profile_test.go index a82e41a7b7..340b115353 100644 --- a/pkg/networkserver/grpc_mac_settings_profile_test.go +++ b/pkg/networkserver/grpc_mac_settings_profile_test.go @@ -973,10 +973,12 @@ func TestMACSettingsProfileRegistryList(t *testing.T) { Name string ContextFunc func(context.Context) context.Context ListFunc func(context.Context, *ttnpb.ApplicationIdentifiers, []string) ([]*ttnpb.MACSettingsProfile, error) // nolint: lll + PaginationFunc func(context.Context, uint32, uint32, *int64) context.Context ProfileRequest *ttnpb.ListMACSettingsProfilesRequest ProfileAssertion func(*testing.T, *ttnpb.ListMACSettingsProfilesResponse) bool ErrorAssertion func(*testing.T, error) bool ListCalls uint64 + PaginationCalls uint64 }{ { Name: "Permission denied", @@ -996,6 +998,16 @@ func TestMACSettingsProfileRegistryList(t *testing.T) { test.MustTFromContext(ctx).Error(err) return nil, err }, + PaginationFunc: func( + ctx context.Context, + _ uint32, + _ uint32, + _ *int64, + ) context.Context { + err := errors.New("PaginationFunc must not be called") + test.MustTFromContext(ctx).Error(err) + return ctx + }, ProfileRequest: &ttnpb.ListMACSettingsProfilesRequest{ ApplicationIds: registeredProfileIDs.ApplicationIds, FieldMask: ttnpb.FieldMask("mac_settings"), @@ -1003,6 +1015,7 @@ func TestMACSettingsProfileRegistryList(t *testing.T) { ProfileAssertion: nilProfileAssertion, ErrorAssertion: permissionDeniedErrorAssertion, ListCalls: 0, + PaginationCalls: 0, }, { Name: "Invalid application ID", @@ -1026,6 +1039,16 @@ func TestMACSettingsProfileRegistryList(t *testing.T) { test.MustTFromContext(ctx).Error(err) return nil, err }, + PaginationFunc: func( + ctx context.Context, + _ uint32, + _ uint32, + _ *int64, + ) context.Context { + err := errors.New("PaginationFunc must not be called") + test.MustTFromContext(ctx).Error(err) + return ctx + }, ProfileRequest: &ttnpb.ListMACSettingsProfilesRequest{ ApplicationIds: registeredProfileIDs.ApplicationIds, FieldMask: ttnpb.FieldMask("mac_settings"), @@ -1033,6 +1056,7 @@ func TestMACSettingsProfileRegistryList(t *testing.T) { ProfileAssertion: nilProfileAssertion, ErrorAssertion: permissionDeniedErrorAssertion, ListCalls: 0, + PaginationCalls: 0, }, { Name: "Not found", @@ -1059,6 +1083,18 @@ func TestMACSettingsProfileRegistryList(t *testing.T) { }) return nil, errNotFound.New() }, + PaginationFunc: func( + ctx context.Context, + limit uint32, + page uint32, + total *int64, + ) context.Context { + a := assertions.New(test.MustTFromContext(ctx)) + a.So(limit, should.Equal, 0) + a.So(page, should.Equal, 0) + a.So(total, should.NotBeNil) + return ctx + }, ProfileRequest: &ttnpb.ListMACSettingsProfilesRequest{ ApplicationIds: registeredProfileIDs.ApplicationIds, FieldMask: ttnpb.FieldMask("mac_settings"), @@ -1066,6 +1102,7 @@ func TestMACSettingsProfileRegistryList(t *testing.T) { ProfileAssertion: nilProfileAssertion, ErrorAssertion: notFoundErrorAssertion, ListCalls: 1, + PaginationCalls: 1, }, { Name: "Found", @@ -1098,9 +1135,23 @@ func TestMACSettingsProfileRegistryList(t *testing.T) { }, })}, nil }, + PaginationFunc: func( + ctx context.Context, + limit uint32, + page uint32, + total *int64, + ) context.Context { + a := assertions.New(test.MustTFromContext(ctx)) + a.So(limit, should.Equal, 2) + a.So(page, should.Equal, 1) + a.So(total, should.NotBeNil) + return ctx + }, ProfileRequest: &ttnpb.ListMACSettingsProfilesRequest{ ApplicationIds: registeredProfileIDs.ApplicationIds, FieldMask: ttnpb.FieldMask("ids", "mac_settings"), + Limit: 2, + Page: 1, }, ProfileAssertion: func(t *testing.T, profile *ttnpb.ListMACSettingsProfilesResponse) bool { t.Helper() @@ -1119,8 +1170,9 @@ func TestMACSettingsProfileRegistryList(t *testing.T) { }, }}) }, - ErrorAssertion: nilErrorAssertion, - ListCalls: 1, + ErrorAssertion: nilErrorAssertion, + ListCalls: 1, + PaginationCalls: 1, }, } { tc := tc @@ -1129,7 +1181,7 @@ func TestMACSettingsProfileRegistryList(t *testing.T) { Parallel: true, Func: func(ctx context.Context, t *testing.T, a *assertions.Assertion) { t.Helper() - var listCalls uint64 + var listCalls, paginationCalls uint64 ns, ctx, _, stop := StartTest( ctx, @@ -1144,6 +1196,15 @@ func TestMACSettingsProfileRegistryList(t *testing.T) { atomic.AddUint64(&listCalls, 1) return tc.ListFunc(ctx, ids, paths) }, + WithPaginationFunc: func( + ctx context.Context, + limit uint32, + page uint32, + total *int64, + ) context.Context { + atomic.AddUint64(&paginationCalls, 1) + return tc.PaginationFunc(ctx, limit, page, total) + }, }, }, TaskStarter: StartTaskExclude( @@ -1167,6 +1228,7 @@ func TestMACSettingsProfileRegistryList(t *testing.T) { } a.So(req, should.Resemble, tc.ProfileRequest) a.So(listCalls, should.Equal, tc.ListCalls) + a.So(paginationCalls, should.Equal, tc.PaginationCalls) }, }) } diff --git a/pkg/networkserver/networkserver_util_internal_test.go b/pkg/networkserver/networkserver_util_internal_test.go index dce552dc05..fa7d0a78e3 100644 --- a/pkg/networkserver/networkserver_util_internal_test.go +++ b/pkg/networkserver/networkserver_util_internal_test.go @@ -2614,6 +2614,12 @@ type MockMACSettingsProfileRegistry struct { ids *ttnpb.ApplicationIdentifiers, paths []string, ) ([]*ttnpb.MACSettingsProfile, error) + WithPaginationFunc func( + ctx context.Context, + limit uint32, + page uint32, + total *int64, + ) context.Context } func (m MockMACSettingsProfileRegistry) Get( @@ -2649,3 +2655,15 @@ func (m MockMACSettingsProfileRegistry) List( } return m.ListFunc(ctx, ids, paths) } + +func (m MockMACSettingsProfileRegistry) WithPagination( + ctx context.Context, + limit uint32, + page uint32, + total *int64, +) context.Context { + if m.WithPaginationFunc == nil { + panic("WithPaginationFunc not set") + } + return m.WithPaginationFunc(ctx, limit, page, total) +} diff --git a/pkg/networkserver/redis/mac_settings_profile.go b/pkg/networkserver/redis/mac_settings_profile.go index a43a4f092b..3907999052 100644 --- a/pkg/networkserver/redis/mac_settings_profile.go +++ b/pkg/networkserver/redis/mac_settings_profile.go @@ -228,26 +228,69 @@ func (r *MACSettingsProfileRegistry) List( return nil, err } - appUID := unique.ID(ctx, ids) var pbs []*ttnpb.MACSettingsProfile - err := ttnredis.FindProtos( - ctx, - r.Redis, - r.appKey(appUID), - r.makeProfileKeyFunc(appUID), - ).Range(func() (proto.Message, func() (bool, error)) { - pb := &ttnpb.MACSettingsProfile{} - return pb, func() (bool, error) { - pb, err := applyMACSettingsProfileFieldMask(nil, pb, paths...) - if err != nil { - return false, err + appUID := unique.ID(ctx, ids) + uidKey := r.appKey(appUID) + + opts := []ttnredis.FindProtosOption{} + limit, offset := ttnredis.PaginationLimitAndOffsetFromContext(ctx) + if limit != 0 { + opts = append(opts, + ttnredis.FindProtosSorted(true), + ttnredis.FindProtosWithOffsetAndCount(offset, limit), + ) + } + + rangeProtos := func(c redis.Cmdable) error { + return ttnredis.FindProtos( + ctx, + c, + uidKey, + r.makeProfileKeyFunc(appUID), + opts..., + ).Range(func() (proto.Message, func() (bool, error)) { + pb := &ttnpb.MACSettingsProfile{} + return pb, func() (bool, error) { + pb, err := applyMACSettingsProfileFieldMask(nil, pb, paths...) + if err != nil { + return false, err + } + pbs = append(pbs, pb) + return true, nil } - pbs = append(pbs, pb) - return true, nil + }) + } + + var err error + if limit != 0 { + var lockerID string + lockerID, err = ttnredis.GenerateLockerID() + if err != nil { + return nil, err } - }) + err = ttnredis.LockedWatch(ctx, r.Redis, uidKey, lockerID, r.LockTTL, func(tx *redis.Tx) (err error) { + total, err := tx.SCard(ctx, uidKey).Result() + if err != nil { + return err + } + ttnredis.SetPaginationTotal(ctx, total) + return rangeProtos(tx) + }) + } else { + err = rangeProtos(r.Redis) + } if err != nil { - return nil, err + return nil, ttnredis.ConvertError(err) } return pbs, nil } + +// WithPagination returns a new context with pagination parameters. +func (*MACSettingsProfileRegistry) WithPagination( + ctx context.Context, + limit uint32, + page uint32, + total *int64, +) context.Context { + return ttnredis.NewContextWithPagination(ctx, int64(limit), int64(page), total) +} diff --git a/pkg/networkserver/redis/mac_settings_profile_test.go b/pkg/networkserver/redis/mac_settings_profile_test.go index b7709d8b11..9e5696d0c0 100644 --- a/pkg/networkserver/redis/mac_settings_profile_test.go +++ b/pkg/networkserver/redis/mac_settings_profile_test.go @@ -17,9 +17,11 @@ package redis import ( "context" + "fmt" "testing" "go.thethings.network/lorawan-stack/v3/pkg/errors" + ttnredis "go.thethings.network/lorawan-stack/v3/pkg/redis" "go.thethings.network/lorawan-stack/v3/pkg/ttnpb" "go.thethings.network/lorawan-stack/v3/pkg/util/test" "go.thethings.network/lorawan-stack/v3/pkg/util/test/assertions/should" @@ -38,7 +40,13 @@ func TestMACSettingsProfileRegistry(t *testing.T) { ids := &ttnpb.MACSettingsProfileIdentifiers{ ApplicationIds: &ttnpb.ApplicationIdentifiers{ - ApplicationId: "myapp", + ApplicationId: "myapp-00", + }, + ProfileId: "prof-00", + } + ids1 := &ttnpb.MACSettingsProfileIdentifiers{ + ApplicationIds: &ttnpb.ApplicationIdentifiers{ + ApplicationId: "myapp-01", }, ProfileId: "prof-01", } @@ -64,7 +72,7 @@ func TestMACSettingsProfileRegistry(t *testing.T) { ) (*ttnpb.MACSettingsProfile, []string, error) { // nolint: unparam a.So(pb, should.BeNil) return &ttnpb.MACSettingsProfile{ - Ids: ids, + Ids: ids1, MacSettings: &ttnpb.MACSettings{ ResetsFCnt: &ttnpb.BoolValue{Value: true}, }, @@ -75,7 +83,7 @@ func TestMACSettingsProfileRegistry(t *testing.T) { ) (*ttnpb.MACSettingsProfile, []string, error) { // nolint: unparam a.So(pb, should.NotBeNil) return &ttnpb.MACSettingsProfile{ - Ids: ids, + Ids: ids1, MacSettings: &ttnpb.MACSettings{ ResetsFCnt: &ttnpb.BoolValue{Value: false}, FactoryPresetFrequencies: frequencies, @@ -87,7 +95,7 @@ func TestMACSettingsProfileRegistry(t *testing.T) { ) (*ttnpb.MACSettingsProfile, []string, error) { // nolint: unparam a.So(pb, should.NotBeNil) return &ttnpb.MACSettingsProfile{ - Ids: ids, + Ids: ids1, MacSettings: &ttnpb.MACSettings{ ResetsFCnt: &ttnpb.BoolValue{Value: true}, FactoryPresetFrequencies: frequencies2, @@ -123,41 +131,41 @@ func TestMACSettingsProfileRegistry(t *testing.T) { t.Run("CreateReadUpdateDelete", func(t *testing.T) { t.Parallel() a, ctx := test.New(t) - profile, err := registry.Set(ctx, ids, []string{"ids", "mac_settings"}, createProfileFunc) + profile, err := registry.Set(ctx, ids1, []string{"ids", "mac_settings"}, createProfileFunc) a.So(err, should.BeNil) a.So(profile, should.NotBeNil) - a.So(profile.Ids, should.Resemble, ids) + a.So(profile.Ids, should.Resemble, ids1) a.So(profile.MacSettings, should.NotBeNil) a.So(profile.MacSettings.ResetsFCnt, should.NotBeNil) a.So(profile.MacSettings.ResetsFCnt.Value, should.BeTrue) - retrieved, err := registry.Get(ctx, ids, []string{"ids", "mac_settings"}) + retrieved, err := registry.Get(ctx, ids1, []string{"ids", "mac_settings"}) a.So(err, should.BeNil) a.So(retrieved, should.NotBeNil) - a.So(retrieved.Ids, should.Resemble, ids) + a.So(retrieved.Ids, should.Resemble, ids1) a.So(retrieved.MacSettings, should.NotBeNil) a.So(retrieved.MacSettings.ResetsFCnt, should.NotBeNil) a.So(retrieved.MacSettings.ResetsFCnt.Value, should.BeTrue) - updated, err := registry.Set(ctx, ids, []string{"ids", "mac_settings"}, updateProfileFunc) + updated, err := registry.Set(ctx, ids1, []string{"ids", "mac_settings"}, updateProfileFunc) a.So(err, should.BeNil) a.So(updated, should.NotBeNil) - a.So(updated.Ids, should.Resemble, ids) + a.So(updated.Ids, should.Resemble, ids1) a.So(updated.MacSettings, should.NotBeNil) a.So(updated.MacSettings.ResetsFCnt, should.NotBeNil) a.So(updated.MacSettings.ResetsFCnt.Value, should.BeFalse) a.So(updated.MacSettings.FactoryPresetFrequencies, should.Resemble, frequencies) - updated2, err := registry.Set(ctx, ids, []string{"ids", "mac_settings"}, updateFieldMaskProfileFunc) + updated2, err := registry.Set(ctx, ids1, []string{"ids", "mac_settings"}, updateFieldMaskProfileFunc) a.So(err, should.BeNil) a.So(updated2, should.NotBeNil) - a.So(updated2.Ids, should.Resemble, ids) + a.So(updated2.Ids, should.Resemble, ids1) a.So(updated2.MacSettings, should.NotBeNil) a.So(updated2.MacSettings.ResetsFCnt, should.NotBeNil) a.So(updated2.MacSettings.ResetsFCnt.Value, should.BeFalse) a.So(updated2.MacSettings.FactoryPresetFrequencies, should.Resemble, frequencies2) - deleted, err := registry.Set(ctx, ids, []string{"ids", "mac_settings"}, deleteProfileFunc) + deleted, err := registry.Set(ctx, ids1, []string{"ids", "mac_settings"}, deleteProfileFunc) a.So(err, should.BeNil) a.So(deleted, should.BeNil) }) @@ -182,4 +190,99 @@ func TestMACSettingsProfileRegistry(t *testing.T) { a.So(profiles[0].MacSettings.ResetsFCnt, should.NotBeNil) a.So(profiles[0].MacSettings.ResetsFCnt.Value, should.BeTrue) }) + + t.Run("Pagination", func(t *testing.T) { + t.Parallel() + a, ctx := test.New(t) + + ttnredis.SetPaginationDefaults(ttnredis.PaginationDefaults{DefaultLimit: 10}) + + for i := 1; i < 21; i++ { + ids3 := &ttnpb.MACSettingsProfileIdentifiers{ + ApplicationIds: &ttnpb.ApplicationIdentifiers{ + ApplicationId: "myapp-pagination", + }, + ProfileId: fmt.Sprintf("listprof-%02d", i), + } + + profile, err := registry.Set( + ctx, + ids3, + []string{"ids", "mac_settings"}, + func(_ context.Context, pb *ttnpb.MACSettingsProfile, + ) (*ttnpb.MACSettingsProfile, []string, error) { // nolint: unparam + a.So(pb, should.BeNil) + return &ttnpb.MACSettingsProfile{ + Ids: ids3, + MacSettings: &ttnpb.MACSettings{ + ResetsFCnt: &ttnpb.BoolValue{Value: true}, + }, + }, []string{"ids", "mac_settings"}, nil + }, + ) + a.So(err, should.BeNil) + a.So(profile, should.NotBeNil) + } + + for _, tc := range []struct { + limit uint32 + page uint32 + idLow string + idHigh string + length int + }{ + { + limit: 10, + page: 0, + idLow: "listprof-01", + idHigh: "listprof-10", + length: 10, + }, + { + limit: 10, + page: 1, + idLow: "listprof-01", + idHigh: "listprof-10", + length: 10, + }, + { + limit: 10, + page: 2, + idLow: "listprof-11", + idHigh: "listprof-20", + length: 10, + }, + { + limit: 10, + page: 3, + length: 0, + }, + { + limit: 0, + page: 0, + idLow: "listprof-01", + idHigh: "listprof-10", + length: 10, + }, + } { + t.Run(fmt.Sprintf("limit:%v_page:%v", tc.limit, tc.page), + func(t *testing.T) { + t.Parallel() + var total int64 + paginationCtx := registry.WithPagination(ctx, tc.limit, tc.page, &total) + + profiles, err := registry.List(paginationCtx, &ttnpb.ApplicationIdentifiers{ + ApplicationId: "myapp-pagination", + }, + []string{"ids", "mac_settings"}, + ) + a.So(err, should.BeNil) + a.So(profiles, should.HaveLength, tc.length) + a.So(total, should.Equal, 20) + for _, profile := range profiles { + a.So(profile.Ids.ProfileId, should.BeBetweenOrEqual, tc.idLow, tc.idHigh) + } + }) + } + }) } diff --git a/pkg/networkserver/registry.go b/pkg/networkserver/registry.go index 9c81df9072..0f10b25734 100644 --- a/pkg/networkserver/registry.go +++ b/pkg/networkserver/registry.go @@ -305,4 +305,5 @@ type MACSettingsProfileRegistry interface { Get(ctx context.Context, ids *ttnpb.MACSettingsProfileIdentifiers, paths []string) (*ttnpb.MACSettingsProfile, error) // nolint: lll Set(ctx context.Context, ids *ttnpb.MACSettingsProfileIdentifiers, paths []string, f func(context.Context, *ttnpb.MACSettingsProfile) (*ttnpb.MACSettingsProfile, []string, error)) (*ttnpb.MACSettingsProfile, error) // nolint: lll List(ctx context.Context, ids *ttnpb.ApplicationIdentifiers, paths []string) ([]*ttnpb.MACSettingsProfile, error) // nolint: lll + WithPagination(ctx context.Context, limit uint32, page uint32, total *int64) context.Context } diff --git a/pkg/redis/pagination.go b/pkg/redis/pagination.go index b1c447b443..73e5e9ef8c 100644 --- a/pkg/redis/pagination.go +++ b/pkg/redis/pagination.go @@ -18,6 +18,18 @@ import ( "context" ) +// PaginationDefaults sets default values for paginations options within the Redis store. +type PaginationDefaults struct { + DefaultLimit int64 +} + +var paginationDefaults = PaginationDefaults{} + +// SetPaginationDefaults should only be called at the initialization of the server. +func SetPaginationDefaults(d PaginationDefaults) { + paginationDefaults = d +} + type paginationOptionsKeyType struct{} var paginationOptionsKey paginationOptionsKeyType @@ -33,6 +45,9 @@ func NewContextWithPagination(ctx context.Context, limit, page int64, total *int if page == 0 { page = 1 } + if limit == 0 { + limit = paginationDefaults.DefaultLimit + } return context.WithValue(ctx, paginationOptionsKey, paginationOptions{ limit: limit, offset: (page - 1) * limit, diff --git a/pkg/redis/pagination_test.go b/pkg/redis/pagination_test.go index 537a8dfb06..ba54fe8e25 100644 --- a/pkg/redis/pagination_test.go +++ b/pkg/redis/pagination_test.go @@ -84,4 +84,16 @@ func TestPagination(t *testing.T) { redis.SetPaginationTotal(ctx, total) a.So(totalCount, should.Equal, total) }) + + t.Run("SetPaginationDefaults", func(t *testing.T) { + t.Parallel() + redis.SetPaginationDefaults(redis.PaginationDefaults{DefaultLimit: 20}) + + ctx := test.Context() + ctx = redis.NewContextWithPagination(ctx, 0, 0, nil) + + limit, _ := redis.PaginationLimitAndOffsetFromContext(ctx) + + a.So(limit, should.Equal, uint64(20)) + }) }