feat: limit concurrent processing of thumbnail requests (#9252)

changelog/unreleased/thumbnail-request-limit.md

@@ -0,0 +1,6 @@
+Enhancement: Limit concurrent thumbnail requests
+
+The number of concurrent requests to the thumbnail service can be limited now
+to have more control over the consumed system resources.
+
+https://github.com/owncloud/ocis/pull/9199

ocis-pkg/middleware/throttle.go

@@ -0,0 +1,19 @@
+package middleware
+
+import (
+	"net/http"
+
+	"github.com/go-chi/chi/v5/middleware"
+)
+
+// Throttle limits the number of concurrent requests.
+func Throttle(limit int) func(http.Handler) http.Handler {
+	if limit > 0 {
+		return middleware.Throttle(limit)
+	}
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			next.ServeHTTP(w, r)
+		})
+	}
+}

services/thumbnails/README.md

@@ -41,41 +41,41 @@ Thumbnails can either be generated as `png`, `jpg` or `gif` files. These types a
 
 ## Thumbnail Query String Parameters
 
-Clients can request thumbnail previews for files by adding `?preview=1` to the file URL. Requests for files with thumbnail availabe respond with HTTP status `404`.
+Clients can request thumbnail previews for files by adding `?preview=1` to the file URL. Requests for files with thumbnail available respond with HTTP status `404`.
 
 The following query parameters are supported:
 
 | Parameter | Required | Default Value                                        | Description                                                                     |
-| --------- | -------- | ---------------------------------------------------- | ------------------------------------------------------------------------------- |
+|-----------|----------|------------------------------------------------------|---------------------------------------------------------------------------------|
 | preview   | YES      | 1                                                    | generates preview                                                               |
 | x         | YES      | first x-value configured in `THUMBNAILS_RESOLUTIONS` | horizontal target size                                                          |
 | y         | YES      | first y-value configured in `THUMBNAILS_RESOLUTIONS` | vertical target size                                                            |
-| scalingup | NO       | 0                                                    | prevents upscaling of small images                                              |
+| scalingup | NO       | 0                                                    | prevents up-scaling of small images                                             |
 | a         | NO       | 1                                                    | aspect ratio                                                                    |
 | c         | NO       | Caching string                                       | Clients should send the etag, so they get a fresh thumbnail after a file change |
-| processor | NO       | `resize` for gif's and `thumbnail` for all others    | preferred thumbnail processor                                                   |
+| processor | NO       | `resize` for gifs and `thumbnail` for all others     | preferred thumbnail processor                                                   |
 
 ## Thumbnail Resolution
 
 Various resolutions can be defined via `THUMBNAILS_RESOLUTIONS`. A requestor can request any arbitrary resolution and the thumbnail service will use the one closest to the requested resolution. If more than one resolution is required, each resolution must be requested individually.
 
 Example:
 
-Requested: 18x12  
-Available: 30x20, 15x10, 9x6  
-Returned: 15x10  
+Requested: 18x12
+Available: 30x20, 15x10, 9x6
+Returned: 15x10
 
 ## Thumbnail Processors
 
-Normally, an image might get cropped when creating a preview, depending on the aspect ratio of the original image. This can have negative 
+Normally, an image might get cropped when creating a preview, depending on the aspect ratio of the original image. This can have negative
 impacts on previews as only a part of the image will be shown. When using an _optional_ processor in the request, cropping can be avoided by defining on how the preview image generation will be done. The following processors are available:
 
 *   `resize` resizes the image to the specified width and height and returns the transformed image. If one of width or height is 0, the image aspect ratio is preserved.
 *   `fit` scales down the image to fit the specified maximum width and height and returns the transformed image.
 *   `fill`: creates an image with the specified dimensions and fills it with the scaled source image. To achieve the correct aspect ratio without stretching, the source image will be cropped.
-*   `thumbnail` scales the image up or down, crops it to the specified width and hight and returns the transformed image.
+*   `thumbnail` scales the image up or down, crops it to the specified width and height and returns the transformed image.
 
-To apply one of those, a query parameter has to be added to the request, like `?processor=fit`. If no query parameter or processor is added, the default behaviour applies which is `resize` for gif's and `thumbnail` for all others.
+To apply one of those, a query parameter has to be added to the request, like `?processor=fit`. If no query parameter or processor is added, the default behaviour applies which is `resize` for gifs and `thumbnail` for all others.
 
 ## Deleting Thumbnails
 
@@ -84,3 +84,4 @@ As of now, there is no automated thumbnail deletion. This is especially true whe
 ## Memory Considerations
 
 Since source files need to be loaded into memory when generating thumbnails, large source files could potentially crash this service if there is insufficient memory available. For bigger instances when using container orchestration deployment methods, this service can be dedicated to its own server(s) with more memory.
+To have more control over memory (and CPU) consumption the maximum number of concurrent requests can be limited by setting the environment variable `THUMBNAILS_MAX_CONCURRENT_REQUESTS`. The default value is 0 which does not apply any restrictions to the number of concurrent requests. As soon as the number of concurrent requests is reached any further request will be responded with `429/Too Many Requests` and the client can retry at a later point in time.

services/thumbnails/pkg/command/server.go

@@ -98,6 +98,7 @@ func Server(cfg *config.Config) *cli.Command {
 				http.Metrics(metrics),
 				http.Namespace(cfg.HTTP.Namespace),
 				http.TraceProvider(traceProvider),
+				http.MaxConcurrentRequests(cfg.HTTP.MaxConcurrentRequests),
 			)
 			if err != nil {
 				logger.Info().

services/thumbnails/pkg/config/defaults/defaultconfig.go

@@ -32,9 +32,10 @@ func DefaultConfig() *config.Config {
 			Namespace: "com.owncloud.api",
 		},
 		HTTP: config.HTTP{
-			Addr:      "127.0.0.1:9186",
-			Root:      "/thumbnails",
-			Namespace: "com.owncloud.web",
+			Addr:                  "127.0.0.1:9186",
+			Root:                  "/thumbnails",
+			Namespace:             "com.owncloud.web",
+			MaxConcurrentRequests: 0,
 		},
 		Service: config.Service{
 			Name: "thumbnails",

services/thumbnails/pkg/config/http.go

@@ -4,8 +4,9 @@ import "github.com/owncloud/ocis/v2/ocis-pkg/shared"
 
 // HTTP defines the available http configuration.
 type HTTP struct {
-	Addr      string                `yaml:"addr" env:"THUMBNAILS_HTTP_ADDR" desc:"The bind address of the HTTP service." introductionVersion:"pre5.0"`
-	TLS       shared.HTTPServiceTLS `yaml:"tls"`
-	Root      string                `yaml:"root" env:"THUMBNAILS_HTTP_ROOT" desc:"Subdirectory that serves as the root for this HTTP service." introductionVersion:"pre5.0"`
-	Namespace string                `yaml:"-"`
+	Addr                  string                `yaml:"addr" env:"THUMBNAILS_HTTP_ADDR" desc:"The bind address of the HTTP service." introductionVersion:"pre5.0"`
+	TLS                   shared.HTTPServiceTLS `yaml:"tls"`
+	Root                  string                `yaml:"root" env:"THUMBNAILS_HTTP_ROOT" desc:"Subdirectory that serves as the root for this HTTP service." introductionVersion:"pre5.0"`
+	Namespace             string                `yaml:"-"`
+	MaxConcurrentRequests int                   `yaml:"max_concurrent_requests" env:"THUMBNAILS_MAX_CONCURRENT_REQUESTS" desc:"Number of maximum concurrent thumbnail requests. Default is 0 which is unlimited." introductionVersion:"6.0"`
 }

services/thumbnails/pkg/server/http/option.go

@@ -15,13 +15,14 @@ type Option func(o *Options)
 
 // Options defines the available options for this package.
 type Options struct {
-	Namespace     string
-	Logger        log.Logger
-	Context       context.Context
-	Config        *config.Config
-	Metrics       *metrics.Metrics
-	Flags         []cli.Flag
-	TraceProvider trace.TracerProvider
+	Namespace             string
+	Logger                log.Logger
+	Context               context.Context
+	Config                *config.Config
+	Metrics               *metrics.Metrics
+	Flags                 []cli.Flag
+	TraceProvider         trace.TracerProvider
+	MaxConcurrentRequests int
 }
 
 // newOptions initializes the available default options.
@@ -80,3 +81,10 @@ func TraceProvider(traceProvider trace.TracerProvider) Option {
 		}
 	}
 }
+
+// MaxConcurrentRequests provides a function to set the MaxConcurrentRequests option.
+func MaxConcurrentRequests(val int) Option {
+	return func(o *Options) {
+		o.MaxConcurrentRequests = val
+	}
+}

services/thumbnails/pkg/server/http/server.go

@@ -40,6 +40,7 @@ func Server(opts ...Option) (http.Service, error) {
 			middleware.RealIP,
 			middleware.RequestID,
 			// ocismiddleware.Secure,
+			ocismiddleware.Throttle(options.MaxConcurrentRequests),
 			ocismiddleware.Version(
 				options.Config.Service.Name,
 				version.GetString(),

services/thumbnails/pkg/service/http/v0/service.go

@@ -3,12 +3,11 @@ package svc
 import (
 	"context"
 	"fmt"
-	"net/http"
-	"strconv"
-
 	"github.com/go-chi/chi/v5"
 	"github.com/golang-jwt/jwt/v4"
 	"github.com/riandyrn/otelchi"
+	"net/http"
+	"strconv"
 
 	"github.com/owncloud/ocis/v2/ocis-pkg/log"
 	"github.com/owncloud/ocis/v2/ocis-pkg/tracing"

services/webdav/pkg/service/v0/search.go

@@ -47,7 +47,7 @@ func (g Webdav) Search(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	t := r.Header.Get(TokenHeader)
+	t := r.Header.Get(revactx.TokenHeader)
 	ctx := revactx.ContextSetToken(r.Context(), t)
 	ctx = metadata.Set(ctx, revactx.TokenHeader, t)
 

services/webdav/pkg/service/v0/service.go

@@ -13,6 +13,7 @@ import (
 	gatewayv1beta1 "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
 	userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1"
 	rpcv1beta1 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
+	revactx "github.com/cs3org/reva/v2/pkg/ctx"
 	"github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool"
 	"github.com/cs3org/reva/v2/pkg/storage/utils/templates"
 	"github.com/go-chi/chi/v5"
@@ -37,10 +38,6 @@ func init() {
 	chi.RegisterMethod("REPORT")
 }
 
-const (
-	TokenHeader = "X-Access-Token"
-)
-
 var (
 	codesEnum = map[int]string{
 		http.StatusBadRequest:       "Sabre\\DAV\\Exception\\BadRequest",
@@ -52,8 +49,8 @@ var (
 
 // Service defines the extension handlers.
 type Service interface {
-	ServeHTTP(http.ResponseWriter, *http.Request)
-	Thumbnail(http.ResponseWriter, *http.Request)
+	ServeHTTP(w http.ResponseWriter, r *http.Request)
+	Thumbnail(w http.ResponseWriter, r *http.Request)
 }
 
 // NewService returns a service implementation for Service.
@@ -235,7 +232,7 @@ func (g Webdav) SpacesThumbnail(w http.ResponseWriter, r *http.Request) {
 		renderError(w, r, errBadRequest(err.Error()))
 		return
 	}
-	t := r.Header.Get(TokenHeader)
+	t := r.Header.Get(revactx.TokenHeader)
 
 	fullPath := filepath.Join(tr.Identifier, tr.Filepath)
 	rsp, err := g.thumbnailsClient.GetThumbnail(r.Context(), &thumbnailssvc.GetThumbnailRequest{
@@ -284,7 +281,7 @@ func (g Webdav) Thumbnail(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	t := r.Header.Get(TokenHeader)
+	t := r.Header.Get(revactx.TokenHeader)
 
 	gatewayClient, err := g.gatewaySelector.Next()
 	if err != nil {
@@ -312,7 +309,7 @@ func (g Webdav) Thumbnail(w http.ResponseWriter, r *http.Request) {
 		user = userRes.GetUser()
 	} else {
 		// look up user from URL via GetUserByClaim
-		ctx := grpcmetadata.AppendToOutgoingContext(r.Context(), TokenHeader, t)
+		ctx := grpcmetadata.AppendToOutgoingContext(r.Context(), revactx.TokenHeader, t)
 		userRes, err := gatewayClient.GetUserByClaim(ctx, &userv1beta1.GetUserByClaimRequest{
 			Claim: "username",
 			Value: tr.Identifier,
@@ -475,11 +472,11 @@ func (g Webdav) sendThumbnailResponse(rsp *thumbnailssvc.GetThumbnailResponse, w
 
 	if dlRsp.StatusCode != http.StatusOK {
 		logger.Debug().
-			Str("transfer_token", rsp.TransferToken).
-			Str("data_endpoint", rsp.DataEndpoint).
+			Str("transfer_token", rsp.GetTransferToken()).
+			Str("data_endpoint", rsp.GetDataEndpoint()).
 			Str("response_status", dlRsp.Status).
 			Msg("could not download thumbnail")
-		renderError(w, r, errInternalError("could not download thumbnail"))
+		renderError(w, r, newErrResponse(dlRsp.StatusCode, "could not download thumbnail"))
 		return
 	}