From 7f601658fe01b549d3270c2b390212ac35a4e0be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Mon, 11 Dec 2023 15:36:00 +0100 Subject: [PATCH 1/5] add more upload session filters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- services/storage-users/pkg/command/uploads.go | 145 +++++++++++++++++- 1 file changed, 143 insertions(+), 2 deletions(-) diff --git a/services/storage-users/pkg/command/uploads.go b/services/storage-users/pkg/command/uploads.go index 7f176378589..7eee1d5121f 100644 --- a/services/storage-users/pkg/command/uploads.go +++ b/services/storage-users/pkg/command/uploads.go @@ -1,12 +1,16 @@ package command import ( + "encoding/json" "fmt" "os" + "strings" "sync" + "time" "github.com/urfave/cli/v2" + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" "github.com/cs3org/reva/v2/pkg/storage" "github.com/cs3org/reva/v2/pkg/storage/fs/registry" "github.com/owncloud/ocis/v2/ocis-pkg/config/configlog" @@ -22,6 +26,7 @@ func Uploads(cfg *config.Config) *cli.Command { Usage: "manage unfinished uploads", Subcommands: []*cli.Command{ ListUploads(cfg), + ListUploadSessions(cfg), PurgeExpiredUploads(cfg), }, } @@ -31,7 +36,7 @@ func Uploads(cfg *config.Config) *cli.Command { func ListUploads(cfg *config.Config) *cli.Command { return &cli.Command{ Name: "list", - Usage: "Print a list of all incomplete uploads", + Usage: "Print a list of all incomplete uploads (deprecated, use sessions)", Before: func(c *cli.Context) error { return configlog.ReturnFatal(parser.ParseConfig(cfg)) }, @@ -50,7 +55,7 @@ func ListUploads(cfg *config.Config) *cli.Command { managingFS, ok := fs.(storage.UploadSessionLister) if !ok { - fmt.Fprintf(os.Stderr, "'%s' storage does not support listing expired uploads\n", cfg.Driver) + fmt.Fprintf(os.Stderr, "'%s' storage does not support listing upload sessions\n", cfg.Driver) os.Exit(1) } expired := false @@ -69,6 +74,142 @@ func ListUploads(cfg *config.Config) *cli.Command { } } +// ListUploadSessions prints a list of upload sessiens +func ListUploadSessions(cfg *config.Config) *cli.Command { + return &cli.Command{ + Name: "sessions", + Usage: "Print a list of upload sessions", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "id", + Usage: "filter sessions by upload session id", + }, + &cli.BoolFlag{ + Name: "processing", + DisableDefaultText: true, + Usage: "filter sessions by processing status", + }, + &cli.BoolFlag{ + Name: "expired", + DisableDefaultText: true, + Usage: "filter sessions by expired status", + }, + &cli.StringFlag{ + Name: "output", + Usage: "output format to use (can be 'plain' or 'json', experimental)", + Value: "plain", + DefaultText: "plain", + }, + }, + Before: func(c *cli.Context) error { + return configlog.ReturnFatal(parser.ParseConfig(cfg)) + }, + Action: func(c *cli.Context) error { + f, ok := registry.NewFuncs[cfg.Driver] + if !ok { + fmt.Fprintf(os.Stderr, "Unknown filesystem driver '%s'\n", cfg.Driver) + os.Exit(1) + } + drivers := revaconfig.StorageProviderDrivers(cfg) + fs, err := f(drivers[cfg.Driver].(map[string]interface{}), nil) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to initialize filesystem driver '%s'\n", cfg.Driver) + return err + } + + managingFS, ok := fs.(storage.UploadSessionLister) + if !ok { + fmt.Fprintf(os.Stderr, "'%s' storage does not support listing upload sessions\n", cfg.Driver) + os.Exit(1) + } + var b strings.Builder + filter := storage.UploadSessionFilter{} + if c.IsSet("processing") { + processingValue := c.Bool("processing") + filter.Processing = &processingValue + if !processingValue { + b.WriteString("Not ") + } + if b.Len() == 0 { + b.WriteString("Processing ") + } else { + b.WriteString("processing ") + } + } + if c.IsSet("expired") { + expiredValue := c.Bool("expired") + filter.Expired = &expiredValue + if !expiredValue { + if b.Len() == 0 { + b.WriteString("Not ") + } else { + b.WriteString(", not ") + } + } + if b.Len() == 0 { + b.WriteString("Expired ") + } else { + b.WriteString("expired ") + } + } + if b.Len() == 0 { + b.WriteString("Sessions") + } else { + b.WriteString("sessions") + } + if c.IsSet("id") { + idValue := c.String("id") + filter.ID = &idValue + b.WriteString(" with id '" + idValue + "'") + } + b.WriteString(":") + uploads, err := managingFS.ListUploadSessions(c.Context, filter) + if err != nil { + return err + } + + asJson := c.String("output") == "json" + if !asJson { + fmt.Println(b.String()) + } + for _, u := range uploads { + ref := u.Reference() + if asJson { + s := struct { + ID string `json:"id"` + Space string `json:"space"` + Filename string `json:"filename"` + Offset int64 `json:"offset"` + Size int64 `json:"size"` + Executant userpb.UserId `json:"executant"` + SpaceOwner *userpb.UserId `json:"spaceowner,omitempty"` + Expires time.Time `json:"expires"` + Processing bool `json:"processing"` + }{ + Space: ref.GetResourceId().GetSpaceId(), + ID: u.ID(), + Filename: u.Filename(), + Offset: u.Offset(), + Size: u.Size(), + Executant: u.Executant(), + SpaceOwner: u.SpaceOwner(), + Expires: u.Expires(), + Processing: u.IsProcessing(), + } + j, err := json.Marshal(s) + if err != nil { + fmt.Println(err) + } + fmt.Println(string(j)) + } else { + fmt.Printf(" - %s (Space: %s, Name: %s, Size: %d/%d, Expires: %s, Processing: %t)\n", ref.GetResourceId().GetSpaceId(), u.ID(), u.Filename(), u.Offset(), u.Size(), u.Expires(), u.IsProcessing()) + } + } + return nil + }, + } +} + // PurgeExpiredUploads is the entry point for the server command. func PurgeExpiredUploads(cfg *config.Config) *cli.Command { return &cli.Command{ From cd9082d717b9971d3fe6a54d1e6254dff7d51e88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Wed, 24 Jan 2024 10:35:46 +0100 Subject: [PATCH 2/5] default descriptions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- services/storage-users/pkg/command/uploads.go | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/services/storage-users/pkg/command/uploads.go b/services/storage-users/pkg/command/uploads.go index 7eee1d5121f..d9ef5673e6c 100644 --- a/services/storage-users/pkg/command/uploads.go +++ b/services/storage-users/pkg/command/uploads.go @@ -81,24 +81,24 @@ func ListUploadSessions(cfg *config.Config) *cli.Command { Usage: "Print a list of upload sessions", Flags: []cli.Flag{ &cli.StringFlag{ - Name: "id", - Usage: "filter sessions by upload session id", + Name: "id", + DefaultText: "unset", + Usage: "filter sessions by upload session id", }, &cli.BoolFlag{ - Name: "processing", - DisableDefaultText: true, - Usage: "filter sessions by processing status", + Name: "processing", + DefaultText: "unset", + Usage: "filter sessions by processing status", }, &cli.BoolFlag{ - Name: "expired", - DisableDefaultText: true, - Usage: "filter sessions by expired status", + Name: "expired", + DefaultText: "unset", + Usage: "filter sessions by expired status", }, &cli.StringFlag{ - Name: "output", - Usage: "output format to use (can be 'plain' or 'json', experimental)", - Value: "plain", - DefaultText: "plain", + Name: "output", + Usage: "output format to use (can be 'plain' or 'json', experimental)", + Value: "plain", }, }, Before: func(c *cli.Context) error { From 1811f3a3e3a0878d958d6222acc9b42cbcbca406 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Wed, 24 Jan 2024 15:40:43 +0100 Subject: [PATCH 3/5] update readme MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- services/storage-users/README.md | 32 ++++++++++++++----- services/storage-users/pkg/command/uploads.go | 2 +- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/services/storage-users/README.md b/services/storage-users/README.md index e89cbd30a91..25292f7db3e 100644 --- a/services/storage-users/README.md +++ b/services/storage-users/README.md @@ -36,10 +36,13 @@ When using Infinite Scale as user storage, a directory named `storage/users/uplo Example cases for expired uploads -* When a user uploads a big file but the file exceeds the user-quota, the upload can't be moved to the target after it has finished. The file stays at the upload location until it is manually cleared. +* In the final step the upload blob is moved from the upload area to the final blobstore (e.g. S3). + * If the bandwidth is limited and the file to transfer can't be transferred completely before the upload expiration time is reached, the file expires and can't be processed. -There are two commands available to manage unfinished uploads +The admin can restart the postprocessing for this with the postprocessing cli. + +The storage users service can only list and clean upload sessions: ```bash ocis storage-users uploads @@ -47,21 +50,23 @@ ocis storage-users uploads ```plaintext COMMANDS: - list Print a list of all incomplete uploads - clean Clean up leftovers from expired uploads + sessions Print a list of upload sessions + clean Clean up leftovers from expired uploads + list Print a list of all incomplete uploads (deprecated) ``` #### Command Examples -Command to identify incomplete uploads +Command to list ongoing upload sessions ```bash -ocis storage-users uploads list +ocis storage-users sessions --expired=false ``` ```plaintext -Incomplete uploads: - - 455bd640-cd08-46e8-a5a0-9304908bd40a (file_example_PPT_1MB.ppt, Size: 1028608, Expires: 2022-08-17T12:35:34+02:00) +Expired sessions: + - b921cccc-bf52-478c-aba0-7d213c4cd3a0 (Space: 068ca28f-5c2c-42c3-87ff-f3e1e94dd429, Name: file.txt, Size: 2/3, Expires: 2024-01-23 14:07:05 +0100 CET, Processing: false) + - 4c510ada-c86b-4815-8820-42cdf82c3d51 (Space: 074f9e32-63f0-48e7-bef6-75a37096b2db, Name: logs_cs3org_reva_3695_14_6.log, Size: 275672/275672, Expires: 2023-12-19 11:43:16 +0100 CET, Processing: true) ``` Command to clear expired uploads @@ -74,6 +79,17 @@ Cleaned uploads: - 455bd640-cd08-46e8-a5a0-9304908bd40a (Filename: file_example_PPT_1MB.ppt, Size: 1028608, Expires: 2022-08-17T12:35:34+02:00) ``` +Deprecated list command to identify unfinished uploads + +```bash +ocis storage-users uploads list +``` + +```plaintext +Incomplete uploads: + - 455bd640-cd08-46e8-a5a0-9304908bd40a (file_example_PPT_1MB.ppt, Size: 1028608, Expires: 2022-08-17T12:35:34+02:00) +``` + ### Purge Expired Space Trash-Bins Items diff --git a/services/storage-users/pkg/command/uploads.go b/services/storage-users/pkg/command/uploads.go index d9ef5673e6c..fd864a12142 100644 --- a/services/storage-users/pkg/command/uploads.go +++ b/services/storage-users/pkg/command/uploads.go @@ -210,7 +210,7 @@ func ListUploadSessions(cfg *config.Config) *cli.Command { } } -// PurgeExpiredUploads is the entry point for the server command. +// PurgeExpiredUploads is the entry point for the clean command func PurgeExpiredUploads(cfg *config.Config) *cli.Command { return &cli.Command{ Name: "clean", From 7801112d1dca6a45a53dfc6dc3ec62a4ab353c1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Thu, 25 Jan 2024 13:14:05 +0100 Subject: [PATCH 4/5] use tablewriter, --json flag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- services/storage-users/pkg/command/uploads.go | 47 ++++++++++++++----- 1 file changed, 34 insertions(+), 13 deletions(-) diff --git a/services/storage-users/pkg/command/uploads.go b/services/storage-users/pkg/command/uploads.go index fd864a12142..1c651cc9f15 100644 --- a/services/storage-users/pkg/command/uploads.go +++ b/services/storage-users/pkg/command/uploads.go @@ -4,10 +4,12 @@ import ( "encoding/json" "fmt" "os" + "strconv" "strings" "sync" "time" + tw "github.com/olekukonko/tablewriter" "github.com/urfave/cli/v2" userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" @@ -95,10 +97,9 @@ func ListUploadSessions(cfg *config.Config) *cli.Command { DefaultText: "unset", Usage: "filter sessions by expired status", }, - &cli.StringFlag{ - Name: "output", - Usage: "output format to use (can be 'plain' or 'json', experimental)", - Value: "plain", + &cli.BoolFlag{ + Name: "json", + Usage: "output as json", }, }, Before: func(c *cli.Context) error { @@ -122,6 +123,7 @@ func ListUploadSessions(cfg *config.Config) *cli.Command { fmt.Fprintf(os.Stderr, "'%s' storage does not support listing upload sessions\n", cfg.Driver) os.Exit(1) } + var b strings.Builder filter := storage.UploadSessionFilter{} if c.IsSet("processing") { @@ -168,13 +170,10 @@ func ListUploadSessions(cfg *config.Config) *cli.Command { return err } - asJson := c.String("output") == "json" - if !asJson { - fmt.Println(b.String()) - } - for _, u := range uploads { - ref := u.Reference() - if asJson { + var table *tw.Table + if c.Bool("json") { + for _, u := range uploads { + ref := u.Reference() s := struct { ID string `json:"id"` Space string `json:"space"` @@ -201,9 +200,31 @@ func ListUploadSessions(cfg *config.Config) *cli.Command { fmt.Println(err) } fmt.Println(string(j)) - } else { - fmt.Printf(" - %s (Space: %s, Name: %s, Size: %d/%d, Expires: %s, Processing: %t)\n", ref.GetResourceId().GetSpaceId(), u.ID(), u.Filename(), u.Offset(), u.Size(), u.Expires(), u.IsProcessing()) } + } else { + + // Print what the user requested + fmt.Println(b.String()) + + // start a table + table = tw.NewWriter(os.Stdout) + table.SetHeader([]string{"Space", "Upload Id", "Name", "Offset", "Size", "Executant", "Owner", "Expires", "Processing"}) + table.SetAutoFormatHeaders(false) + + for _, u := range uploads { + table.Append([]string{ + u.Reference().ResourceId.GetSpaceId(), + u.ID(), + u.Filename(), + strconv.FormatInt(u.Offset(), 10), + strconv.FormatInt(u.Size(), 10), + u.Executant().OpaqueId, + u.SpaceOwner().GetOpaqueId(), + u.Expires().Format(time.RFC3339), + strconv.FormatBool(u.IsProcessing()), + }) + } + table.Render() } return nil }, From 49733eda4971517a190532d23ef51d82bfe58d8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Friedrich=20Dreyer?= Date: Thu, 25 Jan 2024 13:21:00 +0100 Subject: [PATCH 5/5] update readme MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jörn Friedrich Dreyer --- services/storage-users/README.md | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/services/storage-users/README.md b/services/storage-users/README.md index 25292f7db3e..bee130106f7 100644 --- a/services/storage-users/README.md +++ b/services/storage-users/README.md @@ -64,9 +64,24 @@ ocis storage-users sessions --expired=false ``` ```plaintext -Expired sessions: - - b921cccc-bf52-478c-aba0-7d213c4cd3a0 (Space: 068ca28f-5c2c-42c3-87ff-f3e1e94dd429, Name: file.txt, Size: 2/3, Expires: 2024-01-23 14:07:05 +0100 CET, Processing: false) - - 4c510ada-c86b-4815-8820-42cdf82c3d51 (Space: 074f9e32-63f0-48e7-bef6-75a37096b2db, Name: logs_cs3org_reva_3695_14_6.log, Size: 275672/275672, Expires: 2023-12-19 11:43:16 +0100 CET, Processing: true) +Not expired sessions: ++--------------------------------------+--------------------------------------+---------+--------+------+--------------------------------------+--------------------------------------+---------------------------+------------+ +| Space | Upload Id | Name | Offset | Size | Executant | Owner | Expires | Processing | ++--------------------------------------+--------------------------------------+---------+--------+------+--------------------------------------+--------------------------------------+---------------------------+------------+ +| f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c | 5e387954-7313-4223-a904-bf996da6ec0b | foo.txt | 0 | 1234 | f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c | f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c | 2024-01-26T13:04:31+01:00 | false | +| f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c | f066244d-97b2-48e7-a30d-b40fcb60cec6 | bar.txt | 0 | 4321 | f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c | f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c | 2024-01-26T13:18:47+01:00 | false | ++--------------------------------------+--------------------------------------+---------+--------+------+--------------------------------------+--------------------------------------+---------------------------+------------+ +``` + +The sessions command can also output json + +```bash +ocis storage-users sessions --expired=false --json +``` + +```json +{"id":"5e387954-7313-4223-a904-bf996da6ec0b","space":"f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c","filename":"foo.txt","offset":0,"size":1234,"executant":{"idp":"https://cloud.ocis.test","opaque_id":"f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"},"spaceowner":{"opaque_id":"f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"},"expires":"2024-01-26T13:04:31+01:00","processing":false} +{"id":"f066244d-97b2-48e7-a30d-b40fcb60cec6","space":"f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c","filename":"bar.txt","offset":0,"size":4321,"executant":{"idp":"https://cloud.ocis.test","opaque_id":"f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"},"spaceowner":{"opaque_id":"f7fbf8c8-139b-4376-b307-cf0a8c2d0d9c"},"expires":"2024-01-26T13:18:47+01:00","processing":false} ``` Command to clear expired uploads