From 86a6465157ee1d662c8b37d8ee235367bab72d4a Mon Sep 17 00:00:00 2001 From: Roman Perekhod Date: Tue, 12 Dec 2023 15:54:22 +0100 Subject: [PATCH 1/4] trash-bin cli has been exteneded by the list and restore commands --- changelog/unreleased/add-trach-bin-cli.md | 6 + services/gateway/pkg/config/config.go | 2 +- services/storage-users/README.md | 55 ++- .../storage-users/pkg/command/trash_bin.go | 454 ++++++++++++++++++ .../pkg/command/trash_bin_test.go | 65 +++ services/storage-users/pkg/config/config.go | 15 +- .../pkg/config/defaults/defaultconfig.go | 1 + 7 files changed, 587 insertions(+), 11 deletions(-) create mode 100644 changelog/unreleased/add-trach-bin-cli.md create mode 100644 services/storage-users/pkg/command/trash_bin_test.go diff --git a/changelog/unreleased/add-trach-bin-cli.md b/changelog/unreleased/add-trach-bin-cli.md new file mode 100644 index 00000000000..91485a28f0f --- /dev/null +++ b/changelog/unreleased/add-trach-bin-cli.md @@ -0,0 +1,6 @@ +Enhancement: Add cli commands for trash-binq + +We added the `list` and `restore` commands to the trash-bin items to the CLI + +https://github.com/owncloud/ocis/pull/7936 +https://github.com/owncloud/ocis/issues/7845 diff --git a/services/gateway/pkg/config/config.go b/services/gateway/pkg/config/config.go index 73c6a01ee4a..f90d38b7acf 100644 --- a/services/gateway/pkg/config/config.go +++ b/services/gateway/pkg/config/config.go @@ -70,7 +70,7 @@ type Debug struct { } type GRPCConfig struct { - Addr string `yaml:"addr" env:"GATEWAY_GRPC_ADDR" desc:"The bind address of the GRPC service."` + Addr string `yaml:"addr" env:"OCIS_GATEWAY_GRPC_ADDR;GATEWAY_GRPC_ADDR" desc:"The bind address of the GRPC service."` TLS *shared.GRPCServiceTLS `yaml:"tls"` Namespace string `yaml:"-"` Protocol string `yaml:"protocol" env:"GATEWAY_GRPC_PROTOCOL" desc:"The transport protocol of the GRPC service."` diff --git a/services/storage-users/README.md b/services/storage-users/README.md index 1ceb861c580..e6e4ead0435 100644 --- a/services/storage-users/README.md +++ b/services/storage-users/README.md @@ -10,13 +10,13 @@ Starting with ocis version 3.0.0, the default backend for metadata switched to m Starting with Infinite Scale version 3.1, you can define a graceful shutdown period for the `storage-users` service. -IMPORTANT: The graceful shutdown period is only applicable if the `storage-users` service runs as standalone service. It does not apply if the `storage-users` service runs as part of the single binary or as single Docker environment. To build an environment where the `storage-users` service runs as a standalone service, you must start two instances, one _without_ the `storage-users` service and one _only with_ the the `storage-users` service. Note that both instances must be able to communicate on the same network. +IMPORTANT: The graceful shutdown period is only applicable if the `storage-users` service runs as standalone service. It does not apply if the `storage-users` service runs as part of the single binary or as single Docker environment. To build an environment where the `storage-users` service runs as a standalone service, you must start two instances, one _without_ the `storage-users` service and one _only with_ the the `storage-users` service. Note that both instances must be able to communicate on the same network. When hard-stopping Infinite Scale, for example with the `kill ` command (SIGKILL), it is possible and likely that not all data from the decomposedfs (metadata) has been written to the storage which may result in an inconsistent decomposedfs. When gracefully shutting down Infinite Scale, using a command like SIGTERM, the process will no longer accept any write requests from _other_ services and will try to write the internal open requests which can take an undefined duration based on many factors. To mitigate that situation, the following things have been implemented: * With the value of the environment variable `STORAGE_USERS_GRACEFUL_SHUTDOWN_TIMEOUT`, the `storage-users` service will delay its shutdown giving it time to finalize writing necessary data. This delay can be necessary if there is a lot of data to be saved and/or if storage access/thruput is slow. In such a case you would receive an error log entry informing you that not all data could be saved in time. To prevent such occurrences, you must increase the default value. -* If a shutdown error has been logged, the command-line maintenance tool [Inspect and Manipulate Node Metadata](https://doc.owncloud.com/ocis/next/maintenance/commands/commands.html#inspect-and-manipulate-node-metadata) can help to fix the issue. Please contact support for details. +* If a shutdown error has been logged, the command-line maintenance tool [Inspect and Manipulate Node Metadata](https://doc.owncloud.com/ocis/next/maintenance/commands/commands.html#inspect-and-manipulate-node-metadata) can help to fix the issue. Please contact support for details. ## CLI Commands @@ -37,7 +37,7 @@ When using Infinite Scale as user storage, a directory named `storage/users/uplo Example cases for expired uploads * When a user uploads a big file but the file exceeds the user-quota, the upload can't be moved to the target after it has finished. The file stays at the upload location until it is manually cleared. -* If the bandwidth is limited and the file to transfer can't be transferred completely before the upload expiration time is reached, the file expires and can't be processed. +* If the bandwidth is limited and the file to transfer can't be transferred completely before the upload expiration time is reached, the file expires and can't be processed. There are two commands available to manage unfinished uploads @@ -78,12 +78,13 @@ Cleaned uploads: -This command is about purging old trash-bin items of `project` spaces (spaces that have been created manually) and `personal` spaces. +This command is about the trash-bin to get an overview of items, restore items and purging old items of `project` spaces (spaces that have been created manually) and `personal` spaces. ```bash ocis storage-users trash-bin ``` +#### Purge-expired ```plaintext COMMANDS: purge-expired Purge all expired items from the trashbin @@ -97,6 +98,52 @@ The configuration for the `purge-expired` command is done by using the following * `STORAGE_USERS_PURGE_TRASH_BIN_PROJECT_DELETE_BEFORE` has a default value of `30 days`, which means the command will delete all files older than `30 days`. The value is human-readable, valid values are `24h`, `60m`, `60s` etc. `0` is equivalent to disable and prevents the deletion of `project space` trash-bin files. +#### List and Restore Trash-Bins Items + +To authenticate the cli command use `OCIS_MACHINE_AUTH_API_KEY=`. The `storage-users` cli tool uses the default address to establish the connection to the `gateway` service. If the connection is failed check your custom `gateway` +service `GATEWAY_GRPC_ADDR` configuration and set the same address to `storage-users` variable `OCIS_GATEWAY_GRPC_ADDR` or `STORAGE_USERS_GATEWAY_GRPC_ADDR`. + +The ID sources: +- 'userID' in a `https://{host}/graph/v1.0/me` +- personal 'spaceID' in a `https://{host}/graph/v1.0/me/drives?$filter=driveType+eq+personal` +- project 'spaceID' in a `https://{host}/graph/v1.0/me/drives?$filter=driveType+eq+project` + +```bash +NAME: + ocis storage-users trash-bin list - Print a list of all trash-bin items for a space. + +USAGE: + ocis storage-users trash-bin list command [command options] ['userID' required] ['spaceID' required] +``` + +```bash +NAME: + ocis storage-users trash-bin restore-all - Restore all trash-bin items for a space. + +USAGE: + ocis storage-users trash-bin restore-all command [command options] ['userID' required] ['spaceID' required] + +COMMANDS: + help, h Shows a list of commands or help for one command + +OPTIONS: + --option value, -o value The restore option defines the behavior for a file to be restored, where the file name already already exists in the target space. Supported values are: 'skip', 'replace' and 'keep-both'. The default value is 'skip' overwriting an existing file. +``` + +```bash +NAME: + ocis storage-users trash-bin restore - Restore a trash-bin item by ID. + +USAGE: + ocis storage-users trash-bin restore command [command options] ['userID' required] ['spaceID' required] ['itemID' required] + +COMMANDS: + help, h Shows a list of commands or help for one command + +OPTIONS: + --option value, -o value The restore option defines the behavior for a file to be restored, where the file name already already exists in the target space. Supported values are: 'skip', 'replace' and 'keep-both'. The default value is 'skip' overwriting an existing file. +``` + ## Caching The `storage-users` service caches stat, metadata and uuids of files and folders via the configured store in `STORAGE_USERS_STAT_CACHE_STORE`, `STORAGE_USERS_FILEMETADATA_CACHE_STORE` and `STORAGE_USERS_ID_CACHE_STORE`. Possible stores are: diff --git a/services/storage-users/pkg/command/trash_bin.go b/services/storage-users/pkg/command/trash_bin.go index 8489ff90bc2..fd02d304521 100644 --- a/services/storage-users/pkg/command/trash_bin.go +++ b/services/storage-users/pkg/command/trash_bin.go @@ -1,16 +1,50 @@ package command import ( + "context" + "fmt" + "path" + "path/filepath" + "strings" "time" + gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" + userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/v2/pkg/events" + "github.com/cs3org/reva/v2/pkg/rgrpc/todo/pool" + "github.com/cs3org/reva/v2/pkg/storagespace" + "github.com/cs3org/reva/v2/pkg/utils" + "github.com/mohae/deepcopy" "github.com/owncloud/ocis/v2/ocis-pkg/config/configlog" + "github.com/owncloud/ocis/v2/ocis-pkg/tracing" "github.com/owncloud/ocis/v2/services/storage-users/pkg/config" "github.com/owncloud/ocis/v2/services/storage-users/pkg/config/parser" "github.com/owncloud/ocis/v2/services/storage-users/pkg/event" + "github.com/owncloud/ocis/v2/services/storage-users/pkg/logging" + "github.com/rs/zerolog/log" "github.com/urfave/cli/v2" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) +const ( + SKIP = iota + REPLACE + KEEP_BOTH + + retrievingErrorMsg = "trash-bin items retrieving error" +) + +var optionFlagTmpl = cli.StringFlag{ + Name: "option", + Value: "skip", + Aliases: []string{"o"}, + Usage: "The restore option defines the behavior for a file to be restored, where the file name already already exists in the target space. Supported values are: 'skip', 'replace' and 'keep-both'.", + DefaultText: "The default value is 'skip' overwriting an existing file", +} + // TrashBin wraps trash-bin related sub-commands. func TrashBin(cfg *config.Config) *cli.Command { return &cli.Command{ @@ -18,6 +52,9 @@ func TrashBin(cfg *config.Config) *cli.Command { Usage: "manage trash-bin's", Subcommands: []*cli.Command{ PurgeExpiredResources(cfg), + listTrashBinItems(cfg), + restoreAllTrashBinItems(cfg), + restoreTrashBindItem(cfg), }, } } @@ -53,3 +90,420 @@ func PurgeExpiredResources(cfg *config.Config) *cli.Command { }, } } + +func listTrashBinItems(cfg *config.Config) *cli.Command { + return &cli.Command{ + Name: "list", + Usage: "Print a list of all trash-bin items of a space.", + ArgsUsage: "['userID' required] ['spaceID' required]", + Flags: []cli.Flag{}, + Before: func(c *cli.Context) error { + return configlog.ReturnFatal(parser.ParseConfig(cfg)) + }, + Action: func(c *cli.Context) error { + log := logging.Configure(cfg.Service.Name, cfg.Log) + tp, err := tracing.GetServiceTraceProvider(cfg.Tracing, cfg.Service.Name) + if err != nil { + return err + } + var userID, spaceID string + if c.NArg() > 1 { + userID = c.Args().Get(0) + spaceID = c.Args().Get(1) + } + if userID == "" { + _ = cli.ShowSubcommandHelp(c) + return fmt.Errorf("userID is requered") + } + if spaceID == "" { + _ = cli.ShowSubcommandHelp(c) + return fmt.Errorf("spaceID is requered") + } + fmt.Printf("Getting trash-bin items for spaceID: '%s' ...\n", spaceID) + + ref, err := storagespace.ParseReference(spaceID) + if err != nil { + return err + } + client, err := pool.GetGatewayServiceClient(cfg.RevaGatewayGRPCAddr) + if err != nil { + log.Error().Err(err).Msg("error selecting next gateway client") + return err + } + ctx, _, err := utils.Impersonate(&userv1beta1.UserId{OpaqueId: userID}, client, cfg.MachineAuthAPIKey) + if err != nil { + log.Error().Err(err).Msg("could not impersonate") + return err + } + + spanOpts := []trace.SpanStartOption{ + trace.WithSpanKind(trace.SpanKindClient), + trace.WithAttributes( + attribute.KeyValue{Key: "userID", Value: attribute.StringValue(userID)}, + attribute.KeyValue{Key: "spaceID", Value: attribute.StringValue(spaceID)}, + ), + } + ctx, span := tp.Tracer("storage-users trash-bin list").Start(ctx, "serve static asset", spanOpts...) + defer span.End() + + res, err := client.ListRecycle(ctx, &provider.ListRecycleRequest{Ref: &ref, Key: "/"}) + if err != nil { + log.Error().Err(err).Msg(retrievingErrorMsg) + return err + } + if res.Status.Code != rpc.Code_CODE_OK { + return fmt.Errorf("%s %s", retrievingErrorMsg, res.Status.Code) + } + + if len(res.GetRecycleItems()) > 0 { + fmt.Println("The list of the trash-bin items. Use an itemID to restore.") + } else { + fmt.Println("The list is empty.") + } + + for _, item := range res.GetRecycleItems() { + fmt.Printf("itemID: '%s', path: '%s', type: '%s', delited at :%s\n", item.GetKey(), item.GetRef().GetPath(), itemType(item.GetType()), utils.TSToTime(item.GetDeletionTime()).UTC().Format(time.RFC3339)) + } + return nil + }, + } +} + +func restoreAllTrashBinItems(cfg *config.Config) *cli.Command { + var optionFlagVal string + var overwriteOption int + optionFlag := optionFlagTmpl + optionFlag.Destination = &optionFlagVal + return &cli.Command{ + Name: "restore-all", + Usage: "Restore all trash-bin items for a space.", + ArgsUsage: "['userID' required] ['spaceID' required]", + Flags: []cli.Flag{ + &optionFlag, + }, + Before: func(c *cli.Context) error { + return configlog.ReturnFatal(parser.ParseConfig(cfg)) + }, + Action: func(c *cli.Context) error { + log := logging.Configure(cfg.Service.Name, cfg.Log) + tp, err := tracing.GetServiceTraceProvider(cfg.Tracing, cfg.Service.Name) + if err != nil { + return err + } + c.Lineage() + var userID, spaceID string + if c.NArg() > 1 { + userID = c.Args().Get(0) + spaceID = c.Args().Get(1) + } + if userID == "" { + _ = cli.ShowSubcommandHelp(c) + return cli.Exit("The userID is required", 1) + } + if spaceID == "" { + _ = cli.ShowSubcommandHelp(c) + return cli.Exit("The spaceID is required", 1) + } + switch optionFlagVal { + case "skip": + overwriteOption = SKIP + case "replace": + overwriteOption = REPLACE + case "keep-both": + overwriteOption = KEEP_BOTH + default: + _ = cli.ShowSubcommandHelp(c) + return cli.Exit("The option flag is invalid", 1) + } + fmt.Printf("Restoring trash-bin items for spaceID: '%s' ...\n", spaceID) + ref, err := storagespace.ParseReference(spaceID) + if err != nil { + return err + } + client, err := pool.GetGatewayServiceClient(cfg.RevaGatewayGRPCAddr) + if err != nil { + log.Error().Err(err).Msg("error selecting next gateway client") + return err + } + ctx, _, err := utils.Impersonate(&userv1beta1.UserId{OpaqueId: userID}, client, cfg.MachineAuthAPIKey) + if err != nil { + log.Error().Err(err).Msg("could not impersonate") + return err + } + + spanOpts := []trace.SpanStartOption{ + trace.WithSpanKind(trace.SpanKindClient), + trace.WithAttributes( + attribute.KeyValue{Key: "option", Value: attribute.StringValue(optionFlagVal)}, + attribute.KeyValue{Key: "userID", Value: attribute.StringValue(userID)}, + attribute.KeyValue{Key: "spaceID", Value: attribute.StringValue(spaceID)}, + ), + } + ctx, span := tp.Tracer("storage-users trash-bin restore-all").Start(ctx, "serve static asset", spanOpts...) + defer span.End() + + res, err := client.ListRecycle(ctx, &provider.ListRecycleRequest{Ref: &ref, Key: "/"}) + if err != nil { + log.Error().Err(err).Msg(retrievingErrorMsg) + return err + } + if res.Status.Code != rpc.Code_CODE_OK { + return fmt.Errorf("%s %s", retrievingErrorMsg, res.Status.Code) + } + if len(res.GetRecycleItems()) == 0 { + return cli.Exit("The trash-bin is empty. Nothing to restore", 0) + } + + for { + fmt.Printf("Foud %d items that could be restored, continue (Y/n), show the items list (s): ", len(res.GetRecycleItems())) + var i string + _, err := fmt.Scanf("%s", &i) + if err != nil { + log.Err(err).Send() + continue + } + if strings.ToLower(i) == "y" { + break + } else if strings.ToLower(i) == "n" { + return nil + } else if strings.ToLower(i) == "s" { + for _, item := range res.GetRecycleItems() { + fmt.Printf("itemID: '%s', path: '%s', type: '%s', delited at: %s\n", item.GetKey(), item.GetRef().GetPath(), itemType(item.GetType()), utils.TSToTime(item.GetDeletionTime()).UTC().Format(time.RFC3339)) + } + } + } + + fmt.Printf("\nRun restoring-all with option=%s\n", optionFlagVal) + for _, item := range res.GetRecycleItems() { + fmt.Printf("restoring itemID: '%s', path: '%s', type: '%s'\n", item.GetKey(), item.GetRef().GetPath(), itemType(item.GetType())) + dstRes, err := restore(ctx, client, ref, item, overwriteOption) + if err != nil { + return err + } + fmt.Printf("itemID: '%s', path: '%s', restored as '%s'\n", item.GetKey(), item.GetRef().GetPath(), dstRes.GetPath()) + } + return nil + }, + } +} + +func restoreTrashBindItem(cfg *config.Config) *cli.Command { + var optionFlagVal string + var overwriteOption int + optionFlag := optionFlagTmpl + optionFlag.Destination = &optionFlagVal + return &cli.Command{ + Name: "restore", + Usage: "Restore a trash-bin item by ID.", + ArgsUsage: "['userId' required] ['spaceID' required] ['itemID' required]", + Flags: []cli.Flag{ + &optionFlag, + }, + Before: func(c *cli.Context) error { + return configlog.ReturnFatal(parser.ParseConfig(cfg)) + }, + Action: func(c *cli.Context) error { + log := logging.Configure(cfg.Service.Name, cfg.Log) + tp, err := tracing.GetServiceTraceProvider(cfg.Tracing, cfg.Service.Name) + if err != nil { + return err + } + c.Lineage() + var userID, spaceID, itemID string + if c.NArg() > 2 { + userID = c.Args().Get(0) + spaceID = c.Args().Get(1) + itemID = c.Args().Get(2) + } + if userID == "" { + _ = cli.ShowSubcommandHelp(c) + return fmt.Errorf("userID is requered") + } + if spaceID == "" { + _ = cli.ShowSubcommandHelp(c) + return fmt.Errorf("spaceID is requered") + } + if itemID == "" { + _ = cli.ShowSubcommandHelp(c) + return fmt.Errorf("itemID is requered") + } + switch optionFlagVal { + case "skip": + overwriteOption = SKIP + case "replace": + overwriteOption = REPLACE + case "keep-both": + overwriteOption = KEEP_BOTH + default: + _ = cli.ShowSubcommandHelp(c) + return cli.Exit("The option flag is invalid", 1) + } + + ref, err := storagespace.ParseReference(spaceID) + if err != nil { + return err + } + client, err := pool.GetGatewayServiceClient(cfg.RevaGatewayGRPCAddr) + if err != nil { + log.Error().Err(err).Msg("error selecting gateway client") + return err + } + ctx, _, err := utils.Impersonate(&userv1beta1.UserId{OpaqueId: userID}, client, cfg.MachineAuthAPIKey) + if err != nil { + log.Error().Err(err).Msg("could not impersonate") + return err + } + + spanOpts := []trace.SpanStartOption{ + trace.WithSpanKind(trace.SpanKindClient), + trace.WithAttributes( + attribute.KeyValue{Key: "option", Value: attribute.StringValue(optionFlagVal)}, + attribute.KeyValue{Key: "userID", Value: attribute.StringValue(userID)}, + attribute.KeyValue{Key: "spaceID", Value: attribute.StringValue(spaceID)}, + attribute.KeyValue{Key: "itemID", Value: attribute.StringValue(itemID)}, + ), + } + ctx, span := tp.Tracer("storage-users trash-bin restore").Start(ctx, "serve static asset", spanOpts...) + defer span.End() + + res, err := client.ListRecycle(ctx, &provider.ListRecycleRequest{Ref: &ref, Key: "/"}) + if err != nil { + log.Error().Err(err).Msg(retrievingErrorMsg) + return err + } + if res.Status.Code != rpc.Code_CODE_OK { + return fmt.Errorf("%s %s", retrievingErrorMsg, res.Status.Code) + } + + var found bool + var itemRef *provider.RecycleItem + for _, item := range res.GetRecycleItems() { + if item.GetKey() == itemID { + itemRef = item + found = true + break + } + } + if !found { + return fmt.Errorf("itemID '%s' not found", itemID) + } + fmt.Printf("\nRun restoring with option=%s\n", optionFlagVal) + fmt.Printf("restoring itemID: '%s', path: '%s', type: '%s'\n", itemRef.GetKey(), itemRef.GetRef().GetPath(), itemType(itemRef.GetType())) + dstRes, err := restore(ctx, client, ref, itemRef, overwriteOption) + if err != nil { + return err + } + fmt.Printf("itemID: '%s', path: '%s', restored as '%s'\n", itemRef.GetKey(), itemRef.GetRef().GetPath(), dstRes.GetPath()) + return nil + }, + } +} + +func restore(ctx context.Context, client gateway.GatewayAPIClient, ref provider.Reference, item *provider.RecycleItem, overwriteOption int) (*provider.Reference, error) { + dst, _ := deepcopy.Copy(ref).(provider.Reference) + dst.Path = utils.MakeRelativePath(item.GetRef().GetPath()) + // Restore request + req := &provider.RestoreRecycleItemRequest{ + Ref: &ref, + Key: path.Join(item.GetKey(), "/"), + RestoreRef: &dst, + } + + exists, dstStatRes, err := isDestinationExists(ctx, client, dst) + if err != nil { + return &dst, err + } + + if exists { + fmt.Printf("destination '%s' exists.\n", dstStatRes.GetInfo().GetPath()) + switch overwriteOption { + case SKIP: + return &dst, nil + case REPLACE: + // delete existing tree + delReq := &provider.DeleteRequest{Ref: &dst} + delRes, err := client.Delete(ctx, delReq) + if err != nil { + return &dst, fmt.Errorf("error sending grpc delete request %w", err) + } + if delRes.Status.Code != rpc.Code_CODE_OK && delRes.Status.Code != rpc.Code_CODE_NOT_FOUND { + return &dst, fmt.Errorf("deleting error %w", err) + } + case KEEP_BOTH: + // modify the file name + req.RestoreRef, err = resolveDestination(ctx, client, dst) + if err != nil { + return &dst, fmt.Errorf("trash-bin item restoring error %w", err) + } + } + } + + res, err := client.RestoreRecycleItem(ctx, req) + if err != nil { + log.Error().Err(err).Msg("trash-bin item restoring error") + return req.RestoreRef, err + } + if res.Status.Code != rpc.Code_CODE_OK { + return req.RestoreRef, fmt.Errorf("trash-bin item restoring error %s", res.Status.Code) + } + return req.RestoreRef, nil +} + +func resolveDestination(ctx context.Context, client gateway.GatewayAPIClient, dstRef provider.Reference) (*provider.Reference, error) { + dst := dstRef + for i := 1; i < 100; i++ { + dst.Path = modifyFilename(dstRef.Path, i) + exists, _, err := isDestinationExists(ctx, client, dst) + if err != nil { + return nil, err + } + if exists { + continue + } + return &dst, nil + } + return nil, fmt.Errorf("too many attempts to resolve the destination") +} + +func isDestinationExists(ctx context.Context, client gateway.GatewayAPIClient, dst provider.Reference) (bool, *provider.StatResponse, error) { + dstStatReq := &provider.StatRequest{Ref: &dst} + dstStatRes, err := client.Stat(ctx, dstStatReq) + if err != nil { + return false, nil, fmt.Errorf("error sending grpc stat request %w", err) + } + if dstStatRes.GetStatus().GetCode() == rpc.Code_CODE_OK { + return true, dstStatRes, nil + } + if dstStatRes.GetStatus().GetCode() == rpc.Code_CODE_NOT_FOUND { + return false, dstStatRes, nil + } + return false, dstStatRes, fmt.Errorf("stat request failed %s", dstStatRes.GetStatus()) +} + +// modify the file name like UI do +func modifyFilename(filename string, mod int) string { + var extension string + var found bool + expected := []string{".tar.gz", ".tar.bz", ".tar.bz2"} + for _, s := range expected { + var prefix string + prefix, found = strings.CutSuffix(strings.ToLower(filename), s) + if found { + extension = strings.TrimPrefix(filename, prefix) + break + } + } + if !found { + extension = filepath.Ext(filename) + } + name := filename[0 : len(filename)-len(extension)] + return fmt.Sprintf("%s (%d)%s", name, mod, extension) +} + +func itemType(it provider.ResourceType) string { + var itemType = "file" + if it == provider.ResourceType_RESOURCE_TYPE_CONTAINER { + itemType = "folder" + } + return itemType +} diff --git a/services/storage-users/pkg/command/trash_bin_test.go b/services/storage-users/pkg/command/trash_bin_test.go new file mode 100644 index 00000000000..26cfd4bc156 --- /dev/null +++ b/services/storage-users/pkg/command/trash_bin_test.go @@ -0,0 +1,65 @@ +package command + +import ( + "testing" +) + +func Test_modifyFilename(t *testing.T) { + type args struct { + filename string + mod int + } + tests := []struct { + name string + args args + want string + }{ + { + name: "file", + args: args{filename: "file.txt", mod: 1}, + want: "file (1).txt", + }, + { + name: "file with path", + args: args{filename: "./file.txt", mod: 1}, + want: "./file (1).txt", + }, + { + name: "file with path 2", + args: args{filename: "./subdir/file.tar.gz", mod: 99}, + want: "./subdir/file (99).tar.gz", + }, + { + name: "file with path 3", + args: args{filename: "./sub dir/new file.tar.gz", mod: 99}, + want: "./sub dir/new file (99).tar.gz", + }, + { + name: "file without ext", + args: args{filename: "./subdir/file", mod: 2}, + want: "./subdir/file (2)", + }, + { + name: "file without ext 2", + args: args{filename: "./subdir/file 1", mod: 2}, + want: "./subdir/file 1 (2)", + }, + { + name: "file with emoji", + args: args{filename: "./subdir/file 🙂.tar.gz", mod: 3}, + want: "./subdir/file 🙂 (3).tar.gz", + }, + { + name: "file with emoji 2", + args: args{filename: "./subdir/file 🙂", mod: 2}, + want: "./subdir/file 🙂 (2)", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := modifyFilename(tt.args.filename, tt.args.mod); got != tt.want { + t.Errorf("modifyFilename() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/services/storage-users/pkg/config/config.go b/services/storage-users/pkg/config/config.go index 4070394e98c..65c55368d81 100644 --- a/services/storage-users/pkg/config/config.go +++ b/services/storage-users/pkg/config/config.go @@ -18,16 +18,19 @@ type Config struct { GRPC GRPCConfig `yaml:"grpc"` HTTP HTTPConfig `yaml:"http"` - TokenManager *TokenManager `yaml:"token_manager"` - Reva *shared.Reva `yaml:"reva"` + TokenManager *TokenManager `yaml:"token_manager"` + Reva *shared.Reva `yaml:"reva"` + RevaGatewayGRPCAddr string `yaml:"gateway_addr" env:"OCIS_GATEWAY_GRPC_ADDR;STORAGE_USERS_GATEWAY_GRPC_ADDR" desc:"The bind address of the gateway GRPC address."` + MachineAuthAPIKey string `yaml:"machine_auth_api_key" env:"OCIS_MACHINE_AUTH_API_KEY" desc:"Machine auth API key used to validate internal requests necessary for the access to resources from other services."` SkipUserGroupsInToken bool `yaml:"skip_user_groups_in_token" env:"STORAGE_USERS_SKIP_USER_GROUPS_IN_TOKEN" desc:"Disables the loading of user's group memberships from the reva access token."` GracefulShutdownTimeout int `yaml:"graceful_shutdown_timeout" env:"STORAGE_USERS_GRACEFUL_SHUTDOWN_TIMEOUT" desc:"The number of seconds to wait for the 'storage-users' service to shutdown cleanly before exiting with an error that gets logged. Note: This setting is only applicable when running the 'storage-users' service as a standalone service. See the text description for more details."` - Driver string `yaml:"driver" env:"STORAGE_USERS_DRIVER" desc:"The storage driver which should be used by the service. Defaults to 'ocis', Supported values are: 'ocis', 's3ng' and 'owncloudsql'. The 'ocis' driver stores all data (blob and meta data) in an POSIX compliant volume. The 's3ng' driver stores metadata in a POSIX compliant volume and uploads blobs to the s3 bucket."` - Drivers Drivers `yaml:"drivers"` - DataServerURL string `yaml:"data_server_url" env:"STORAGE_USERS_DATA_SERVER_URL" desc:"URL of the data server, needs to be reachable by the data gateway provided by the frontend service or the user if directly exposed."` - DataGatewayURL string `yaml:"data_gateway_url" env:"STORAGE_USERS_DATA_GATEWAY_URL" desc:"URL of the data gateway server"` + Driver string `yaml:"driver" env:"STORAGE_USERS_DRIVER" desc:"The storage driver which should be used by the service. Defaults to 'ocis', Supported values are: 'ocis', 's3ng' and 'owncloudsql'. The 'ocis' driver stores all data (blob and meta data) in an POSIX compliant volume. The 's3ng' driver stores metadata in a POSIX compliant volume and uploads blobs to the s3 bucket."` + Drivers Drivers `yaml:"drivers"` + DataServerURL string `yaml:"data_server_url" env:"STORAGE_USERS_DATA_SERVER_URL" desc:"URL of the data server, needs to be reachable by the data gateway provided by the frontend service or the user if directly exposed."` + DataGatewayURL string `yaml:"data_gateway_url" env:"STORAGE_USERS_DATA_GATEWAY_URL" desc:"URL of the data gateway server"` + TransferExpires int64 `yaml:"transfer_expires" env:"STORAGE_USERS_TRANSFER_EXPIRES" desc:"the time after which the token for upload postprocessing expires"` Events Events `yaml:"events"` StatCache StatCache `yaml:"stat_cache"` diff --git a/services/storage-users/pkg/config/defaults/defaultconfig.go b/services/storage-users/pkg/config/defaults/defaultconfig.go index 1bb34eab579..68c649b8758 100644 --- a/services/storage-users/pkg/config/defaults/defaultconfig.go +++ b/services/storage-users/pkg/config/defaults/defaultconfig.go @@ -44,6 +44,7 @@ func DefaultConfig() *config.Config { Reva: shared.DefaultRevaConfig(), DataServerURL: "http://localhost:9158/data", DataGatewayURL: "https://localhost:9200/data", + RevaGatewayGRPCAddr: "127.0.0.1:9142", TransferExpires: 86400, UploadExpiration: 24 * 60 * 60, GracefulShutdownTimeout: 30, From 656da3bdd0a965a9be36d88ab7c5ab96f209da6f Mon Sep 17 00:00:00 2001 From: Roman Perekhod Date: Tue, 12 Dec 2023 17:31:55 +0100 Subject: [PATCH 2/4] v4 to v5 changes --- changelog/unreleased/add-trach-bin-cli.md | 3 +- services/storage-users/README.md | 9 ++-- .../storage-users/pkg/command/trash_bin.go | 51 ++++++------------- 3 files changed, 22 insertions(+), 41 deletions(-) diff --git a/changelog/unreleased/add-trach-bin-cli.md b/changelog/unreleased/add-trach-bin-cli.md index 91485a28f0f..828180ab027 100644 --- a/changelog/unreleased/add-trach-bin-cli.md +++ b/changelog/unreleased/add-trach-bin-cli.md @@ -2,5 +2,6 @@ Enhancement: Add cli commands for trash-binq We added the `list` and `restore` commands to the trash-bin items to the CLI -https://github.com/owncloud/ocis/pull/7936 +https://github.com/owncloud/ocis/pull/7917 +https://github.com/cs3org/reva/pull/4392 https://github.com/owncloud/ocis/issues/7845 diff --git a/services/storage-users/README.md b/services/storage-users/README.md index e6e4ead0435..56c5da2b862 100644 --- a/services/storage-users/README.md +++ b/services/storage-users/README.md @@ -100,11 +100,10 @@ The configuration for the `purge-expired` command is done by using the following #### List and Restore Trash-Bins Items -To authenticate the cli command use `OCIS_MACHINE_AUTH_API_KEY=`. The `storage-users` cli tool uses the default address to establish the connection to the `gateway` service. If the connection is failed check your custom `gateway` +To authenticate the cli command use `OCIS_SERVICE_ACCOUNT_SECRET=` and `OCIS_SERVICE_ACCOUNT_ID=`. The `storage-users` cli tool uses the default address to establish the connection to the `gateway` service. If the connection is failed check your custom `gateway` service `GATEWAY_GRPC_ADDR` configuration and set the same address to `storage-users` variable `OCIS_GATEWAY_GRPC_ADDR` or `STORAGE_USERS_GATEWAY_GRPC_ADDR`. The ID sources: -- 'userID' in a `https://{host}/graph/v1.0/me` - personal 'spaceID' in a `https://{host}/graph/v1.0/me/drives?$filter=driveType+eq+personal` - project 'spaceID' in a `https://{host}/graph/v1.0/me/drives?$filter=driveType+eq+project` @@ -113,7 +112,7 @@ NAME: ocis storage-users trash-bin list - Print a list of all trash-bin items for a space. USAGE: - ocis storage-users trash-bin list command [command options] ['userID' required] ['spaceID' required] + ocis storage-users trash-bin list command [command options] ['spaceID' required] ``` ```bash @@ -121,7 +120,7 @@ NAME: ocis storage-users trash-bin restore-all - Restore all trash-bin items for a space. USAGE: - ocis storage-users trash-bin restore-all command [command options] ['userID' required] ['spaceID' required] + ocis storage-users trash-bin restore-all command [command options] ['spaceID' required] COMMANDS: help, h Shows a list of commands or help for one command @@ -135,7 +134,7 @@ NAME: ocis storage-users trash-bin restore - Restore a trash-bin item by ID. USAGE: - ocis storage-users trash-bin restore command [command options] ['userID' required] ['spaceID' required] ['itemID' required] + ocis storage-users trash-bin restore command [command options] ['spaceID' required] ['itemID' required] COMMANDS: help, h Shows a list of commands or help for one command diff --git a/services/storage-users/pkg/command/trash_bin.go b/services/storage-users/pkg/command/trash_bin.go index fd02d304521..96fc974cf09 100644 --- a/services/storage-users/pkg/command/trash_bin.go +++ b/services/storage-users/pkg/command/trash_bin.go @@ -9,7 +9,6 @@ import ( "time" gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1" - userv1beta1 "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" "github.com/cs3org/reva/v2/pkg/events" @@ -95,7 +94,7 @@ func listTrashBinItems(cfg *config.Config) *cli.Command { return &cli.Command{ Name: "list", Usage: "Print a list of all trash-bin items of a space.", - ArgsUsage: "['userID' required] ['spaceID' required]", + ArgsUsage: "['spaceID' required]", Flags: []cli.Flag{}, Before: func(c *cli.Context) error { return configlog.ReturnFatal(parser.ParseConfig(cfg)) @@ -106,14 +105,9 @@ func listTrashBinItems(cfg *config.Config) *cli.Command { if err != nil { return err } - var userID, spaceID string - if c.NArg() > 1 { - userID = c.Args().Get(0) - spaceID = c.Args().Get(1) - } - if userID == "" { - _ = cli.ShowSubcommandHelp(c) - return fmt.Errorf("userID is requered") + var spaceID string + if c.NArg() > 0 { + spaceID = c.Args().Get(0) } if spaceID == "" { _ = cli.ShowSubcommandHelp(c) @@ -130,7 +124,7 @@ func listTrashBinItems(cfg *config.Config) *cli.Command { log.Error().Err(err).Msg("error selecting next gateway client") return err } - ctx, _, err := utils.Impersonate(&userv1beta1.UserId{OpaqueId: userID}, client, cfg.MachineAuthAPIKey) + ctx, err := utils.GetServiceUserContext(cfg.ServiceAccount.ServiceAccountID, client, cfg.ServiceAccount.ServiceAccountSecret) if err != nil { log.Error().Err(err).Msg("could not impersonate") return err @@ -139,7 +133,6 @@ func listTrashBinItems(cfg *config.Config) *cli.Command { spanOpts := []trace.SpanStartOption{ trace.WithSpanKind(trace.SpanKindClient), trace.WithAttributes( - attribute.KeyValue{Key: "userID", Value: attribute.StringValue(userID)}, attribute.KeyValue{Key: "spaceID", Value: attribute.StringValue(spaceID)}, ), } @@ -177,7 +170,7 @@ func restoreAllTrashBinItems(cfg *config.Config) *cli.Command { return &cli.Command{ Name: "restore-all", Usage: "Restore all trash-bin items for a space.", - ArgsUsage: "['userID' required] ['spaceID' required]", + ArgsUsage: "['spaceID' required]", Flags: []cli.Flag{ &optionFlag, }, @@ -191,14 +184,9 @@ func restoreAllTrashBinItems(cfg *config.Config) *cli.Command { return err } c.Lineage() - var userID, spaceID string - if c.NArg() > 1 { - userID = c.Args().Get(0) - spaceID = c.Args().Get(1) - } - if userID == "" { - _ = cli.ShowSubcommandHelp(c) - return cli.Exit("The userID is required", 1) + var spaceID string + if c.NArg() > 0 { + spaceID = c.Args().Get(0) } if spaceID == "" { _ = cli.ShowSubcommandHelp(c) @@ -225,7 +213,7 @@ func restoreAllTrashBinItems(cfg *config.Config) *cli.Command { log.Error().Err(err).Msg("error selecting next gateway client") return err } - ctx, _, err := utils.Impersonate(&userv1beta1.UserId{OpaqueId: userID}, client, cfg.MachineAuthAPIKey) + ctx, err := utils.GetServiceUserContext(cfg.ServiceAccount.ServiceAccountID, client, cfg.ServiceAccount.ServiceAccountSecret) if err != nil { log.Error().Err(err).Msg("could not impersonate") return err @@ -235,7 +223,6 @@ func restoreAllTrashBinItems(cfg *config.Config) *cli.Command { trace.WithSpanKind(trace.SpanKindClient), trace.WithAttributes( attribute.KeyValue{Key: "option", Value: attribute.StringValue(optionFlagVal)}, - attribute.KeyValue{Key: "userID", Value: attribute.StringValue(userID)}, attribute.KeyValue{Key: "spaceID", Value: attribute.StringValue(spaceID)}, ), } @@ -295,7 +282,7 @@ func restoreTrashBindItem(cfg *config.Config) *cli.Command { return &cli.Command{ Name: "restore", Usage: "Restore a trash-bin item by ID.", - ArgsUsage: "['userId' required] ['spaceID' required] ['itemID' required]", + ArgsUsage: "['spaceID' required] ['itemID' required]", Flags: []cli.Flag{ &optionFlag, }, @@ -309,15 +296,10 @@ func restoreTrashBindItem(cfg *config.Config) *cli.Command { return err } c.Lineage() - var userID, spaceID, itemID string - if c.NArg() > 2 { - userID = c.Args().Get(0) - spaceID = c.Args().Get(1) - itemID = c.Args().Get(2) - } - if userID == "" { - _ = cli.ShowSubcommandHelp(c) - return fmt.Errorf("userID is requered") + var spaceID, itemID string + if c.NArg() > 1 { + spaceID = c.Args().Get(0) + itemID = c.Args().Get(1) } if spaceID == "" { _ = cli.ShowSubcommandHelp(c) @@ -348,7 +330,7 @@ func restoreTrashBindItem(cfg *config.Config) *cli.Command { log.Error().Err(err).Msg("error selecting gateway client") return err } - ctx, _, err := utils.Impersonate(&userv1beta1.UserId{OpaqueId: userID}, client, cfg.MachineAuthAPIKey) + ctx, err := utils.GetServiceUserContext(cfg.ServiceAccount.ServiceAccountID, client, cfg.ServiceAccount.ServiceAccountSecret) if err != nil { log.Error().Err(err).Msg("could not impersonate") return err @@ -358,7 +340,6 @@ func restoreTrashBindItem(cfg *config.Config) *cli.Command { trace.WithSpanKind(trace.SpanKindClient), trace.WithAttributes( attribute.KeyValue{Key: "option", Value: attribute.StringValue(optionFlagVal)}, - attribute.KeyValue{Key: "userID", Value: attribute.StringValue(userID)}, attribute.KeyValue{Key: "spaceID", Value: attribute.StringValue(spaceID)}, attribute.KeyValue{Key: "itemID", Value: attribute.StringValue(itemID)}, ), From 3d0ab8172fffef6dc6c361e8bb7043ccaaa5dec0 Mon Sep 17 00:00:00 2001 From: Roman Perekhod Date: Thu, 14 Dec 2023 14:24:24 +0100 Subject: [PATCH 3/4] loging and tracing updated --- services/storage-users/README.md | 3 +- .../storage-users/pkg/command/trash_bin.go | 82 +++++++++---------- services/storage-users/pkg/config/config.go | 11 ++- 3 files changed, 48 insertions(+), 48 deletions(-) diff --git a/services/storage-users/README.md b/services/storage-users/README.md index 56c5da2b862..449b677fdee 100644 --- a/services/storage-users/README.md +++ b/services/storage-users/README.md @@ -101,7 +101,8 @@ The configuration for the `purge-expired` command is done by using the following #### List and Restore Trash-Bins Items To authenticate the cli command use `OCIS_SERVICE_ACCOUNT_SECRET=` and `OCIS_SERVICE_ACCOUNT_ID=`. The `storage-users` cli tool uses the default address to establish the connection to the `gateway` service. If the connection is failed check your custom `gateway` -service `GATEWAY_GRPC_ADDR` configuration and set the same address to `storage-users` variable `OCIS_GATEWAY_GRPC_ADDR` or `STORAGE_USERS_GATEWAY_GRPC_ADDR`. +service `GATEWAY_GRPC_ADDR` configuration and set the same address to `storage-users` variable `OCIS_GATEWAY_GRPC_ADDR` or `STORAGE_USERS_GATEWAY_GRPC_ADDR`. The variable `STORAGE_USERS_CLI_MAX_ATTEMPTS_RENAME_FILE` +defines a maximum number of attempts to rename a file when the user restores the file with `--option keep-both` to existing destination with the same name. The ID sources: - personal 'spaceID' in a `https://{host}/graph/v1.0/me/drives?$filter=driveType+eq+personal` diff --git a/services/storage-users/pkg/command/trash_bin.go b/services/storage-users/pkg/command/trash_bin.go index 96fc974cf09..af86b7e161c 100644 --- a/services/storage-users/pkg/command/trash_bin.go +++ b/services/storage-users/pkg/command/trash_bin.go @@ -17,12 +17,12 @@ import ( "github.com/cs3org/reva/v2/pkg/utils" "github.com/mohae/deepcopy" "github.com/owncloud/ocis/v2/ocis-pkg/config/configlog" + zlog "github.com/owncloud/ocis/v2/ocis-pkg/log" "github.com/owncloud/ocis/v2/ocis-pkg/tracing" "github.com/owncloud/ocis/v2/services/storage-users/pkg/config" "github.com/owncloud/ocis/v2/services/storage-users/pkg/config/parser" "github.com/owncloud/ocis/v2/services/storage-users/pkg/event" "github.com/owncloud/ocis/v2/services/storage-users/pkg/logging" - "github.com/rs/zerolog/log" "github.com/urfave/cli/v2" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -33,10 +33,10 @@ const ( REPLACE KEEP_BOTH - retrievingErrorMsg = "trash-bin items retrieving error" + _retrievingErrorMsg = "trash-bin items retrieving error" ) -var optionFlagTmpl = cli.StringFlag{ +var _optionFlagTmpl = cli.StringFlag{ Name: "option", Value: "skip", Aliases: []string{"o"}, @@ -113,7 +113,7 @@ func listTrashBinItems(cfg *config.Config) *cli.Command { _ = cli.ShowSubcommandHelp(c) return fmt.Errorf("spaceID is requered") } - fmt.Printf("Getting trash-bin items for spaceID: '%s' ...\n", spaceID) + log.Info().Msgf("Getting trash-bin items for spaceID: '%s' ...\n", spaceID) ref, err := storagespace.ParseReference(spaceID) if err != nil { @@ -121,13 +121,11 @@ func listTrashBinItems(cfg *config.Config) *cli.Command { } client, err := pool.GetGatewayServiceClient(cfg.RevaGatewayGRPCAddr) if err != nil { - log.Error().Err(err).Msg("error selecting next gateway client") - return err + return fmt.Errorf("error selecting gateway client %w", err) } ctx, err := utils.GetServiceUserContext(cfg.ServiceAccount.ServiceAccountID, client, cfg.ServiceAccount.ServiceAccountSecret) if err != nil { - log.Error().Err(err).Msg("could not impersonate") - return err + return fmt.Errorf("could not get service user context %w", err) } spanOpts := []trace.SpanStartOption{ @@ -136,16 +134,15 @@ func listTrashBinItems(cfg *config.Config) *cli.Command { attribute.KeyValue{Key: "spaceID", Value: attribute.StringValue(spaceID)}, ), } - ctx, span := tp.Tracer("storage-users trash-bin list").Start(ctx, "serve static asset", spanOpts...) + ctx, span := tp.Tracer("storage-users trash-bin").Start(ctx, "list", spanOpts...) defer span.End() res, err := client.ListRecycle(ctx, &provider.ListRecycleRequest{Ref: &ref, Key: "/"}) if err != nil { - log.Error().Err(err).Msg(retrievingErrorMsg) - return err + return fmt.Errorf("%s %w", _retrievingErrorMsg, err) } if res.Status.Code != rpc.Code_CODE_OK { - return fmt.Errorf("%s %s", retrievingErrorMsg, res.Status.Code) + return fmt.Errorf("%s %s", _retrievingErrorMsg, res.Status.Code) } if len(res.GetRecycleItems()) > 0 { @@ -165,7 +162,7 @@ func listTrashBinItems(cfg *config.Config) *cli.Command { func restoreAllTrashBinItems(cfg *config.Config) *cli.Command { var optionFlagVal string var overwriteOption int - optionFlag := optionFlagTmpl + optionFlag := _optionFlagTmpl optionFlag.Destination = &optionFlagVal return &cli.Command{ Name: "restore-all", @@ -203,20 +200,19 @@ func restoreAllTrashBinItems(cfg *config.Config) *cli.Command { _ = cli.ShowSubcommandHelp(c) return cli.Exit("The option flag is invalid", 1) } - fmt.Printf("Restoring trash-bin items for spaceID: '%s' ...\n", spaceID) + log.Info().Msgf("Restoring trash-bin items for spaceID: '%s' ...\n", spaceID) + ref, err := storagespace.ParseReference(spaceID) if err != nil { return err } client, err := pool.GetGatewayServiceClient(cfg.RevaGatewayGRPCAddr) if err != nil { - log.Error().Err(err).Msg("error selecting next gateway client") - return err + return fmt.Errorf("error selecting gateway client %w", err) } ctx, err := utils.GetServiceUserContext(cfg.ServiceAccount.ServiceAccountID, client, cfg.ServiceAccount.ServiceAccountSecret) if err != nil { - log.Error().Err(err).Msg("could not impersonate") - return err + return fmt.Errorf("could not get service user context %w", err) } spanOpts := []trace.SpanStartOption{ @@ -226,16 +222,15 @@ func restoreAllTrashBinItems(cfg *config.Config) *cli.Command { attribute.KeyValue{Key: "spaceID", Value: attribute.StringValue(spaceID)}, ), } - ctx, span := tp.Tracer("storage-users trash-bin restore-all").Start(ctx, "serve static asset", spanOpts...) + ctx, span := tp.Tracer("storage-users trash-bin").Start(ctx, "restore-all", spanOpts...) defer span.End() res, err := client.ListRecycle(ctx, &provider.ListRecycleRequest{Ref: &ref, Key: "/"}) if err != nil { - log.Error().Err(err).Msg(retrievingErrorMsg) - return err + return fmt.Errorf("%s %w", _retrievingErrorMsg, err) } if res.Status.Code != rpc.Code_CODE_OK { - return fmt.Errorf("%s %s", retrievingErrorMsg, res.Status.Code) + return fmt.Errorf("%s %s", _retrievingErrorMsg, res.Status.Code) } if len(res.GetRecycleItems()) == 0 { return cli.Exit("The trash-bin is empty. Nothing to restore", 0) @@ -263,9 +258,10 @@ func restoreAllTrashBinItems(cfg *config.Config) *cli.Command { fmt.Printf("\nRun restoring-all with option=%s\n", optionFlagVal) for _, item := range res.GetRecycleItems() { fmt.Printf("restoring itemID: '%s', path: '%s', type: '%s'\n", item.GetKey(), item.GetRef().GetPath(), itemType(item.GetType())) - dstRes, err := restore(ctx, client, ref, item, overwriteOption) + dstRes, err := restore(ctx, client, ref, item, overwriteOption, cfg.CliMaxAttemptsRenameFile, log) if err != nil { - return err + log.Err(err).Msg("trash-bin item restoring error") + continue } fmt.Printf("itemID: '%s', path: '%s', restored as '%s'\n", item.GetKey(), item.GetRef().GetPath(), dstRes.GetPath()) } @@ -277,7 +273,7 @@ func restoreAllTrashBinItems(cfg *config.Config) *cli.Command { func restoreTrashBindItem(cfg *config.Config) *cli.Command { var optionFlagVal string var overwriteOption int - optionFlag := optionFlagTmpl + optionFlag := _optionFlagTmpl optionFlag.Destination = &optionFlagVal return &cli.Command{ Name: "restore", @@ -320,6 +316,7 @@ func restoreTrashBindItem(cfg *config.Config) *cli.Command { _ = cli.ShowSubcommandHelp(c) return cli.Exit("The option flag is invalid", 1) } + log.Info().Msgf("Restoring trash-bin item for spaceID: '%s' itemID: '%s' ...\n", spaceID, itemID) ref, err := storagespace.ParseReference(spaceID) if err != nil { @@ -327,13 +324,11 @@ func restoreTrashBindItem(cfg *config.Config) *cli.Command { } client, err := pool.GetGatewayServiceClient(cfg.RevaGatewayGRPCAddr) if err != nil { - log.Error().Err(err).Msg("error selecting gateway client") - return err + return fmt.Errorf("error selecting gateway client %w", err) } ctx, err := utils.GetServiceUserContext(cfg.ServiceAccount.ServiceAccountID, client, cfg.ServiceAccount.ServiceAccountSecret) if err != nil { - log.Error().Err(err).Msg("could not impersonate") - return err + return fmt.Errorf("could not get service user context %w", err) } spanOpts := []trace.SpanStartOption{ @@ -344,16 +339,15 @@ func restoreTrashBindItem(cfg *config.Config) *cli.Command { attribute.KeyValue{Key: "itemID", Value: attribute.StringValue(itemID)}, ), } - ctx, span := tp.Tracer("storage-users trash-bin restore").Start(ctx, "serve static asset", spanOpts...) + ctx, span := tp.Tracer("storage-users trash-bin").Start(ctx, "restore", spanOpts...) defer span.End() res, err := client.ListRecycle(ctx, &provider.ListRecycleRequest{Ref: &ref, Key: "/"}) if err != nil { - log.Error().Err(err).Msg(retrievingErrorMsg) - return err + return fmt.Errorf("%s %w", _retrievingErrorMsg, err) } if res.Status.Code != rpc.Code_CODE_OK { - return fmt.Errorf("%s %s", retrievingErrorMsg, res.Status.Code) + return fmt.Errorf("%s %s", _retrievingErrorMsg, res.Status.Code) } var found bool @@ -370,7 +364,7 @@ func restoreTrashBindItem(cfg *config.Config) *cli.Command { } fmt.Printf("\nRun restoring with option=%s\n", optionFlagVal) fmt.Printf("restoring itemID: '%s', path: '%s', type: '%s'\n", itemRef.GetKey(), itemRef.GetRef().GetPath(), itemType(itemRef.GetType())) - dstRes, err := restore(ctx, client, ref, itemRef, overwriteOption) + dstRes, err := restore(ctx, client, ref, itemRef, overwriteOption, cfg.CliMaxAttemptsRenameFile, log) if err != nil { return err } @@ -380,7 +374,7 @@ func restoreTrashBindItem(cfg *config.Config) *cli.Command { } } -func restore(ctx context.Context, client gateway.GatewayAPIClient, ref provider.Reference, item *provider.RecycleItem, overwriteOption int) (*provider.Reference, error) { +func restore(ctx context.Context, client gateway.GatewayAPIClient, ref provider.Reference, item *provider.RecycleItem, overwriteOption int, maxRenameAttempt int, log zlog.Logger) (*provider.Reference, error) { dst, _ := deepcopy.Copy(ref).(provider.Reference) dst.Path = utils.MakeRelativePath(item.GetRef().GetPath()) // Restore request @@ -396,7 +390,7 @@ func restore(ctx context.Context, client gateway.GatewayAPIClient, ref provider. } if exists { - fmt.Printf("destination '%s' exists.\n", dstStatRes.GetInfo().GetPath()) + log.Info().Msgf("destination '%s' exists.\n", dstStatRes.GetInfo().GetPath()) switch overwriteOption { case SKIP: return &dst, nil @@ -412,27 +406,29 @@ func restore(ctx context.Context, client gateway.GatewayAPIClient, ref provider. } case KEEP_BOTH: // modify the file name - req.RestoreRef, err = resolveDestination(ctx, client, dst) + req.RestoreRef, err = resolveDestination(ctx, client, dst, maxRenameAttempt) if err != nil { - return &dst, fmt.Errorf("trash-bin item restoring error %w", err) + return &dst, err } } } res, err := client.RestoreRecycleItem(ctx, req) if err != nil { - log.Error().Err(err).Msg("trash-bin item restoring error") - return req.RestoreRef, err + return req.RestoreRef, fmt.Errorf("restoring error %w", err) } if res.Status.Code != rpc.Code_CODE_OK { - return req.RestoreRef, fmt.Errorf("trash-bin item restoring error %s", res.Status.Code) + return req.RestoreRef, fmt.Errorf("can not restore %s", res.Status.Code) } return req.RestoreRef, nil } -func resolveDestination(ctx context.Context, client gateway.GatewayAPIClient, dstRef provider.Reference) (*provider.Reference, error) { +func resolveDestination(ctx context.Context, client gateway.GatewayAPIClient, dstRef provider.Reference, maxRenameAttempt int) (*provider.Reference, error) { dst := dstRef - for i := 1; i < 100; i++ { + if maxRenameAttempt < 100 { + maxRenameAttempt = 100 + } + for i := 1; i < maxRenameAttempt; i++ { dst.Path = modifyFilename(dstRef.Path, i) exists, _, err := isDestinationExists(ctx, client, dst) if err != nil { diff --git a/services/storage-users/pkg/config/config.go b/services/storage-users/pkg/config/config.go index 65c55368d81..15a44e253a3 100644 --- a/services/storage-users/pkg/config/config.go +++ b/services/storage-users/pkg/config/config.go @@ -18,10 +18,8 @@ type Config struct { GRPC GRPCConfig `yaml:"grpc"` HTTP HTTPConfig `yaml:"http"` - TokenManager *TokenManager `yaml:"token_manager"` - Reva *shared.Reva `yaml:"reva"` - RevaGatewayGRPCAddr string `yaml:"gateway_addr" env:"OCIS_GATEWAY_GRPC_ADDR;STORAGE_USERS_GATEWAY_GRPC_ADDR" desc:"The bind address of the gateway GRPC address."` - MachineAuthAPIKey string `yaml:"machine_auth_api_key" env:"OCIS_MACHINE_AUTH_API_KEY" desc:"Machine auth API key used to validate internal requests necessary for the access to resources from other services."` + TokenManager *TokenManager `yaml:"token_manager"` + Reva *shared.Reva `yaml:"reva"` SkipUserGroupsInToken bool `yaml:"skip_user_groups_in_token" env:"STORAGE_USERS_SKIP_USER_GROUPS_IN_TOKEN" desc:"Disables the loading of user's group memberships from the reva access token."` GracefulShutdownTimeout int `yaml:"graceful_shutdown_timeout" env:"STORAGE_USERS_GRACEFUL_SHUTDOWN_TIMEOUT" desc:"The number of seconds to wait for the 'storage-users' service to shutdown cleanly before exiting with an error that gets logged. Note: This setting is only applicable when running the 'storage-users' service as a standalone service. See the text description for more details."` @@ -43,6 +41,11 @@ type Config struct { Tasks Tasks `yaml:"tasks"` ServiceAccount ServiceAccount `yaml:"service_account"` + // CLI + RevaGatewayGRPCAddr string `yaml:"gateway_addr" env:"OCIS_GATEWAY_GRPC_ADDR;STORAGE_USERS_GATEWAY_GRPC_ADDR" desc:"The bind address of the gateway GRPC address."` + MachineAuthAPIKey string `yaml:"machine_auth_api_key" env:"OCIS_MACHINE_AUTH_API_KEY" desc:"Machine auth API key used to validate internal requests necessary for the access to resources from other services."` + CliMaxAttemptsRenameFile int `yaml:"max_attempts_rename_file" env:"STORAGE_USERS_CLI_MAX_ATTEMPTS_RENAME_FILE" desc:"Maximum number of attempts to rename a file when user restores the file to existing destination with the same name. The minimum value is 100."` + Supervised bool `yaml:"-"` Context context.Context `yaml:"-"` } From 492e511010e7a2002100baadc33bfaf83ea3bcec Mon Sep 17 00:00:00 2001 From: Roman Perekhod Date: Mon, 18 Dec 2023 18:54:02 +0100 Subject: [PATCH 4/4] rework the cli --- changelog/unreleased/add-trach-bin-cli.md | 2 +- services/storage-users/README.md | 20 +- .../storage-users/pkg/command/trash_bin.go | 212 +++++++++--------- services/storage-users/pkg/config/config.go | 2 +- 4 files changed, 126 insertions(+), 110 deletions(-) diff --git a/changelog/unreleased/add-trach-bin-cli.md b/changelog/unreleased/add-trach-bin-cli.md index 828180ab027..5b2771035ba 100644 --- a/changelog/unreleased/add-trach-bin-cli.md +++ b/changelog/unreleased/add-trach-bin-cli.md @@ -1,4 +1,4 @@ -Enhancement: Add cli commands for trash-binq +Enhancement: Add cli commands for trash-bin We added the `list` and `restore` commands to the trash-bin items to the CLI diff --git a/services/storage-users/README.md b/services/storage-users/README.md index 449b677fdee..98b26bac57b 100644 --- a/services/storage-users/README.md +++ b/services/storage-users/README.md @@ -110,10 +110,18 @@ The ID sources: ```bash NAME: - ocis storage-users trash-bin list - Print a list of all trash-bin items for a space. + ocis storage-users trash-bin list - Print a list of all trash-bin items of a space. USAGE: ocis storage-users trash-bin list command [command options] ['spaceID' required] + +COMMANDS: + help, h Shows a list of commands or help for one command + +OPTIONS: + --verbose, -v Get more verbose output (default: false) + --help, -h show help + ``` ```bash @@ -127,7 +135,11 @@ COMMANDS: help, h Shows a list of commands or help for one command OPTIONS: - --option value, -o value The restore option defines the behavior for a file to be restored, where the file name already already exists in the target space. Supported values are: 'skip', 'replace' and 'keep-both'. The default value is 'skip' overwriting an existing file. + --option value, -o value The restore option defines the behavior for a file to be restored, where the file name already already exists in the target space. Supported values are: 'skip', 'replace' and 'keep-both'. (default: The default value is 'skip' overwriting an existing file) + --verbose, -v Get more verbose output (default: false) + --yes, -y Automatic yes to prompts. Assume 'yes' as answer to all prompts and run non-interactively. (default: false) + --help, -h show help + ``` ```bash @@ -141,7 +153,9 @@ COMMANDS: help, h Shows a list of commands or help for one command OPTIONS: - --option value, -o value The restore option defines the behavior for a file to be restored, where the file name already already exists in the target space. Supported values are: 'skip', 'replace' and 'keep-both'. The default value is 'skip' overwriting an existing file. + --option value, -o value The restore option defines the behavior for a file to be restored, where the file name already already exists in the target space. Supported values are: 'skip', 'replace' and 'keep-both'. (default: The default value is 'skip' overwriting an existing file) + --verbose, -v Get more verbose output (default: false) + --help, -h show help ``` ## Caching diff --git a/services/storage-users/pkg/command/trash_bin.go b/services/storage-users/pkg/command/trash_bin.go index af86b7e161c..dcdb079594f 100644 --- a/services/storage-users/pkg/command/trash_bin.go +++ b/services/storage-users/pkg/command/trash_bin.go @@ -3,8 +3,10 @@ package command import ( "context" "fmt" + "os" "path" "path/filepath" + "strconv" "strings" "time" @@ -16,24 +18,20 @@ import ( "github.com/cs3org/reva/v2/pkg/storagespace" "github.com/cs3org/reva/v2/pkg/utils" "github.com/mohae/deepcopy" + tw "github.com/olekukonko/tablewriter" "github.com/owncloud/ocis/v2/ocis-pkg/config/configlog" zlog "github.com/owncloud/ocis/v2/ocis-pkg/log" - "github.com/owncloud/ocis/v2/ocis-pkg/tracing" "github.com/owncloud/ocis/v2/services/storage-users/pkg/config" "github.com/owncloud/ocis/v2/services/storage-users/pkg/config/parser" "github.com/owncloud/ocis/v2/services/storage-users/pkg/event" - "github.com/owncloud/ocis/v2/services/storage-users/pkg/logging" + "github.com/rs/zerolog" "github.com/urfave/cli/v2" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" ) const ( SKIP = iota REPLACE KEEP_BOTH - - _retrievingErrorMsg = "trash-bin items retrieving error" ) var _optionFlagTmpl = cli.StringFlag{ @@ -44,6 +42,18 @@ var _optionFlagTmpl = cli.StringFlag{ DefaultText: "The default value is 'skip' overwriting an existing file", } +var _verboseFlagTmpl = cli.BoolFlag{ + Name: "verbose", + Aliases: []string{"v"}, + Usage: "Get more verbose output", +} + +var _applyYesFlagTmpl = cli.BoolFlag{ + Name: "yes", + Aliases: []string{"y"}, + Usage: "Automatic yes to prompts. Assume 'yes' as answer to all prompts and run non-interactively.", +} + // TrashBin wraps trash-bin related sub-commands. func TrashBin(cfg *config.Config) *cli.Command { return &cli.Command{ @@ -91,20 +101,21 @@ func PurgeExpiredResources(cfg *config.Config) *cli.Command { } func listTrashBinItems(cfg *config.Config) *cli.Command { + var verboseVal bool + verboseFlag := _verboseFlagTmpl + verboseFlag.Destination = &verboseVal return &cli.Command{ Name: "list", Usage: "Print a list of all trash-bin items of a space.", ArgsUsage: "['spaceID' required]", - Flags: []cli.Flag{}, + Flags: []cli.Flag{ + &verboseFlag, + }, Before: func(c *cli.Context) error { return configlog.ReturnFatal(parser.ParseConfig(cfg)) }, Action: func(c *cli.Context) error { - log := logging.Configure(cfg.Service.Name, cfg.Log) - tp, err := tracing.GetServiceTraceProvider(cfg.Tracing, cfg.Service.Name) - if err != nil { - return err - } + log := cliLogger(verboseVal) var spaceID string if c.NArg() > 0 { spaceID = c.Args().Get(0) @@ -113,7 +124,7 @@ func listTrashBinItems(cfg *config.Config) *cli.Command { _ = cli.ShowSubcommandHelp(c) return fmt.Errorf("spaceID is requered") } - log.Info().Msgf("Getting trash-bin items for spaceID: '%s' ...\n", spaceID) + log.Info().Msgf("Getting trash-bin items for spaceID: '%s' ...", spaceID) ref, err := storagespace.ParseReference(spaceID) if err != nil { @@ -127,33 +138,17 @@ func listTrashBinItems(cfg *config.Config) *cli.Command { if err != nil { return fmt.Errorf("could not get service user context %w", err) } - - spanOpts := []trace.SpanStartOption{ - trace.WithSpanKind(trace.SpanKindClient), - trace.WithAttributes( - attribute.KeyValue{Key: "spaceID", Value: attribute.StringValue(spaceID)}, - ), - } - ctx, span := tp.Tracer("storage-users trash-bin").Start(ctx, "list", spanOpts...) - defer span.End() - - res, err := client.ListRecycle(ctx, &provider.ListRecycleRequest{Ref: &ref, Key: "/"}) + res, err := listRecycle(ctx, client, ref) if err != nil { - return fmt.Errorf("%s %w", _retrievingErrorMsg, err) - } - if res.Status.Code != rpc.Code_CODE_OK { - return fmt.Errorf("%s %s", _retrievingErrorMsg, res.Status.Code) - } - - if len(res.GetRecycleItems()) > 0 { - fmt.Println("The list of the trash-bin items. Use an itemID to restore.") - } else { - fmt.Println("The list is empty.") + return err } + table := itemsTable(len(res.GetRecycleItems())) for _, item := range res.GetRecycleItems() { - fmt.Printf("itemID: '%s', path: '%s', type: '%s', delited at :%s\n", item.GetKey(), item.GetRef().GetPath(), itemType(item.GetType()), utils.TSToTime(item.GetDeletionTime()).UTC().Format(time.RFC3339)) + table.Append([]string{item.GetKey(), item.GetRef().GetPath(), itemType(item.GetType()), utils.TSToTime(item.GetDeletionTime()).UTC().Format(time.RFC3339)}) } + table.Render() + fmt.Println("Use an itemID to restore an item.") return nil }, } @@ -164,23 +159,26 @@ func restoreAllTrashBinItems(cfg *config.Config) *cli.Command { var overwriteOption int optionFlag := _optionFlagTmpl optionFlag.Destination = &optionFlagVal + var verboseVal bool + verboseFlag := _verboseFlagTmpl + verboseFlag.Destination = &verboseVal + var applyYesVal bool + applyYesFlag := _applyYesFlagTmpl + applyYesFlag.Destination = &applyYesVal return &cli.Command{ Name: "restore-all", Usage: "Restore all trash-bin items for a space.", ArgsUsage: "['spaceID' required]", Flags: []cli.Flag{ &optionFlag, + &verboseFlag, + &applyYesFlag, }, Before: func(c *cli.Context) error { return configlog.ReturnFatal(parser.ParseConfig(cfg)) }, Action: func(c *cli.Context) error { - log := logging.Configure(cfg.Service.Name, cfg.Log) - tp, err := tracing.GetServiceTraceProvider(cfg.Tracing, cfg.Service.Name) - if err != nil { - return err - } - c.Lineage() + log := cliLogger(verboseVal) var spaceID string if c.NArg() > 0 { spaceID = c.Args().Get(0) @@ -200,7 +198,7 @@ func restoreAllTrashBinItems(cfg *config.Config) *cli.Command { _ = cli.ShowSubcommandHelp(c) return cli.Exit("The option flag is invalid", 1) } - log.Info().Msgf("Restoring trash-bin items for spaceID: '%s' ...\n", spaceID) + log.Info().Msgf("Restoring trash-bin items for spaceID: '%s' ...", spaceID) ref, err := storagespace.ParseReference(spaceID) if err != nil { @@ -214,50 +212,37 @@ func restoreAllTrashBinItems(cfg *config.Config) *cli.Command { if err != nil { return fmt.Errorf("could not get service user context %w", err) } - - spanOpts := []trace.SpanStartOption{ - trace.WithSpanKind(trace.SpanKindClient), - trace.WithAttributes( - attribute.KeyValue{Key: "option", Value: attribute.StringValue(optionFlagVal)}, - attribute.KeyValue{Key: "spaceID", Value: attribute.StringValue(spaceID)}, - ), - } - ctx, span := tp.Tracer("storage-users trash-bin").Start(ctx, "restore-all", spanOpts...) - defer span.End() - - res, err := client.ListRecycle(ctx, &provider.ListRecycleRequest{Ref: &ref, Key: "/"}) + res, err := listRecycle(ctx, client, ref) if err != nil { - return fmt.Errorf("%s %w", _retrievingErrorMsg, err) - } - if res.Status.Code != rpc.Code_CODE_OK { - return fmt.Errorf("%s %s", _retrievingErrorMsg, res.Status.Code) - } - if len(res.GetRecycleItems()) == 0 { - return cli.Exit("The trash-bin is empty. Nothing to restore", 0) + return err } - for { - fmt.Printf("Foud %d items that could be restored, continue (Y/n), show the items list (s): ", len(res.GetRecycleItems())) - var i string - _, err := fmt.Scanf("%s", &i) - if err != nil { - log.Err(err).Send() - continue - } - if strings.ToLower(i) == "y" { - break - } else if strings.ToLower(i) == "n" { - return nil - } else if strings.ToLower(i) == "s" { - for _, item := range res.GetRecycleItems() { - fmt.Printf("itemID: '%s', path: '%s', type: '%s', delited at: %s\n", item.GetKey(), item.GetRef().GetPath(), itemType(item.GetType()), utils.TSToTime(item.GetDeletionTime()).UTC().Format(time.RFC3339)) + if !applyYesVal { + for { + fmt.Printf("Found %d items that could be restored, continue (Y/n), show the items list (s): ", len(res.GetRecycleItems())) + var i string + _, err := fmt.Scanf("%s", &i) + if err != nil { + log.Err(err).Send() + continue + } + if strings.ToLower(i) == "y" { + break + } else if strings.ToLower(i) == "n" { + return nil + } else if strings.ToLower(i) == "s" { + table := itemsTable(len(res.GetRecycleItems())) + for _, item := range res.GetRecycleItems() { + table.Append([]string{item.GetKey(), item.GetRef().GetPath(), itemType(item.GetType()), utils.TSToTime(item.GetDeletionTime()).UTC().Format(time.RFC3339)}) + } + table.Render() } } } - fmt.Printf("\nRun restoring-all with option=%s\n", optionFlagVal) + log.Info().Msgf("Run restoring-all with option=%s", optionFlagVal) for _, item := range res.GetRecycleItems() { - fmt.Printf("restoring itemID: '%s', path: '%s', type: '%s'\n", item.GetKey(), item.GetRef().GetPath(), itemType(item.GetType())) + log.Info().Msgf("restoring itemID: '%s', path: '%s', type: '%s'", item.GetKey(), item.GetRef().GetPath(), itemType(item.GetType())) dstRes, err := restore(ctx, client, ref, item, overwriteOption, cfg.CliMaxAttemptsRenameFile, log) if err != nil { log.Err(err).Msg("trash-bin item restoring error") @@ -275,23 +260,22 @@ func restoreTrashBindItem(cfg *config.Config) *cli.Command { var overwriteOption int optionFlag := _optionFlagTmpl optionFlag.Destination = &optionFlagVal + var verboseVal bool + verboseFlag := _verboseFlagTmpl + verboseFlag.Destination = &verboseVal return &cli.Command{ Name: "restore", Usage: "Restore a trash-bin item by ID.", ArgsUsage: "['spaceID' required] ['itemID' required]", Flags: []cli.Flag{ &optionFlag, + &verboseFlag, }, Before: func(c *cli.Context) error { return configlog.ReturnFatal(parser.ParseConfig(cfg)) }, Action: func(c *cli.Context) error { - log := logging.Configure(cfg.Service.Name, cfg.Log) - tp, err := tracing.GetServiceTraceProvider(cfg.Tracing, cfg.Service.Name) - if err != nil { - return err - } - c.Lineage() + log := cliLogger(verboseVal) var spaceID, itemID string if c.NArg() > 1 { spaceID = c.Args().Get(0) @@ -316,7 +300,7 @@ func restoreTrashBindItem(cfg *config.Config) *cli.Command { _ = cli.ShowSubcommandHelp(c) return cli.Exit("The option flag is invalid", 1) } - log.Info().Msgf("Restoring trash-bin item for spaceID: '%s' itemID: '%s' ...\n", spaceID, itemID) + log.Info().Msgf("Restoring trash-bin item for spaceID: '%s' itemID: '%s' ...", spaceID, itemID) ref, err := storagespace.ParseReference(spaceID) if err != nil { @@ -330,24 +314,9 @@ func restoreTrashBindItem(cfg *config.Config) *cli.Command { if err != nil { return fmt.Errorf("could not get service user context %w", err) } - - spanOpts := []trace.SpanStartOption{ - trace.WithSpanKind(trace.SpanKindClient), - trace.WithAttributes( - attribute.KeyValue{Key: "option", Value: attribute.StringValue(optionFlagVal)}, - attribute.KeyValue{Key: "spaceID", Value: attribute.StringValue(spaceID)}, - attribute.KeyValue{Key: "itemID", Value: attribute.StringValue(itemID)}, - ), - } - ctx, span := tp.Tracer("storage-users trash-bin").Start(ctx, "restore", spanOpts...) - defer span.End() - - res, err := client.ListRecycle(ctx, &provider.ListRecycleRequest{Ref: &ref, Key: "/"}) + res, err := listRecycle(ctx, client, ref) if err != nil { - return fmt.Errorf("%s %w", _retrievingErrorMsg, err) - } - if res.Status.Code != rpc.Code_CODE_OK { - return fmt.Errorf("%s %s", _retrievingErrorMsg, res.Status.Code) + return err } var found bool @@ -362,8 +331,8 @@ func restoreTrashBindItem(cfg *config.Config) *cli.Command { if !found { return fmt.Errorf("itemID '%s' not found", itemID) } - fmt.Printf("\nRun restoring with option=%s\n", optionFlagVal) - fmt.Printf("restoring itemID: '%s', path: '%s', type: '%s'\n", itemRef.GetKey(), itemRef.GetRef().GetPath(), itemType(itemRef.GetType())) + log.Info().Msgf("Run restoring with option=%s", optionFlagVal) + log.Info().Msgf("restoring itemID: '%s', path: '%s', type: '%s", itemRef.GetKey(), itemRef.GetRef().GetPath(), itemType(itemRef.GetType())) dstRes, err := restore(ctx, client, ref, itemRef, overwriteOption, cfg.CliMaxAttemptsRenameFile, log) if err != nil { return err @@ -374,6 +343,21 @@ func restoreTrashBindItem(cfg *config.Config) *cli.Command { } } +func listRecycle(ctx context.Context, client gateway.GatewayAPIClient, ref provider.Reference) (*provider.ListRecycleResponse, error) { + _retrievingErrorMsg := "trash-bin items retrieving error" + res, err := client.ListRecycle(ctx, &provider.ListRecycleRequest{Ref: &ref, Key: "/"}) + if err != nil { + return nil, fmt.Errorf("%s %w", _retrievingErrorMsg, err) + } + if res.Status.Code != rpc.Code_CODE_OK { + return nil, fmt.Errorf("%s %s", _retrievingErrorMsg, res.Status.Code) + } + if len(res.GetRecycleItems()) == 0 { + return res, cli.Exit("The trash-bin is empty. Nothing to restore", 0) + } + return res, nil +} + func restore(ctx context.Context, client gateway.GatewayAPIClient, ref provider.Reference, item *provider.RecycleItem, overwriteOption int, maxRenameAttempt int, log zlog.Logger) (*provider.Reference, error) { dst, _ := deepcopy.Copy(ref).(provider.Reference) dst.Path = utils.MakeRelativePath(item.GetRef().GetPath()) @@ -390,7 +374,7 @@ func restore(ctx context.Context, client gateway.GatewayAPIClient, ref provider. } if exists { - log.Info().Msgf("destination '%s' exists.\n", dstStatRes.GetInfo().GetPath()) + log.Info().Msgf("destination '%s' exists.", dstStatRes.GetInfo().GetPath()) switch overwriteOption { case SKIP: return &dst, nil @@ -484,3 +468,21 @@ func itemType(it provider.ResourceType) string { } return itemType } + +func itemsTable(total int) *tw.Table { + table := tw.NewWriter(os.Stdout) + table.SetHeader([]string{"itemID", "path", "type", "delete at"}) + table.SetAutoFormatHeaders(false) + table.SetFooter([]string{"", "", "", "total count: " + strconv.Itoa(total)}) + return table +} + +func cliLogger(verbose bool) zlog.Logger { + logLvl := zerolog.ErrorLevel + if verbose { + logLvl = zerolog.InfoLevel + } + zerolog.SetGlobalLevel(zerolog.TraceLevel) + output := zerolog.ConsoleWriter{Out: os.Stdout, TimeFormat: time.RFC3339, NoColor: true} + return zlog.Logger{zerolog.New(output).With().Timestamp().Logger().Level(logLvl)} +} diff --git a/services/storage-users/pkg/config/config.go b/services/storage-users/pkg/config/config.go index 15a44e253a3..19da5998a54 100644 --- a/services/storage-users/pkg/config/config.go +++ b/services/storage-users/pkg/config/config.go @@ -44,7 +44,7 @@ type Config struct { // CLI RevaGatewayGRPCAddr string `yaml:"gateway_addr" env:"OCIS_GATEWAY_GRPC_ADDR;STORAGE_USERS_GATEWAY_GRPC_ADDR" desc:"The bind address of the gateway GRPC address."` MachineAuthAPIKey string `yaml:"machine_auth_api_key" env:"OCIS_MACHINE_AUTH_API_KEY" desc:"Machine auth API key used to validate internal requests necessary for the access to resources from other services."` - CliMaxAttemptsRenameFile int `yaml:"max_attempts_rename_file" env:"STORAGE_USERS_CLI_MAX_ATTEMPTS_RENAME_FILE" desc:"Maximum number of attempts to rename a file when user restores the file to existing destination with the same name. The minimum value is 100."` + CliMaxAttemptsRenameFile int `yaml:"max_attempts_rename_file" env:"STORAGE_USERS_CLI_MAX_ATTEMPTS_RENAME_FILE" desc:"The maximum number of attempts to rename a file when a user restores a file to an existing destination with the same name. The minimum value is 100."` Supervised bool `yaml:"-"` Context context.Context `yaml:"-"`