From 4af4bba8b119ae86702e935b404328c6bd0f261d Mon Sep 17 00:00:00 2001 From: Ralf Haferkamp Date: Thu, 15 Jun 2023 17:49:43 +0200 Subject: [PATCH] Add cli tool for verifing and repairing treesizes in decomposedfs --- ...add-decomposedfs-check-treesize-command.md | 6 + ocis/pkg/command/decomposedfs.go | 160 +++++++++++++++++- 2 files changed, 161 insertions(+), 5 deletions(-) create mode 100644 changelog/unreleased/add-decomposedfs-check-treesize-command.md diff --git a/changelog/unreleased/add-decomposedfs-check-treesize-command.md b/changelog/unreleased/add-decomposedfs-check-treesize-command.md new file mode 100644 index 00000000000..d26c9a237bb --- /dev/null +++ b/changelog/unreleased/add-decomposedfs-check-treesize-command.md @@ -0,0 +1,6 @@ +Enhancement: add 'ocis decomposedfs check-treesize' command + +We added a 'ocis decomposedfs check-treesize' command for checking (and reparing) +the treesize metadata of a storage space. + +https://github.com/owncloud/ocis/pull/6556 diff --git a/ocis/pkg/command/decomposedfs.go b/ocis/pkg/command/decomposedfs.go index fb91f0d7458..d00c9faadc0 100644 --- a/ocis/pkg/command/decomposedfs.go +++ b/ocis/pkg/command/decomposedfs.go @@ -4,15 +4,23 @@ import ( "context" "encoding/base64" "encoding/hex" + "errors" "fmt" "sort" "strings" + userpb "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + revactx "github.com/cs3org/reva/v2/pkg/ctx" "github.com/cs3org/reva/v2/pkg/storage/cache" + "github.com/cs3org/reva/v2/pkg/storage/fs/ocis/blobstore" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/lookup" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/metadata" + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/node" "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/options" + "github.com/cs3org/reva/v2/pkg/storage/utils/decomposedfs/tree" "github.com/cs3org/reva/v2/pkg/storagespace" + "github.com/cs3org/reva/v2/pkg/store" "github.com/owncloud/ocis/v2/ocis-pkg/config" "github.com/owncloud/ocis/v2/ocis/pkg/register" "github.com/urfave/cli/v2" @@ -21,10 +29,13 @@ import ( // DecomposedfsCommand is the entrypoint for the groups command. func DecomposedfsCommand(cfg *config.Config) *cli.Command { return &cli.Command{ - Name: "decomposedfs", - Usage: `cli tools to inspect and manipulate a decomposedfs storage.`, - Category: "maintenance", - Subcommands: []*cli.Command{metadataCmd(cfg)}, + Name: "decomposedfs", + Usage: `cli tools to inspect and manipulate a decomposedfs storage.`, + Category: "maintenance", + Subcommands: []*cli.Command{ + metadataCmd(cfg), + checkCmd(cfg), + }, } } @@ -32,6 +43,145 @@ func init() { register.AddCommand(DecomposedfsCommand) } +func checkCmd(cfg *config.Config) *cli.Command { + return &cli.Command{ + Name: "check-treesize", + Usage: `cli tool to check the treesize metadata of a Space`, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "root", + Aliases: []string{"r"}, + Required: true, + Usage: "Path to the root directory of the decomposedfs", + }, + &cli.StringFlag{ + Name: "node", + Required: true, + Aliases: []string{"n"}, + Usage: "Space ID of the Space to inspect", + }, + &cli.BoolFlag{ + Name: "repair", + Usage: "Try to repair nodes with incorrect treesize metadata. IMPORTANT: Only use this while ownCloud Infinite Scale is not running.", + }, + &cli.BoolFlag{ + Name: "force", + Usage: "Do not prompt for confirmation when running in repair mode.", + }, + }, + Action: check, + } +} + +func check(c *cli.Context) error { + rootFlag := c.String("root") + repairFlag := c.Bool("repair") + + if repairFlag && !c.Bool("force") { + answer := strings.ToLower(stringPrompt("IMPORTANT: Only use '--repair' when ownCloud Infinite Scale is not running. Do you want to continue? [yes | no = default]")) + if answer != "yes" && answer != "y" { + return nil + } + } + + lu, backend := getBackend(c) + o := &options.Options{ + MetadataBackend: backend.Name(), + MaxConcurrency: 100, + } + bs, err := blobstore.New(rootFlag) + if err != nil { + fmt.Println("Failed to init blobstore") + return err + } + + tree := tree.New(lu, bs, o, store.Create()) + + nId := c.String("node") + n, err := lu.NodeFromSpaceID(context.Background(), nId) + if err != nil || !n.Exists { + fmt.Println("Can not find node '" + nId + "'") + return err + } + fmt.Printf("Checking treesizes in space: %s (id: %s)\n", n.Name, n.ID) + ctx := revactx.ContextSetUser(context.Background(), + &userpb.User{ + Id: &userpb.UserId{ + OpaqueId: "00000000-0000-0000-0000-000000000000", + }, + Username: "offline", + }) + + treeSize, err := walkTree(ctx, tree, lu, n, repairFlag) + treesizeFromMetadata, err := n.GetTreeSize() + if err != nil { + fmt.Printf("failed to read treesize of node: %s: %s", n.ID, err) + } + if treesizeFromMetadata != treeSize { + fmt.Printf("Tree sizes mismatch for space: %s\n\tNodeId: %s\n\tInternalPath: %s\n\tcalculated treesize: %d\n\ttreesize in metadata: %d\n", + n.Name, n.ID, n.InternalPath(), treeSize, treesizeFromMetadata) + if repairFlag { + fmt.Printf("Fixing tree size for node: %s. Calculated treesize: %d\n", + n.ID, treeSize) + n.SetTreeSize(treeSize) + } + } + return nil +} + +func walkTree(ctx context.Context, tree *tree.Tree, lu *lookup.Lookup, root *node.Node, repair bool) (uint64, error) { + if root.Type() != provider.ResourceType_RESOURCE_TYPE_CONTAINER { + return 0, errors.New("can't travers non-container nodes") + } + children, err := tree.ListFolder(ctx, root) + if err != nil { + fmt.Println("Can not list children for space'" + root.ID + "'") + return 0, err + } + + var treesize uint64 + for _, child := range children { + switch child.Type() { + case provider.ResourceType_RESOURCE_TYPE_CONTAINER: + subtreesize, err := walkTree(ctx, tree, lu, child, repair) + if err != nil { + fmt.Printf("error calculating tree size of node: %s: %s", child.ID, err) + return 0, err + } + treesizeFromMetadata, err := child.GetTreeSize() + if err != nil { + fmt.Printf("failed to read tree size of node: %s: %s", child.ID, err) + return 0, err + } + if treesizeFromMetadata != subtreesize { + origin, err := lu.Path(ctx, child, node.NoCheck) + if err != nil { + fmt.Printf("error get path: %s\n", err) + } + fmt.Printf("Tree sizes mismatch for node: %s\n\tNodeId: %s\n\tInternalPath: %s\n\tcalculated treesize: %d\n\ttreesize in metadata: %d\n", + origin, child.ID, child.InternalPath(), subtreesize, treesizeFromMetadata) + if repair { + fmt.Printf("Fixing tree size for node: %s. Calculated treesize: %d\n", + child.ID, subtreesize) + child.SetTreeSize(subtreesize) + } + } + treesize += subtreesize + case provider.ResourceType_RESOURCE_TYPE_FILE: + blobsize, err := child.GetBlobSize() + if err != nil { + fmt.Printf("error reading blobsize of node: %s: %s", child.ID, err) + return 0, err + } + treesize += blobsize + default: + fmt.Printf("Ignoring type: %v, node: %s %s\n", child.Type(), child.Name, child.ID) + } + } + + return treesize, nil +} + func metadataCmd(cfg *config.Config) *cli.Command { return &cli.Command{ Name: "metadata", @@ -192,7 +342,7 @@ func getPath(c *cli.Context, lu *lookup.Lookup) (string, error) { fmt.Println("Invalid node id.") return "", err } - n, _ := lu.NodeFromID(context.Background(), &id) + n, err := lu.NodeFromID(context.Background(), &id) if err != nil || !n.Exists { fmt.Println("Can not find node '" + nId + "'") return "", err