Purge-Revision Cli

changelog/unreleased/remove-revisions-cli.md

@@ -0,0 +1,5 @@
+Enhancement: Add cli to purge revisions
+
+Adds a cli that allows removing all revisions for a storage-provider.
+
+https://github.com/owncloud/ocis/pull/9497

ocis/README.md

@@ -23,4 +23,82 @@ To authenticate the connection to the etcd registry, you have to set `ETCD_USERN
 
 ## Memory limits
 
-oCIS will automatically set the go native `GOMEMLIMIT` to `0.9`. To disable the limit set `AUTOMEMEMLIMIT=off`. For more information take a look at the official [Guide to the Go Garbage Collector](https://go.dev/doc/gc-guide).
+oCIS will automatically set the go native `GOMEMLIMIT` to `0.9`. To disable the limit set `AUTOMEMEMLIMIT=off`. For more information take a look at the official [Guide to the Go Garbage Collector](https://go.dev/doc/gc-guide).
+
+## CLI Commands
+
+The ocis package offers a variety of cli commands to monitor or repair ocis installations. All these commands have a common mandatory parameter: `--basePath` (or `-p`) which needs to point to a storage provider. Example paths are:
+
+```bash
+.ocis/storage/users          # bare metal installation
+/var/tmp/ocis/storage/users  # docker installation
+...
+```
+
+These paths can vary depending on your ocis installation.
+
+### Backup CLI
+
+The backup command allows inspecting the consistency of an ocis storage:
+
+```bash
+ocis backup consistency -p /base/path/storage/users
+```
+
+This will check the consistency of the storage and output a list of inconsistencies. Inconsistencies can be:
+
+* **Orphaned Blobs**\
+A blob in the blobstore that is not referenced by any file metadata.
+* **Missing Blobs**\
+A blob referenced by file metadata that is not present in the blobstore.
+* **Missing Nodes**\
+A node that is referenced by a symlink but doesn't exist.
+* **Missing Link**\
+A node that is not referenced by any symlink but should be.
+* **Missing Files**\
+A node that is missing essential files (such as the `.mpk` metadata file).
+* **Missing/Malformed Metadata**\
+A node that doesn't have any (or malformed) metadata.
+
+This command provides additional options:
+
+* `-b` / `--blobstore`\
+Allows specifying the blobstore to use. Defaults to `ocis`. Empty blobs will not be checked. Can also be switched to `s3ng`, but needs addtional envvar configuration (see the `storage-users` service for more details).
+* `--fail`\
+Exits with non-zero exit code if inconsistencies are found. Useful for automation.
+
+### Revisions CLI
+
+The revisions command allows removing the revisions of files in the storage.
+
+```bash
+ocis revisions purge -p /base/path/storage/users
+```
+
+It takes the `--resource-id` (or `--r`) parameter which specify the scope of the command:
+
+* An empty string (default) removes all revisions from all spaces.
+* A spaceID (like `d419032c-65b9-4f4e-b1e4-0c69a946181d\$44b5a63b-540c-4002-a674-0e9c833bbe49`) removes all revisions in that space.
+* A resourceID (e.g. `d419032c-65b9-4f4e-b1e4-0c69a946181d\$44b5a63b-540c-4002-a674-0e9c833bbe49\!e8a73d49-2e00-4322-9f34-9d7f178577b2`) removes all revisions from that specific file.
+
+This command provides additional options:
+
+* `--dry-run` (default: `true`)\
+Do not remove any revisions but print the revisions that would be removed.
+* `-b` / `--blobstore`\
+Allows specifying the blobstore to use. Defaults to `ocis`. Can be switched to `s3ng` but needs addtional envvar configuration (see the `storage-users` service for more details).
+* `-v` / `--verbose`\
+Prints additional information about the revisions that are removed.
+
+### Trash CLI
+
+The trash cli allows removing empty folders from the trashbin. This should be used to speed up trash bin operations.
+
+```bash
+ocis trash purge-empty-dirs -p /base/path/storage/users
+```
+
+This command provides additional options:
+
+* `--dry-run` (default: `true`)\
+Do not remove any empty folders but print the empty folders that would be removed.

ocis/pkg/command/revisions.go

@@ -0,0 +1,152 @@
+package command
+
+import (
+	"errors"
+	"fmt"
+	"path/filepath"
+
+	ocisbs "github.com/cs3org/reva/v2/pkg/storage/fs/ocis/blobstore"
+	"github.com/cs3org/reva/v2/pkg/storage/fs/posix/lookup"
+	s3bs "github.com/cs3org/reva/v2/pkg/storage/fs/s3ng/blobstore"
+	"github.com/cs3org/reva/v2/pkg/storagespace"
+	"github.com/owncloud/ocis/v2/ocis-pkg/config"
+	"github.com/owncloud/ocis/v2/ocis-pkg/config/configlog"
+	"github.com/owncloud/ocis/v2/ocis-pkg/config/parser"
+	"github.com/owncloud/ocis/v2/ocis/pkg/register"
+	"github.com/owncloud/ocis/v2/ocis/pkg/revisions"
+	"github.com/urfave/cli/v2"
+)
+
+var (
+	// _nodesGlobPattern is the glob pattern to find all nodes
+	_nodesGlobPattern = "spaces/*/*/*/*/*/*/*/*"
+)
+
+// RevisionsCommand is the entrypoint for the revisions command.
+func RevisionsCommand(cfg *config.Config) *cli.Command {
+	return &cli.Command{
+		Name:  "revisions",
+		Usage: "ocis revisions functionality",
+		Subcommands: []*cli.Command{
+			PurgeRevisionsCommand(cfg),
+		},
+		Before: func(c *cli.Context) error {
+			return configlog.ReturnError(parser.ParseConfig(cfg, true))
+		},
+		Action: func(_ *cli.Context) error {
+			fmt.Println("Read the docs")
+			return nil
+		},
+	}
+}
+
+// PurgeRevisionsCommand allows removing all revisions from a storage provider.
+func PurgeRevisionsCommand(cfg *config.Config) *cli.Command {
+	return &cli.Command{
+		Name:  "purge",
+		Usage: "purge revisions",
+		Flags: []cli.Flag{
+			&cli.StringFlag{
+				Name:     "basepath",
+				Aliases:  []string{"p"},
+				Usage:    "the basepath of the decomposedfs (e.g. /var/tmp/ocis/storage/metadata)",
+				Required: true,
+			},
+			&cli.StringFlag{
+				Name:    "blobstore",
+				Aliases: []string{"b"},
+				Usage:   "the blobstore type. Can be (none, ocis, s3ng). Default ocis. Note: When using s3ng this needs same configuration as the storage-users service",
+				Value:   "ocis",
+			},
+			&cli.BoolFlag{
+				Name:  "dry-run",
+				Usage: "do not delete anything, just print what would be deleted",
+				Value: true,
+			},
+			&cli.BoolFlag{
+				Name:    "verbose",
+				Aliases: []string{"v"},
+				Usage:   "print verbose output",
+				Value:   false,
+			},
+			&cli.StringFlag{
+				Name:    "resource-id",
+				Aliases: []string{"r"},
+				Usage:   "purge all revisions of this file/space. If not set, all revisions will be purged",
+			},
+		},
+		Action: func(c *cli.Context) error {
+			basePath := c.String("basepath")
+			if basePath == "" {
+				fmt.Println("basepath is required")
+				return cli.ShowCommandHelp(c, "revisions")
+			}
+
+			var (
+				bs  revisions.DelBlobstore
+				err error
+			)
+			switch c.String("blobstore") {
+			case "s3ng":
+				bs, err = s3bs.New(
+					cfg.StorageUsers.Drivers.S3NG.Endpoint,
+					cfg.StorageUsers.Drivers.S3NG.Region,
+					cfg.StorageUsers.Drivers.S3NG.Bucket,
+					cfg.StorageUsers.Drivers.S3NG.AccessKey,
+					cfg.StorageUsers.Drivers.S3NG.SecretKey,
+					s3bs.Options{},
+				)
+			case "ocis":
+				bs, err = ocisbs.New(basePath)
+			case "none":
+				bs = nil
+			default:
+				err = errors.New("blobstore type not supported")
+			}
+			if err != nil {
+				fmt.Println(err)
+				return err
+			}
+
+			p, err := generatePath(basePath, c.String("resource-id"))
+			if err != nil {
+				fmt.Printf("❌ Error parsing resourceID: %s", err)
+				return err
+			}
+
+			if err := revisions.PurgeRevisions(p, bs, c.Bool("dry-run"), c.Bool("verbose")); err != nil {
+				fmt.Printf("❌ Error purging revisions: %s", err)
+				return err
+			}
+
+			return nil
+		},
+	}
+}
+
+func generatePath(basePath string, resourceID string) (string, error) {
+	if resourceID == "" {
+		return filepath.Join(basePath, _nodesGlobPattern), nil
+	}
+
+	rid, err := storagespace.ParseID(resourceID)
+	if err != nil {
+		return "", err
+	}
+
+	sid := lookup.Pathify(rid.GetSpaceId(), 1, 2)
+	if sid == "" {
+		sid = "*/*"
+	}
+
+	nid := lookup.Pathify(rid.GetOpaqueId(), 4, 2)
+	if nid == "" {
+		nid = "*/*/*/*/"
+	}
+
+	return filepath.Join(basePath, "spaces", sid, "nodes", nid+"*"), nil
+}
+
+func init() {
+	register.AddCommand(RevisionsCommand)
+}