Skip to content

Commit

Permalink
Merge pull request #9497 from kobergj/RemoveVersionsCli
Browse files Browse the repository at this point in the history
Purge-Revision Cli
  • Loading branch information
dragonchaser authored Jul 8, 2024
2 parents c4101fa + a08860c commit 6b71bfe
Show file tree
Hide file tree
Showing 4 changed files with 393 additions and 1 deletion.
5 changes: 5 additions & 0 deletions changelog/unreleased/remove-revisions-cli.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
Enhancement: Add cli to purge revisions

Adds a cli that allows removing all revisions for a storage-provider.

https://github.com/owncloud/ocis/pull/9497
80 changes: 79 additions & 1 deletion ocis/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,4 +23,82 @@ To authenticate the connection to the etcd registry, you have to set `ETCD_USERN

## Memory limits

oCIS will automatically set the go native `GOMEMLIMIT` to `0.9`. To disable the limit set `AUTOMEMEMLIMIT=off`. For more information take a look at the official [Guide to the Go Garbage Collector](https://go.dev/doc/gc-guide).
oCIS will automatically set the go native `GOMEMLIMIT` to `0.9`. To disable the limit set `AUTOMEMEMLIMIT=off`. For more information take a look at the official [Guide to the Go Garbage Collector](https://go.dev/doc/gc-guide).

## CLI Commands

The ocis package offers a variety of cli commands to monitor or repair ocis installations. All these commands have a common mandatory parameter: `--basePath` (or `-p`) which needs to point to a storage provider. Example paths are:

```bash
.ocis/storage/users # bare metal installation
/var/tmp/ocis/storage/users # docker installation
...
```

These paths can vary depending on your ocis installation.

### Backup CLI

The backup command allows inspecting the consistency of an ocis storage:

```bash
ocis backup consistency -p /base/path/storage/users
```

This will check the consistency of the storage and output a list of inconsistencies. Inconsistencies can be:

* **Orphaned Blobs**\
A blob in the blobstore that is not referenced by any file metadata.
* **Missing Blobs**\
A blob referenced by file metadata that is not present in the blobstore.
* **Missing Nodes**\
A node that is referenced by a symlink but doesn't exist.
* **Missing Link**\
A node that is not referenced by any symlink but should be.
* **Missing Files**\
A node that is missing essential files (such as the `.mpk` metadata file).
* **Missing/Malformed Metadata**\
A node that doesn't have any (or malformed) metadata.

This command provides additional options:

* `-b` / `--blobstore`\
Allows specifying the blobstore to use. Defaults to `ocis`. Empty blobs will not be checked. Can also be switched to `s3ng`, but needs addtional envvar configuration (see the `storage-users` service for more details).
* `--fail`\
Exits with non-zero exit code if inconsistencies are found. Useful for automation.

### Revisions CLI

The revisions command allows removing the revisions of files in the storage.

```bash
ocis revisions purge -p /base/path/storage/users
```

It takes the `--resource-id` (or `--r`) parameter which specify the scope of the command:

* An empty string (default) removes all revisions from all spaces.
* A spaceID (like `d419032c-65b9-4f4e-b1e4-0c69a946181d\$44b5a63b-540c-4002-a674-0e9c833bbe49`) removes all revisions in that space.
* A resourceID (e.g. `d419032c-65b9-4f4e-b1e4-0c69a946181d\$44b5a63b-540c-4002-a674-0e9c833bbe49\!e8a73d49-2e00-4322-9f34-9d7f178577b2`) removes all revisions from that specific file.

This command provides additional options:

* `--dry-run` (default: `true`)\
Do not remove any revisions but print the revisions that would be removed.
* `-b` / `--blobstore`\
Allows specifying the blobstore to use. Defaults to `ocis`. Can be switched to `s3ng` but needs addtional envvar configuration (see the `storage-users` service for more details).
* `-v` / `--verbose`\
Prints additional information about the revisions that are removed.

### Trash CLI

The trash cli allows removing empty folders from the trashbin. This should be used to speed up trash bin operations.

```bash
ocis trash purge-empty-dirs -p /base/path/storage/users
```

This command provides additional options:

* `--dry-run` (default: `true`)\
Do not remove any empty folders but print the empty folders that would be removed.
152 changes: 152 additions & 0 deletions ocis/pkg/command/revisions.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,152 @@
package command

import (
"errors"
"fmt"
"path/filepath"

ocisbs "github.com/cs3org/reva/v2/pkg/storage/fs/ocis/blobstore"
"github.com/cs3org/reva/v2/pkg/storage/fs/posix/lookup"
s3bs "github.com/cs3org/reva/v2/pkg/storage/fs/s3ng/blobstore"
"github.com/cs3org/reva/v2/pkg/storagespace"
"github.com/owncloud/ocis/v2/ocis-pkg/config"
"github.com/owncloud/ocis/v2/ocis-pkg/config/configlog"
"github.com/owncloud/ocis/v2/ocis-pkg/config/parser"
"github.com/owncloud/ocis/v2/ocis/pkg/register"
"github.com/owncloud/ocis/v2/ocis/pkg/revisions"
"github.com/urfave/cli/v2"
)

var (
// _nodesGlobPattern is the glob pattern to find all nodes
_nodesGlobPattern = "spaces/*/*/*/*/*/*/*/*"
)

// RevisionsCommand is the entrypoint for the revisions command.
func RevisionsCommand(cfg *config.Config) *cli.Command {
return &cli.Command{
Name: "revisions",
Usage: "ocis revisions functionality",
Subcommands: []*cli.Command{
PurgeRevisionsCommand(cfg),
},
Before: func(c *cli.Context) error {
return configlog.ReturnError(parser.ParseConfig(cfg, true))
},
Action: func(_ *cli.Context) error {
fmt.Println("Read the docs")
return nil
},
}
}

// PurgeRevisionsCommand allows removing all revisions from a storage provider.
func PurgeRevisionsCommand(cfg *config.Config) *cli.Command {
return &cli.Command{
Name: "purge",
Usage: "purge revisions",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "basepath",
Aliases: []string{"p"},
Usage: "the basepath of the decomposedfs (e.g. /var/tmp/ocis/storage/metadata)",
Required: true,
},
&cli.StringFlag{
Name: "blobstore",
Aliases: []string{"b"},
Usage: "the blobstore type. Can be (none, ocis, s3ng). Default ocis. Note: When using s3ng this needs same configuration as the storage-users service",
Value: "ocis",
},
&cli.BoolFlag{
Name: "dry-run",
Usage: "do not delete anything, just print what would be deleted",
Value: true,
},
&cli.BoolFlag{
Name: "verbose",
Aliases: []string{"v"},
Usage: "print verbose output",
Value: false,
},
&cli.StringFlag{
Name: "resource-id",
Aliases: []string{"r"},
Usage: "purge all revisions of this file/space. If not set, all revisions will be purged",
},
},
Action: func(c *cli.Context) error {
basePath := c.String("basepath")
if basePath == "" {
fmt.Println("basepath is required")
return cli.ShowCommandHelp(c, "revisions")
}

var (
bs revisions.DelBlobstore
err error
)
switch c.String("blobstore") {
case "s3ng":
bs, err = s3bs.New(
cfg.StorageUsers.Drivers.S3NG.Endpoint,
cfg.StorageUsers.Drivers.S3NG.Region,
cfg.StorageUsers.Drivers.S3NG.Bucket,
cfg.StorageUsers.Drivers.S3NG.AccessKey,
cfg.StorageUsers.Drivers.S3NG.SecretKey,
s3bs.Options{},
)
case "ocis":
bs, err = ocisbs.New(basePath)
case "none":
bs = nil
default:
err = errors.New("blobstore type not supported")
}
if err != nil {
fmt.Println(err)
return err
}

p, err := generatePath(basePath, c.String("resource-id"))
if err != nil {
fmt.Printf("❌ Error parsing resourceID: %s", err)
return err
}

if err := revisions.PurgeRevisions(p, bs, c.Bool("dry-run"), c.Bool("verbose")); err != nil {
fmt.Printf("❌ Error purging revisions: %s", err)
return err
}

return nil
},
}
}

func generatePath(basePath string, resourceID string) (string, error) {
if resourceID == "" {
return filepath.Join(basePath, _nodesGlobPattern), nil
}

rid, err := storagespace.ParseID(resourceID)
if err != nil {
return "", err
}

sid := lookup.Pathify(rid.GetSpaceId(), 1, 2)
if sid == "" {
sid = "*/*"
}

nid := lookup.Pathify(rid.GetOpaqueId(), 4, 2)
if nid == "" {
nid = "*/*/*/*/"
}

return filepath.Join(basePath, "spaces", sid, "nodes", nid+"*"), nil
}

func init() {
register.AddCommand(RevisionsCommand)
}
Loading

0 comments on commit 6b71bfe

Please sign in to comment.