make repo gc call CollectGarbage on datastore

License: MIT
Signed-off-by: Łukasz Magiera <[email protected]>

core/corerepo/gc.go

@@ -86,7 +86,7 @@ func GarbageCollect(n *core.IpfsNode, ctx context.Context) error {
 	if err != nil {
 		return err
 	}
-	rmed := gc.GC(ctx, n.Blockstore, n.DAG, n.Pinning, roots)
+	rmed := gc.GC(ctx, n.Blockstore, n.Repo.Datastore(), n.DAG, n.Pinning, roots)
 
 	return CollectResult(ctx, rmed, nil)
 }
@@ -154,7 +154,7 @@ func GarbageCollectAsync(n *core.IpfsNode, ctx context.Context) <-chan gc.Result
 		return out
 	}
 
-	return gc.GC(ctx, n.Blockstore, n.DAG, n.Pinning, roots)
+	return gc.GC(ctx, n.Blockstore, n.Repo.Datastore(), n.DAG, n.Pinning, roots)
 }
 
 func PeriodicGC(ctx context.Context, node *core.IpfsNode) error {

core/coreunix/add_test.go

@@ -102,7 +102,7 @@ func TestAddGCLive(t *testing.T) {
 	gcstarted := make(chan struct{})
 	go func() {
 		defer close(gcstarted)
-		gcout = gc.GC(context.Background(), node.Blockstore, node.DAG, node.Pinning, nil)
+		gcout = gc.GC(context.Background(), node.Blockstore, node.Repo.Datastore(), node.DAG, node.Pinning, nil)
 	}()
 
 	// gc shouldnt start until we let the add finish its current file.

package.json

@@ -208,9 +208,9 @@
     },
     {
       "author": "whyrusleeping",
-      "hash": "QmU7tt6mHJ5Wocjy2omBxpDfN8g9pkRimzJae7EXdrs96k",
+      "hash": "QmRhjB5Mnha4k6VH6qRFNabAVkxpbqC7bVw2daMKLHPXXN",
       "name": "go-ds-measure",
-      "version": "1.2.3"
+      "version": "1.3.0"
     },
     {
       "author": "whyrusleeping",
@@ -470,9 +470,9 @@
     },
     {
       "author": "magik6k",
-      "hash": "Qmdin8YL17fL1BC5ej6o9b8es6MBoiQjKVdyxEwJh3HVmf",
+      "hash": "Qmb1MciZErHUnT1MsyAwUL8FGtZqmRPAXjxU6SktqFUs23",
       "name": "go-ds-badger",
-      "version": "1.3.0"
+      "version": "1.4.1"
     },
     {
       "author": "whyrusleeping",

pin/gc/gc.go

@@ -11,6 +11,7 @@ import (
 
 	node "gx/ipfs/QmNwUEK7QbwSqyKBu3mMtToo8SUc6wQJ7gdZq4gGGJqfnf/go-ipld-format"
 	logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
+	dstore "gx/ipfs/QmdHG8MAuARdGHxx4rPQASLcvhz24fzjSQq7AJRAQEorq5/go-datastore"
 	cid "gx/ipfs/QmeSrf6pzut73u6zLQkRFQ3ygt3k6XFT2kjdYP8Tnkwwyg/go-cid"
 )
 
@@ -33,7 +34,7 @@ type Result struct {
 // The routine then iterates over every block in the blockstore and
 // deletes any block that is not found in the marked set.
 //
-func GC(ctx context.Context, bs bstore.GCBlockstore, ls dag.LinkService, pn pin.Pinner, bestEffortRoots []*cid.Cid) <-chan Result {
+func GC(ctx context.Context, bs bstore.GCBlockstore, ds dstore.Datastore, ls dag.LinkService, pn pin.Pinner, bestEffortRoots []*cid.Cid) <-chan Result {
 
 	elock := log.EventBegin(ctx, "GC.lockWait")
 	unlocker := bs.GCLock()
@@ -104,6 +105,18 @@ func GC(ctx context.Context, bs bstore.GCBlockstore, ls dag.LinkService, pn pin.
 		if errors {
 			output <- Result{Error: ErrCannotDeleteSomeBlocks}
 		}
+
+		defer log.EventBegin(ctx, "GC.datastore").Done()
+		gds, ok := ds.(dstore.GCDatastore)
+		if !ok {
+			return
+		}
+
+		err = gds.CollectGarbage()
+		if err != nil {
+			output <- Result{Error: err}
+			return
+		}
 	}()
 
 	return output

repo/fsrepo/datastores.go

@@ -10,15 +10,16 @@ import (
 
 	repo "github.com/ipfs/go-ipfs/repo"
 
-	measure "gx/ipfs/QmU7tt6mHJ5Wocjy2omBxpDfN8g9pkRimzJae7EXdrs96k/go-ds-measure"
+	measure "gx/ipfs/QmRhjB5Mnha4k6VH6qRFNabAVkxpbqC7bVw2daMKLHPXXN/go-ds-measure"
 	flatfs "gx/ipfs/Qmak5iFUErGmKhYUC5StZtCGFp5bdhvye8PdKgTMk9B9Fw/go-ds-flatfs"
 
 	ds "gx/ipfs/QmdHG8MAuARdGHxx4rPQASLcvhz24fzjSQq7AJRAQEorq5/go-datastore"
 	mount "gx/ipfs/QmdHG8MAuARdGHxx4rPQASLcvhz24fzjSQq7AJRAQEorq5/go-datastore/syncmount"
 
+	humanize "gx/ipfs/QmPSBJL4momYnE7DcUyk2DVhD6rH488ZmHBGLbxNdhU44K/go-humanize"
 	levelds "gx/ipfs/QmYnCBXxoyoS38vtNQjjpRwZTiUnpuuKpapxMNaDfyQRLf/go-ds-leveldb"
+	badgerds "gx/ipfs/Qmb1MciZErHUnT1MsyAwUL8FGtZqmRPAXjxU6SktqFUs23/go-ds-badger"
 	ldbopts "gx/ipfs/QmbBhyDKsY4mbY6xsKt3qu9Y7FPvMJ6qbD8AMjYYvPRw1g/goleveldb/leveldb/opt"
-	badgerds "gx/ipfs/Qmdin8YL17fL1BC5ej6o9b8es6MBoiQjKVdyxEwJh3HVmf/go-ds-badger"
 )
 
 // ConfigFromMap creates a new datastore config from a map
@@ -342,6 +343,8 @@ func (c measureDatastoreConfig) Create(path string) (repo.Datastore, error) {
 type badgerdsDatastoreConfig struct {
 	path       string
 	syncWrites bool
+
+	vlogFileSize int64
 }
 
 // BadgerdsDatastoreConfig returns a configuration stub for a badger datastore
@@ -366,6 +369,22 @@ func BadgerdsDatastoreConfig(params map[string]interface{}) (DatastoreConfig, er
 		}
 	}
 
+	vls, ok := params["vlogFileSize"]
+	if !ok {
+		// default to 1GiB
+		c.vlogFileSize = badgerds.DefaultOptions.ValueLogFileSize
+	} else {
+		if vlogSize, ok := vls.(string); ok {
+			s, err := humanize.ParseBytes(vlogSize)
+			if err != nil {
+				return nil, err
+			}
+			c.vlogFileSize = int64(s)
+		} else {
+			return nil, fmt.Errorf("'vlogFileSize' field was not a string")
+		}
+	}
+
 	return &c, nil
 }
 
@@ -389,6 +408,7 @@ func (c *badgerdsDatastoreConfig) Create(path string) (repo.Datastore, error) {
 
 	defopts := badgerds.DefaultOptions
 	defopts.SyncWrites = c.syncWrites
+	defopts.ValueLogFileSize = c.vlogFileSize
 
 	return badgerds.NewDatastore(p, &defopts)
 }

repo/fsrepo/fsrepo.go

@@ -24,8 +24,8 @@ import (
 	"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/mitchellh/go-homedir"
 
 	util "gx/ipfs/QmPsAfmDBnZN3kZGSuNwvCNDZiHneERSKmRcFyG3UkvcT3/go-ipfs-util"
+	measure "gx/ipfs/QmRhjB5Mnha4k6VH6qRFNabAVkxpbqC7bVw2daMKLHPXXN/go-ds-measure"
 	logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
-	measure "gx/ipfs/QmU7tt6mHJ5Wocjy2omBxpDfN8g9pkRimzJae7EXdrs96k/go-ds-measure"
 	ma "gx/ipfs/QmW8s4zTsUoX1Q6CeYxVKPyqSKbF7H1YDUyTostBtZ8DaG/go-multiaddr"
 )