Merge pull request #120 from power-more/zhaoshang/blobmgr

Fix BlobMgr Remove

.github/workflows/image.yml

@@ -119,6 +119,7 @@ jobs:
         run: |
           # start nydus-snapshotter
           docker run -d --name snapshotter --device /dev/fuse --cap-add SYS_ADMIN --security-opt apparmor:unconfined -e BACKEND_TYPE=localfs -e CONTAINERD_ROOT=/var/lib/containerd-test -v /var/lib/containerd-test:/var/lib/containerd-test:shared -v ~/.docker/config.json:/root/.docker/config.json:shared ${{ env.REGISTRY }}/${{ env.ORGANIZATION }}/nydus-snapshotter:local
+          docker_snapshotter=$(docker ps -q)
           # start containerd
           sudo /usr/local/bin/containerd --config /etc/containerd/containerd-test-config.toml -l debug &
           # wait for containerd to start up
@@ -133,6 +134,10 @@ jobs:
           sudo crictl exec $container ls
           echo "delete pod"
           sudo crictl rmp -fa
+          echo "delete nydus-latest image"
+          sudo crictl rmi ghcr.io/dragonflyoss/image-service/ubuntu:nydus-latest
+          echo "check whether the blobs of nydus-latest image has been deleted"
+          docker exec -i $docker_snapshotter sh -c 'if [ "`ls -A /tmp/blobs/`" != "" ]; then exit 1; fi'
       - name: Cleanup containerd and snapshotter
         if: ${{ github.event_name == 'pull_request' }}
         run: |

misc/snapshotter/Dockerfile

@@ -9,7 +9,7 @@ RUN mv nydus-static/* /; mv nydusd-fusedev nydusd
 FROM ubuntu:20.04
 
 WORKDIR /root/
-RUN mkdir -p /usr/local/bin/ /etc/nydus/ /var/lib/nydus/cache/
+RUN mkdir -p /usr/local/bin/ /etc/nydus/ /var/lib/nydus/cache/ /tmp/blobs/
 COPY --from=sourcer /nydusd /nydus-image /usr/local/bin/
 COPY containerd-nydus-grpc /usr/local/bin/
 COPY nydusd-config.fusedev.json /etc/nydus/config.json

misc/snapshotter/nydusd-config-localfs.json

@@ -3,7 +3,7 @@
       "backend": {
         "type": "localfs",
         "config": {
-            "dir": "/tmp/"
+            "dir": "/tmp/blobs/"
         }
       },
       "cache": {

pkg/filesystem/fs/fs.go

@@ -144,11 +144,11 @@ func NewFileSystem(ctx context.Context, opt ...NewFSOpt) (*Filesystem, error) {
 	return &fs, nil
 }
 
-func (fs *Filesystem) CleanupBlobLayer(ctx context.Context, key string, async bool) error {
+func (fs *Filesystem) CleanupBlobLayer(ctx context.Context, blobDigest string, async bool) error {
 	if fs.blobMgr == nil {
 		return nil
 	}
-	return fs.blobMgr.Remove(key, async)
+	return fs.blobMgr.Remove(blobDigest, async)
 }
 
 func (fs *Filesystem) PrepareBlobLayer(ctx context.Context, snapshot storage.Snapshot, labels map[string]string) error {

snapshot/snapshot.go

@@ -402,9 +402,10 @@ func (o *snapshotter) Remove(ctx context.Context, key string) error {
 	if err != nil {
 		return errors.Wrap(err, "failed to remove")
 	}
-	var cleanupBlob bool
-	if _, ok := snap.Labels[label.NydusDataLayer]; ok {
-		cleanupBlob = true
+	var blobDigest string
+	_, cleanupBlob := snap.Labels[label.NydusDataLayer]
+	if cleanupBlob {
+		blobDigest = snap.Labels[label.CRILayerDigest]
 	}
 
 	if o.syncRemove {
@@ -425,7 +426,7 @@ func (o *snapshotter) Remove(ctx context.Context, key string) error {
 					}
 				}
 				if cleanupBlob {
-					if err := o.fs.CleanupBlobLayer(ctx, key, false); err != nil {
+					if err := o.fs.CleanupBlobLayer(ctx, blobDigest, false); err != nil {
 						log.G(ctx).WithError(err).WithField("id", key).Warn("failed to remove blob")
 					}
 				}
@@ -434,7 +435,7 @@ func (o *snapshotter) Remove(ctx context.Context, key string) error {
 	} else {
 		defer func() {
 			if cleanupBlob {
-				if err := o.fs.CleanupBlobLayer(ctx, key, true); err != nil {
+				if err := o.fs.CleanupBlobLayer(ctx, blobDigest, true); err != nil {
 					log.G(ctx).WithError(err).WithField("id", key).Warn("failed to remove blob")
 				}
 			}