Handle partial failure on deleting blocks to leave the manifest

basho · Jul 7, 2015 · 0e88aa6 · 0e88aa6
1 parent 327557d
commit 0e88aa6
Showing 1 changed file with 39 additions and 24 deletions.
diff --git a/src/riak_cs_gc.erl b/src/riak_cs_gc.erl
@@ -386,38 +386,18 @@ mark_manifests(RiakObject, Bucket, Key, UUIDsToMark, ManiFunction, RcPid) ->
                                         {[cs_uuid_and_manifest()], [cs_uuid()]}.
 maybe_delete_small_objects(Manifests, RcPid, Threshold) ->
     {ok, BagId} = riak_cs_riak_client:get_manifest_bag(RcPid),
-    Self = self(),
-    FinishedFun = fun(Msg) -> Self ! Msg end,
     DelFun= fun({UUID, Manifest = ?MANIFEST{state=pending_delete,
                                             content_length=ContentLength}},
                 {Survivors, UUIDsToDelete})
                   when ContentLength < Threshold ->
                     %% actually this won't be scheduled :P
                     UUIDManifest = {UUID, Manifest?MANIFEST{state=scheduled_delete}},
                     _ = lager:debug("trying to delete ~p at ~p", [UUIDManifest, BagId]),
-                    Args = [BagId, UUIDManifest, FinishedFun,
-                            dummy_gc_key_in_sync_delete,
-                            [{cleanup_manifests, false}]],
-                    {ok, Pid} = riak_cs_delete_fsm_sup:start_delete_fsm(node(), Args),
-                    Ref = erlang:monitor(process, Pid),
-                    receive
-                        {Pid, {ok, _}} ->
-                            %% successfully deleted
-                            erlang:demonitor(Ref, [flush]),
-                            _ = lager:debug("Active deletion of ~p succeeded", [UUID]),
+                    case try_delete_blocks(BagId, UUIDManifest) of
+                        ok ->
                             {Survivors, [UUID|UUIDsToDelete]};
-                        {Pid, {error, _} = E} ->
-                            erlang:demonitor(Ref, [flush]),
-                            _ = lager:warning("Active deletion of ~p failed. Reason: ~p",
-                                              [UUID, E]),
-                            {[{UUID, Manifest}|Survivors], UUIDsToDelete};
-                        {'DOWN', Ref, _Type, Pid, Reason} ->
-                            _ = lager:warning("Delete FSM for ~p crashed. Reason: ~p", [Reason]),
-                            {[{UUID, Manifest}|Survivors], UUIDsToDelete};
-                        Other ->
-                            %% Handling unknown error, or died unexpectedly
-                            erlang:demonitor(Ref, [flush]),
-                            _ = lager:error("Active deletion failed. Reason: ~p", [Other]),
+                        {error, _} ->
+                            %% Error logs were handled in the function
                             {[{UUID, Manifest}|Survivors], UUIDsToDelete}
                     end;
                ({UUID, M}, {Survivors, UUIDsToDelete}) ->
@@ -429,6 +409,41 @@ maybe_delete_small_objects(Manifests, RcPid, Threshold) ->
     %% Obtain a new history!
     lists:foldl(DelFun, {[], []}, Manifests).
 
+try_delete_blocks(BagId, {UUID, _} = UUIDManifest) ->
+    Self = self(),
+    FinishedFun = fun(Msg) -> Self ! Msg end,
+    Args = [BagId, UUIDManifest, FinishedFun,
+            dummy_gc_key_in_sync_delete,
+            [{cleanup_manifests, false}]],
+    {ok, Pid} = riak_cs_delete_fsm_sup:start_delete_fsm(node(), Args),
+    Ref = erlang:monitor(process, Pid),
+    receive
+        {Pid, {ok, {_, _, _, TotalBlocks, TotalBlocks}}} ->
+            %% successfully deleted
+            erlang:demonitor(Ref, [flush]),
+            _ = lager:debug("Active deletion of ~p succeeded", [UUID]),
+            ok;
+        {Pid, {ok, {_, _, _, NumDeleted, TotalBlocks}}} ->
+            erlang:demonitor(Ref, [flush]),
+            _ = lager:debug("Only ~p/~p blocks of ~p deleted",
+                            [NumDeleted, TotalBlocks, UUID]),
+            {error, partially_deleted};
+        {Pid, {error, _} = E} ->
+            erlang:demonitor(Ref, [flush]),
+            _ = lager:warning("Active deletion of ~p failed. Reason: ~p",
+                              [UUID, E]),
+            E;
+        {'DOWN', Ref, _Type, Pid, Reason} ->
+            _ = lager:warning("Delete FSM for ~p crashed. Reason: ~p", [Reason]),
+            {error, Reason};
+        Other ->
+            %% Handling unknown error, or died unexpectedly
+            erlang:demonitor(Ref, [flush]),
+            _ = lager:error("Active deletion failed. Reason: ~p", [Other]),
+            {error, Other}
+    end.
+
+
 %% @doc Copy data for a list of manifests to the
 %% `riak-cs-gc' bucket to schedule them for deletion.
 -spec move_manifests_to_gc_bucket([cs_uuid_and_manifest()], riak_client()) ->