Skip to content

Commit

Permalink
Fix leo-project/leofs#207 ,again
Browse files Browse the repository at this point in the history
  • Loading branch information
@yosukehara
yosukehara committed Jul 29, 2014
1 parent 20cc1ac commit 8c27292
Show file tree
Hide file tree
Showing 3 changed files with 54 additions and 23 deletions.
34 changes: 16 additions & 18 deletions src/leo_object_storage_haystack.erl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -655,15 +655,14 @@ compact_get(#?METADATA{ksize = KSize,
end
catch
_:Reason ->
{error, Reason}
{error, {?LINE, Reason}}
end
end.

%% @private
compact_get_1(_ReadHandler,_HeaderBin, #?METADATA{ksize = 0},_DSize,_Bin,_TotalSize) ->
{error, {?LINE, ?ERROR_DATA_SIZE_DID_NOT_MATCH}};
compact_get_1(ReadHandler, HeaderBin, #?METADATA{ksize = KSize,
msize = 0} = Metadata,
compact_get_1(ReadHandler, HeaderBin, #?METADATA{ksize = KSize} = Metadata,
DSize, Bin, TotalSize) ->
<< KeyBin:KSize/binary,
BodyBin:DSize/binary,
Expand All @@ -676,29 +675,32 @@ compact_get_2(ReadHandler, HeaderBin, Metadata, KeyBin, BodyBin, TotalSize) ->
Checksum_1 = leo_hex:raw_binary_to_integer(crypto:hash(md5, BodyBin)),
Metadata_1 = Metadata#?METADATA{key = KeyBin},

case (Checksum == Checksum_1 orelse
Checksum_1 == ?MD5_EMPTY_BIN) of
case (Checksum == Checksum_1) of
true ->
HeaderSize = erlang:round(?BLEN_HEADER/8),
Offset = Metadata#?METADATA.offset,
KSize = Metadata#?METADATA.ksize,
DSize = Metadata#?METADATA.dsize,
MSize = Metadata#?METADATA.msize,
#?METADATA{offset = Offset,
ksize = KSize,
dsize = DSize,
msize = MSize} = Metadata,

case (?MAX_DATABLOCK_SIZE < MSize andalso MSize > 0) of
case (?MAX_DATABLOCK_SIZE > MSize andalso MSize > 0) of
true ->
MPos = Offset + HeaderSize + KSize + DSize,
case file:pread(ReadHandler, MPos, MSize) of
{ok, CMetaBin} ->
case leo_object_storage_transformer:cmeta_bin_into_metadata(
CMetaBin, Metadata) of
CMetaBin, Metadata_1) of
{error,_Cause} ->
{ok, Metadata_1#?METADATA{msize = 0},
[HeaderBin, KeyBin, BodyBin, TotalSize]};
Metadata_2 ->
{ok, Metadata_2#?METADATA{key = KeyBin},
[HeaderBin, KeyBin, BodyBin, TotalSize + MSize]}
{ok, Metadata_2,
[HeaderBin, KeyBin, BodyBin,
Offset + HeaderSize + KSize
+ DSize + MSize + ?LEN_PADDING]}
end;
eof = Cause ->
{error, Cause};
{error,_Cause} ->
{error, invalid_data}
end;
Expand All @@ -708,9 +710,5 @@ compact_get_2(ReadHandler, HeaderBin, Metadata, KeyBin, BodyBin, TotalSize) ->
end;
false ->
Reason = ?ERROR_INVALID_DATA,
error_logger:error_msg("~p,~p,~p,~p~n",
[{module, ?MODULE_STRING},
{function, "compact_get_2/4"},
{line, ?LINE}, {body, Reason}]),
{error, Reason}
{error, {?LINE, Reason}}
end.
1 change: 1 addition & 0 deletions src/leo_object_storage_transformer.erl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ metadata_to_object(#?METADATA{} = Metadata) ->
num_of_replicas = NumOfReplicas,
ver = Ver,
del = Del} = Metadata,

#?OBJECT{key = Key,
addr_id = AddrId,
ksize = KSize,
Expand Down
42 changes: 37 additions & 5 deletions test/leo_object_storage_api_tests.erl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,26 +68,28 @@ compact() ->
ok = put_irregular_bin(),
ok = put_regular_bin(36, 25),
ok = put_irregular_bin(),
ok = put_regular_bin(51, 50),
ok = put_regular_bin_with_cmeta(51, 50),
ok = put_irregular_bin(),
ok = put_regular_bin(101, 50),
ok = put_irregular_bin(),
ok = put_regular_bin(136, 25),
ok = put_irregular_bin(),
ok = put_regular_bin(151, 50),
ok = put_regular_bin_with_cmeta(151, 50),
ok = put_irregular_bin(),
ok = put_regular_bin(201, 50),
ok = put_irregular_bin(),
ok = put_regular_bin(236, 25),
ok = put_irregular_bin(),
ok = put_regular_bin(251, 50),
ok = put_regular_bin_with_cmeta(251, 50),
ok = put_irregular_bin(),
ok = put_regular_bin(301, 50),
ok = put_irregular_bin(),
ok = put_regular_bin(336, 25),
ok = put_irregular_bin(),
ok = put_regular_bin(351, 50),
ok = put_irregular_bin(),
ok = put_regular_bin_with_cmeta(401, 10),
ok = put_irregular_bin(),

%% Execute compaction
timer:sleep(3000),
Expand All @@ -106,13 +108,13 @@ compact() ->
active_num = ActiveNum
}}|_]} = leo_object_storage_api:stats(),
?debugVal({TotalNum, ActiveNum}),
?assertEqual(400, TotalNum),
?assertEqual(410, TotalNum),
?assertEqual(TotalNum, ActiveNum),
ok.

check_status() ->
timer:sleep(100),
case leo_compaction_manager_fsm:status() of
case leo_compaction_manager_fsm:status() of
{ok, #compaction_stats{status = 'idle'}} ->
ok;
{ok, _} ->
Expand Down Expand Up @@ -142,6 +144,36 @@ put_regular_bin(Index, Counter) ->
{ok, _} = leo_object_storage_api:put({AddrId, Key}, Object),
put_regular_bin(Index + 1, Counter -1).


%% @doc Put data with custom-metadata
%% @private
put_regular_bin_with_cmeta(_, 0) ->
ok;
put_regular_bin_with_cmeta(Index, Counter) ->
AddrId = 1,
Key = list_to_binary(lists:append(["TEST_", integer_to_list(Index)])),
Bin = crypto:rand_bytes(erlang:phash2(leo_date:clock(), (1024 * 1024))),
CMetaBin = leo_object_storage_transformer:list_to_cmeta_bin(
[{?PROP_CMETA_CLUSTER_ID, 'remote_cluster'},
{?PROP_CMETA_NUM_OF_REPLICAS, 3}]),
Object = #?OBJECT{method = put,
addr_id = AddrId,
key = Key,
ksize = byte_size(Key),
data = Bin,
dsize = byte_size(Bin),
meta = CMetaBin,
msize = byte_size(CMetaBin),
checksum = leo_hex:raw_binary_to_integer(crypto:hash(md5, Bin)),
timestamp = leo_date:now(),
clock = leo_date:clock()
},
{ok, _} = leo_object_storage_api:put({AddrId, Key}, Object),
put_regular_bin_with_cmeta(Index + 1, Counter -1).


%% @doc Put irregular data
%% @private
put_irregular_bin() ->
Min = 1024 * 16,
Len = case erlang:phash2(leo_date:clock(), (1024 * 512)) of
Expand Down

0 comments on commit 8c27292

Please sign in to comment.