From d7c6e4dcd55a90436b461cc127e75837836d2863 Mon Sep 17 00:00:00 2001 From: Nick Marino Date: Tue, 21 Feb 2017 11:57:19 -0500 Subject: [PATCH 1/9] Add s3_lifecycle_hooks skeleton --- tests/s3_lifecycle_hooks.erl | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 tests/s3_lifecycle_hooks.erl diff --git a/tests/s3_lifecycle_hooks.erl b/tests/s3_lifecycle_hooks.erl new file mode 100644 index 000000000..c530f3331 --- /dev/null +++ b/tests/s3_lifecycle_hooks.erl @@ -0,0 +1,26 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2017 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- + +-module(s3_lifecycle_hooks). + +-export([confirm/0]). + +confirm() -> + pass. From 82ab46c7ab11f608b6ef031bc716d7af78706023 Mon Sep 17 00:00:00 2001 From: Fred Dushin Date: Tue, 21 Feb 2017 16:35:35 -0500 Subject: [PATCH 2/9] Added skeletal support for creating a (mocked) S3 bucket and putting a (mock) S3 object in it. --- tests/s3_lifecycle_hooks.erl | 52 ++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/tests/s3_lifecycle_hooks.erl b/tests/s3_lifecycle_hooks.erl index c530f3331..0872a581e 100644 --- a/tests/s3_lifecycle_hooks.erl +++ b/tests/s3_lifecycle_hooks.erl @@ -20,7 +20,59 @@ -module(s3_lifecycle_hooks). +%% +%% This test uses some standard Riak APIs to create what the S3 web facade +%% would otherwise create via its REST APIs, specifically, a Riak +%% + -export([confirm/0]). +-define(BUCKET_TYPE, <<"s3_lifecycle_hooks">>). + +-define(CONFIG, [ + {riak_core, [ + {ring_creation_size, 8}, + {handoff_concurrency, 10}, + {vnode_management_timer, 1000} + ]}, + {riak_s3_api, [ + {lifecycle_hook_url, "http://localhost:8765/lifecycle_hook"} + %% TODO Add magic incantations for lifecycle sweeper + ]} +]). +-define(NUM_NODES, 1). + + confirm() -> + %% + %% Build the cluster + %% + Cluster = rt:build_cluster(?NUM_NODES, ?CONFIG), + Node = lists:nth(random:uniform(length((Cluster))), Cluster), + %% + %% Create a mock S3 "bucket" (aka Riak bucket type) with + %% + rt:create_and_activate_bucket_type(Node, ?BUCKET_TYPE, [{riak_s3_lifecycle, [{'standard_ia.days', 1}]}]), + rt:wait_until_bucket_type_visible(Cluster, ?BUCKET_TYPE), + + Bucket = rpc:call(Node, riak_s3_bucket, to_riak_bucket, [?BUCKET_TYPE]), + lager:info("Bucket: ~p", [Bucket]), + %% + %% Populate an S3 "object" TODO refactor as needed -- maybe lots of puts? + %% + Client = rt:pbc(Node), + _Ret = riakc_pb_socket:put( + Client, riakc_obj:new( + Bucket, <<"test_key">>, <<"test_value">> + ) + ), + %% + %% confirm webhook has been called TODO + %% + + + + + + pass. From d96c4592735cc5afebda24e6a5ca32ce19bacdaf Mon Sep 17 00:00:00 2001 From: Nick Marino Date: Wed, 22 Feb 2017 13:09:23 -0500 Subject: [PATCH 3/9] Add sweeper settings to s3_lifecycle_hooks test --- tests/s3_lifecycle_hooks.erl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/s3_lifecycle_hooks.erl b/tests/s3_lifecycle_hooks.erl index 0872a581e..f6b9d35c9 100644 --- a/tests/s3_lifecycle_hooks.erl +++ b/tests/s3_lifecycle_hooks.erl @@ -35,9 +35,12 @@ {handoff_concurrency, 10}, {vnode_management_timer, 1000} ]}, + {riak_kv, [ + {sweep_tick, 1000} + ]}, {riak_s3_api, [ - {lifecycle_hook_url, "http://localhost:8765/lifecycle_hook"} - %% TODO Add magic incantations for lifecycle sweeper + {lifecycle_hook_url, "http://localhost:8765/lifecycle_hook"}, + {lifecycle_sweep_interval, 1} ]} ]). -define(NUM_NODES, 1). From 2619ec71a91481c483f8214c7f2eba8945050fc0 Mon Sep 17 00:00:00 2001 From: Nick Marino Date: Wed, 22 Feb 2017 17:57:17 -0500 Subject: [PATCH 4/9] WIP add webhook server The hook isn't getting hit yet. Not sure why. But the test still makes it through everything else fine, so I'm committing the code so far. --- tests/s3_lifecycle_hooks.erl | 36 ++++++++++++++++++++++++++++-------- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/tests/s3_lifecycle_hooks.erl b/tests/s3_lifecycle_hooks.erl index f6b9d35c9..e45fb8f4d 100644 --- a/tests/s3_lifecycle_hooks.erl +++ b/tests/s3_lifecycle_hooks.erl @@ -29,6 +29,10 @@ -define(BUCKET_TYPE, <<"s3_lifecycle_hooks">>). +-define(WEBHOOK_PATH, "/lifecycle_hook"). +-define(WEBHOOK_PORT, 8765). +-define(WEBHOOK_URL, "http://localhost:" ++ integer_to_list(?WEBHOOK_PORT) ++ ?WEBHOOK_PATH). + -define(CONFIG, [ {riak_core, [ {ring_creation_size, 8}, @@ -39,7 +43,7 @@ {sweep_tick, 1000} ]}, {riak_s3_api, [ - {lifecycle_hook_url, "http://localhost:8765/lifecycle_hook"}, + {lifecycle_hook_url, ?WEBHOOK_URL}, {lifecycle_sweep_interval, 1} ]} ]). @@ -60,7 +64,11 @@ confirm() -> Bucket = rpc:call(Node, riak_s3_bucket, to_riak_bucket, [?BUCKET_TYPE]), lager:info("Bucket: ~p", [Bucket]), - %% + + %% Other misc setup: + lager:info("Starting mock lifecycle webhook server", []), + start_webhook_server(), + %% Populate an S3 "object" TODO refactor as needed -- maybe lots of puts? %% Client = rt:pbc(Node), @@ -69,13 +77,25 @@ confirm() -> Bucket, <<"test_key">>, <<"test_value">> ) ), - %% - %% confirm webhook has been called TODO - %% - - - + %% TODO Force a sweep of our object's partition: + %% + %% confirm webhook has been called + %% + receive + {got_http_req, Req} -> + lager:info("got request ~p ~p", [Req:get(method), Req:get(path)]) + after + 60000 -> + lager:info("failed to get response", []) + end, pass. + +start_webhook_server() -> + TestPid = self(), + Loop = fun(Req) -> + TestPid ! {got_http_req, Req} + end, + mochiweb_http:start([{name, ?MODULE}, {loop, Loop}, {port, ?WEBHOOK_PORT}]). From 04a994acaff42c9d4354c5a2b7b466462a452b8d Mon Sep 17 00:00:00 2001 From: Nick Marino Date: Wed, 22 Feb 2017 18:29:06 -0500 Subject: [PATCH 5/9] Use debug setting to force fast lifecycle expiry --- tests/s3_lifecycle_hooks.erl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/s3_lifecycle_hooks.erl b/tests/s3_lifecycle_hooks.erl index e45fb8f4d..a78565068 100644 --- a/tests/s3_lifecycle_hooks.erl +++ b/tests/s3_lifecycle_hooks.erl @@ -44,7 +44,9 @@ ]}, {riak_s3_api, [ {lifecycle_hook_url, ?WEBHOOK_URL}, - {lifecycle_sweep_interval, 1} + {lifecycle_sweep_interval, 1}, + %% Make objects "expire" instantly, as soon as a sweep happens: + {debug_lifecycle_expiration_bypass, true} ]} ]). -define(NUM_NODES, 1). From 481661ab71791a792a5a9bb8d76d012a32e3f628 Mon Sep 17 00:00:00 2001 From: Nick Marino Date: Thu, 23 Feb 2017 14:10:59 -0500 Subject: [PATCH 6/9] Add more checks to the lifecycle webhook handler --- tests/s3_lifecycle_hooks.erl | 39 ++++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/tests/s3_lifecycle_hooks.erl b/tests/s3_lifecycle_hooks.erl index a78565068..ecb5944ac 100644 --- a/tests/s3_lifecycle_hooks.erl +++ b/tests/s3_lifecycle_hooks.erl @@ -19,6 +19,7 @@ %% ------------------------------------------------------------------- -module(s3_lifecycle_hooks). +-include_lib("eunit/include/eunit.hrl"). %% %% This test uses some standard Riak APIs to create what the S3 web facade @@ -85,19 +86,41 @@ confirm() -> %% %% confirm webhook has been called %% - receive - {got_http_req, Req} -> - lager:info("got request ~p ~p", [Req:get(method), Req:get(path)]) - after - 60000 -> - lager:info("failed to get response", []) - end, + lager:info("Waiting for lifecycle webhook to be executed", []), + ?assertEqual(ok, rt:wait_until(fun check_for_webhook_request/0)), pass. start_webhook_server() -> TestPid = self(), Loop = fun(Req) -> - TestPid ! {got_http_req, Req} + %% Beware, recv_body will fail if we call it outside this request + %% handler process, because mochiweb secretly stores stuff + %% in the process dictionary. The failure is also silent and + %% mysterious, due to mochiweb_request calling `exit(normal)` + %% in several places instead of crashing or properly handling + %% the error... o_O + Body = Req:recv_body(), + Req:respond({200, [], []}), + TestPid ! {got_http_req, Req, Body} end, mochiweb_http:start([{name, ?MODULE}, {loop, Loop}, {port, ?WEBHOOK_PORT}]). + +check_for_webhook_request() -> + receive + {got_http_req, Req, Body} -> + verify_webhook_request_parameters(Req), + verify_webhook_request_body(Body) + after + 0 -> + false + end. + +verify_webhook_request_parameters(Req) -> + ?assertEqual('PUT', Req:get(method)), + ?assertEqual(?WEBHOOK_PATH, Req:get(path)). + +verify_webhook_request_body(BodyBin) -> + Body = mochijson2:decode(BodyBin), + lager:info("Got body ~p", [Body]), + true. From 6bbc062e360c0567855018739c082f2119af898d Mon Sep 17 00:00:00 2001 From: Fred Dushin Date: Fri, 3 Mar 2017 20:37:19 -0500 Subject: [PATCH 7/9] Pulled web hook server into it's own module for use with other tests. --- src/webhook_server.erl | 97 ++++++++++++++++++++++++++++++++++++ tests/s3_lifecycle_hooks.erl | 38 ++++++-------- 2 files changed, 111 insertions(+), 24 deletions(-) create mode 100644 src/webhook_server.erl diff --git a/src/webhook_server.erl b/src/webhook_server.erl new file mode 100644 index 000000000..5c2e86ec3 --- /dev/null +++ b/src/webhook_server.erl @@ -0,0 +1,97 @@ +%% ------------------------------------------------------------------- +%% +%% Copyright (c) 2017 Basho Technologies, Inc. +%% +%% This file is provided to you under the Apache License, +%% Version 2.0 (the "License"); you may not use this file +%% except in compliance with the License. You may obtain +%% a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, +%% software distributed under the License is distributed on an +%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +%% KIND, either express or implied. See the License for the +%% specific language governing permissions and limitations +%% under the License. +%% +%% ------------------------------------------------------------------- +-module(webhook_server). + +-behaviour(gen_server). + +%% API +-export([start_link/1, get_next/0]). + +%% gen_server callbacks +-export([ + init/1, + handle_call/3, + handle_cast/2, + handle_info/2, + terminate/2, + code_change/3 +]). + +%% mochiweb callbacks +-export([receive_callback/1]). + +-define(SERVER, ?MODULE). + +-record(state, { + receipts = queue:new() +}). + +%%%=================================================================== +%%% API +%%%=================================================================== + +start_link(WebhookPort) -> + {ok, _Pid} = gen_server:start_link({local, ?SERVER}, ?MODULE, [WebhookPort], []), + ok. + +get_next() -> + gen_server:call(?MODULE, get_next). + +receive_callback(Req) -> + %% Beware, recv_body will fail if we call it outside this request + %% handler process, because mochiweb secretly stores stuff + %% in the process dictionary. The failure is also silent and + %% mysterious, due to mochiweb_request calling `exit(normal)` + %% in several places instead of crashing or properly handling + %% the error... o_O + Body = Req:recv_body(), + Req:respond({200, [], []}), + lager:info("Received call to webhook."), + gen_server:cast(?MODULE, {got_http_req, Req, Body}). + +%%%=================================================================== +%%% gen_server callbacks +%%%=================================================================== + +init([WebhookPort]) -> + {ok, _Pid} = mochiweb_http:start_link([{name, webhook_server_internal}, {loop, fun receive_callback/1}, {port, WebhookPort}]), + {ok, #state{}}. + +handle_call(get_next, _From, #state{receipts=Receipts} = State) -> + case queue:out(Receipts) of + {{value, Receipt}, Receipts2} -> + {reply, Receipt, State#state{receipts=Receipts2}}; + {empty, Receipts} -> + {reply, empty, State} + end. + +handle_cast({got_http_req, _Req, _Body} = Receipt, #state{receipts=Receipts} = State) -> + {noreply, State#state{receipts = queue:in(Receipt, Receipts)}}. + +handle_info(_Info, State) -> + {noreply, State}. + +terminate(_Reason, _State) -> + mochiweb_http:stop(webhook_server_internal), + ok. + +code_change(_OldVsn, State, _Extra) -> + {ok, State}. + diff --git a/tests/s3_lifecycle_hooks.erl b/tests/s3_lifecycle_hooks.erl index ecb5944ac..25f80dd0c 100644 --- a/tests/s3_lifecycle_hooks.erl +++ b/tests/s3_lifecycle_hooks.erl @@ -54,11 +54,17 @@ confirm() -> + %% + %% Start our webhook + %% + lager:info("Starting mock lifecycle webhook server..."), + ok = webhook_server:start_link(?WEBHOOK_PORT), %% %% Build the cluster %% Cluster = rt:build_cluster(?NUM_NODES, ?CONFIG), Node = lists:nth(random:uniform(length((Cluster))), Cluster), + rt:wait_for_service(Node, [riak_kv]), %% %% Create a mock S3 "bucket" (aka Riak bucket type) with %% @@ -68,12 +74,10 @@ confirm() -> Bucket = rpc:call(Node, riak_s3_bucket, to_riak_bucket, [?BUCKET_TYPE]), lager:info("Bucket: ~p", [Bucket]), - %% Other misc setup: - lager:info("Starting mock lifecycle webhook server", []), - start_webhook_server(), %% Populate an S3 "object" TODO refactor as needed -- maybe lots of puts? %% + lager:info("Writing an object to the S3 bucket..."), Client = rt:pbc(Node), _Ret = riakc_pb_socket:put( Client, riakc_obj:new( @@ -86,34 +90,20 @@ confirm() -> %% %% confirm webhook has been called %% - lager:info("Waiting for lifecycle webhook to be executed", []), + lager:info("Waiting for lifecycle webhook to be executed..."), ?assertEqual(ok, rt:wait_until(fun check_for_webhook_request/0)), pass. -start_webhook_server() -> - TestPid = self(), - Loop = fun(Req) -> - %% Beware, recv_body will fail if we call it outside this request - %% handler process, because mochiweb secretly stores stuff - %% in the process dictionary. The failure is also silent and - %% mysterious, due to mochiweb_request calling `exit(normal)` - %% in several places instead of crashing or properly handling - %% the error... o_O - Body = Req:recv_body(), - Req:respond({200, [], []}), - TestPid ! {got_http_req, Req, Body} - end, - mochiweb_http:start([{name, ?MODULE}, {loop, Loop}, {port, ?WEBHOOK_PORT}]). - check_for_webhook_request() -> - receive + case webhook_server:get_next() of + empty -> + lager:info("No entry received yet."), + false; {got_http_req, Req, Body} -> verify_webhook_request_parameters(Req), - verify_webhook_request_body(Body) - after - 0 -> - false + verify_webhook_request_body(Body), + true end. verify_webhook_request_parameters(Req) -> From e7b6d446182763c12e0fc615e39c653ec9d5de7d Mon Sep 17 00:00:00 2001 From: Nick Marino Date: Thu, 23 Feb 2017 14:30:28 -0500 Subject: [PATCH 8/9] Verify metadata in lifecycle webhook request body --- tests/s3_lifecycle_hooks.erl | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/s3_lifecycle_hooks.erl b/tests/s3_lifecycle_hooks.erl index 25f80dd0c..4fba5c87e 100644 --- a/tests/s3_lifecycle_hooks.erl +++ b/tests/s3_lifecycle_hooks.erl @@ -29,6 +29,7 @@ -export([confirm/0]). -define(BUCKET_TYPE, <<"s3_lifecycle_hooks">>). +-define(OBJECT_KEY, <<"lifecycle_test_key">>). -define(WEBHOOK_PATH, "/lifecycle_hook"). -define(WEBHOOK_PORT, 8765). @@ -81,7 +82,7 @@ confirm() -> Client = rt:pbc(Node), _Ret = riakc_pb_socket:put( Client, riakc_obj:new( - Bucket, <<"test_key">>, <<"test_value">> + Bucket, ?OBJECT_KEY, <<"test_value">> ) ), @@ -111,6 +112,7 @@ verify_webhook_request_parameters(Req) -> ?assertEqual(?WEBHOOK_PATH, Req:get(path)). verify_webhook_request_body(BodyBin) -> - Body = mochijson2:decode(BodyBin), - lager:info("Got body ~p", [Body]), + {struct, Body} = mochijson2:decode(BodyBin), + ?assertEqual({<<"bucket_name">>, ?BUCKET_TYPE}, lists:keyfind(<<"bucket_name">>, 1, Body)), + ?assertEqual({<<"object_name">>, ?OBJECT_KEY}, lists:keyfind(<<"object_name">>, 1, Body)), true. From c7fb0273323269db77463ae255f264cc688f81ff Mon Sep 17 00:00:00 2001 From: Nick Marino Date: Tue, 7 Mar 2017 14:56:11 -0500 Subject: [PATCH 9/9] Add check that owner is passed through to webhook --- tests/s3_lifecycle_hooks.erl | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/tests/s3_lifecycle_hooks.erl b/tests/s3_lifecycle_hooks.erl index 4fba5c87e..fc9a5b121 100644 --- a/tests/s3_lifecycle_hooks.erl +++ b/tests/s3_lifecycle_hooks.erl @@ -30,6 +30,7 @@ -define(BUCKET_TYPE, <<"s3_lifecycle_hooks">>). -define(OBJECT_KEY, <<"lifecycle_test_key">>). +-define(OBJECT_OWNER, <<"Nick Marino">>). -define(WEBHOOK_PATH, "/lifecycle_hook"). -define(WEBHOOK_PORT, 8765). @@ -79,14 +80,16 @@ confirm() -> %% Populate an S3 "object" TODO refactor as needed -- maybe lots of puts? %% lager:info("Writing an object to the S3 bucket..."), - Client = rt:pbc(Node), - _Ret = riakc_pb_socket:put( - Client, riakc_obj:new( - Bucket, ?OBJECT_KEY, <<"test_value">> - ) - ), + %% We have to use the internal client here, since the standard clients won't + %% let us write arbitrary keys to the object metadata... + {ok, Client} = rpc:call(Node, riak, local_client, []), + Obj0 = riak_object:new(Bucket, ?OBJECT_KEY, <<"test_value">>), + MD0 = riak_object:get_update_metadata(Obj0), + MD = dict:store(<<"X-Riak-S3-Owner">>, ?OBJECT_OWNER, MD0), + Obj = riak_object:update_metadata(Obj0, MD), + _Ret = Client:put(Obj), - %% TODO Force a sweep of our object's partition: + %% TODO Force a sweep of our object's partition, just to speed up the test a bit? %% %% confirm webhook has been called @@ -115,4 +118,6 @@ verify_webhook_request_body(BodyBin) -> {struct, Body} = mochijson2:decode(BodyBin), ?assertEqual({<<"bucket_name">>, ?BUCKET_TYPE}, lists:keyfind(<<"bucket_name">>, 1, Body)), ?assertEqual({<<"object_name">>, ?OBJECT_KEY}, lists:keyfind(<<"object_name">>, 1, Body)), + ?assertEqual({<<"object_owner_id">>, ?OBJECT_OWNER}, + lists:keyfind(<<"object_owner_id">>, 1, Body)), true.