diff --git a/rebar.config b/rebar.config index 62483d276..09293f966 100644 --- a/rebar.config +++ b/rebar.config @@ -10,9 +10,9 @@ {deps, [ {blockchain, {git, "https://github.com/helium/blockchain-core.git", - {branch, "andymck/poc-grpc-v2"}}}, + {branch, "andymck/poc-grpc-v2-plus-vals-as-chall"}}}, {sibyl, {git, "https://github.com/helium/sibyl.git", - {branch, "andymck/poc-grpc"}}}, + {branch, "andymck/poc-grpc-plus-vals-as-chall"}}}, {hbbft, {git, "https://github.com/helium/erlang-hbbft.git", {branch, "master"}}}, {dkg, {git, "https://github.com/helium/erlang-dkg.git", {branch, "master"}}}, diff --git a/rebar.lock b/rebar.lock index f83e38b39..d2935df51 100644 --- a/rebar.lock +++ b/rebar.lock @@ -5,7 +5,7 @@ {<<"base64url">>,{pkg,<<"base64url">>,<<"1.0.1">>},1}, {<<"blockchain">>, {git,"https://github.com/helium/blockchain-core.git", - {ref,"5881ad571903c1908fbbaed396a59bc7efa505ce"}}, + {ref,"61fd00a24e93ef67f4d526e2955910d79f330965"}}, 0}, {<<"certifi">>,{pkg,<<"certifi">>,<<"2.8.0">>},2}, {<<"chatterbox">>, @@ -107,7 +107,7 @@ 0}, {<<"helium_proto">>, {git,"https://github.com/helium/proto.git", - {ref,"f743a80e534bdc78805e3c5438cb466bec3c0b6f"}}, + {ref,"5e663437950c3e6d71fbd230e1435005be26603f"}}, 1}, {<<"hpack">>,{pkg,<<"hpack_erl">>,<<"0.2.3">>},3}, {<<"http2_client">>, @@ -176,7 +176,7 @@ 3}, {<<"sibyl">>, {git,"https://github.com/helium/sibyl.git", - {ref,"a5a86c9441a0db5e0db136af47fa09fb0ae50c02"}}, + {ref,"f735165f4413198e1cfeb981cadce382a95a88ed"}}, 0}, {<<"sidejob">>,{pkg,<<"sidejob">>,<<"2.1.0">>},2}, {<<"small_ints">>,{pkg,<<"small_ints">>,<<"0.1.0">>},4}, diff --git a/src/handlers/miner_hbbft_handler.erl b/src/handlers/miner_hbbft_handler.erl index 1d04fc7cb..71fe50c30 100644 --- a/src/handlers/miner_hbbft_handler.erl +++ b/src/handlers/miner_hbbft_handler.erl @@ -106,24 +106,15 @@ metadata(Version, Meta, Chain) -> _ -> 1 end, lager:debug("poc challenge rate ~p", [ChallengeRate] ), - %% if a val is in the ignore list then dont generate poc keys for it - %% TODO: this is a temp hack. remove when testing finished - IgnoreVals = application:get_env(sibyl, validator_ignore_list, []), SelfPubKeyBin = blockchain_swarm:pubkey_bin(), - case not lists:member(SelfPubKeyBin, IgnoreVals) of - true -> - {EmpKeys, EmpKeyHashes} = generate_ephemeral_keys(N, ChallengeRate), - lager:debug("poc ephemeral keys ~p", [EmpKeys]), - lager:debug("node ~p generating poc ephemeral key hashes ~p", [SelfPubKeyBin, EmpKeyHashes]), - ok = miner_poc_mgr:save_poc_keys(Height, EmpKeys), - maps:put(poc_keys, {SelfPubKeyBin, EmpKeyHashes}, ChainMeta); - false -> - ChainMeta - end; + NumKeys = max(1, trunc(ChallengeRate / (((N-1)/3) * 2 ))), + POCEphemeralKeys = miner_poc_mgr:get_random_poc_key_proposals(NumKeys, Ledger), + lager:debug("node ~p submitting poc ephemeral key hashes ~p", [SelfPubKeyBin, POCEphemeralKeys]), + maps:put(poc_keys, POCEphemeralKeys, ChainMeta); _ -> ChainMeta - end, + end, lager:info("ChainMeta1 ~p", [ChainMeta1]), t2b(maps:merge(Meta, ChainMeta1)) end. @@ -854,18 +845,6 @@ bin_to_msg(<>) -> {error, truncated} end. --spec generate_ephemeral_keys(pos_integer(), pos_integer()) ->{[#{secret => libp2p_crypto:privkey(), public => libp2p_crypto:pubkey()}], [binary()]}. -generate_ephemeral_keys(N, ChallengeRate) -> - NumKeys = max(1, trunc(ChallengeRate / (((N-1)/3) * 2 ))), - lists:foldl( - fun(_N, {AccKeys, AccHashes})-> - Keys = libp2p_crypto:generate_keys(ecc_compact), - #{public := OnionCompactKey} = Keys, - OnionHash = crypto:hash(sha256, libp2p_crypto:pubkey_to_bin(OnionCompactKey)), - {[Keys | AccKeys], [OnionHash | AccHashes]} - end, - {[], []}, lists:seq(1, NumKeys)). - -ifdef(TEST). -include_lib("eunit/include/eunit.hrl"). diff --git a/src/miner.erl b/src/miner.erl index ebe6b2dba..b188896ed 100644 --- a/src/miner.erl +++ b/src/miner.erl @@ -754,7 +754,7 @@ snapshot_hash(Ledger, BlockHeightNext, Metadata, VotesNeeded) -> M :: metadata(), B :: blockchain_block:hash(). poc_keys(Ledger, Metadata, BlockHash) -> - %% Construct a set of poc keys. Each node will define its own set within the metadata + %% Construct a set of poc keys. Each node will pull a random list from a pool of keys %% We want to take a deterministic random subset of these up to a max of poc challenge rate %% Use the blockhash as the seed RandState = blockchain_utils:rand_state(BlockHash), @@ -763,15 +763,8 @@ poc_keys(Ledger, Metadata, BlockHash) -> {ok, V} -> V; _ -> 1 end, - PocKeys0 = [{MinerAddr, Keys} || {_, #{poc_keys := {MinerAddr, Keys}}} <- metadata_only_v2(Metadata)], - {ok, CGMembers} = blockchain_ledger_v1:consensus_members(Ledger), - PocKeys1 = lists:foldl( - fun({MinerAddr, PocKeys}, Acc)-> - Pos = miner_util:index_of(MinerAddr, CGMembers), - NormalisedKeys = lists:map(fun(PocKey) -> {Pos, PocKey} end, PocKeys), - [NormalisedKeys | Acc] - end, [], PocKeys0), - sort_and_truncate_poc_keys(lists:flatten(PocKeys1), ChallengeRate, RandState). + PocKeys0 = [POCKeys || {_, #{poc_keys := POCKeys}} <- metadata_only_v2(Metadata)], + sort_and_truncate_poc_keys(lists:flatten(PocKeys0), ChallengeRate, RandState). sort_and_truncate_poc_keys(L, MaxKeys, RandState) -> {_, TruncList} = blockchain_utils:deterministic_subset(MaxKeys, RandState, L), diff --git a/src/miner_keys.erl b/src/miner_keys.erl index 580ebdea7..6b44ec017 100644 --- a/src/miner_keys.erl +++ b/src/miner_keys.erl @@ -198,8 +198,10 @@ keys(#{pubkey := _PubKey, ecdh_fun := _ECDH, sig_fun := _Sig} = KeyInfo) -> key_config() -> BaseDir = application:get_env(blockchain, base_dir, "data"), case application:get_env(blockchain, key, undefined) of - undefined -> {file, BaseDir}; - KC -> KC + undefined -> + {file, BaseDir}; + KC -> + KC end. -spec libp2p_to_gateway_key(libp2p_crypto:key_map()) -> libp2p_crypto:key_map(). diff --git a/src/miner_restart_sup.erl b/src/miner_restart_sup.erl index 7ff4b2a10..2e451cd5e 100644 --- a/src/miner_restart_sup.erl +++ b/src/miner_restart_sup.erl @@ -98,10 +98,14 @@ init(_Opts) -> %% NOTE: validators do not require the onion or lora server %% however removing these here breaks tests %% there is no harm done by leaving them running + + %% core and sibyl need to callback to miner_poc_mgr + %% and so we need to set an env var to let it know the mod name + application:set_env(blockchain, poc_mgr_mod, miner_poc_mgr), application:set_env(sibyl, poc_mgr_mod, miner_poc_mgr), application:set_env(sibyl, poc_report_handler, miner_poc_report_handler), - PocMgrTab = miner_poc_mgr:make_ets_table(), - POCMgrOpts = #{tab1 => PocMgrTab}, + [PocMgrTab1, PocMgrTab2] = miner_poc_mgr:make_ets_table(), + POCMgrOpts = #{tab1 => PocMgrTab1, tab2 => PocMgrTab2}, POCOpts = #{base_dir => BaseDir, cfs => ["default", "poc_mgr_cf" diff --git a/src/miner_val_heartbeat.erl b/src/miner_val_heartbeat.erl index 5b28a575c..78a502412 100644 --- a/src/miner_val_heartbeat.erl +++ b/src/miner_val_heartbeat.erl @@ -87,8 +87,11 @@ handle_info({blockchain_event, {add_block, Hash, Sync, _Ledger}}, %% we need to construct and submit a heartbeat txn {ok, CBMod} = blockchain_ledger_v1:config(?predicate_callback_mod, Ledger), {ok, Callback} = blockchain_ledger_v1:config(?predicate_callback_fun, Ledger), + {EmpKeys, EmpKeyHashes} = generate_poc_keys(Ledger), + lager:debug("HB poc ephemeral keys ~p", [EmpKeys]), + ok = miner_poc_mgr:save_local_poc_keys(Height, EmpKeys), UnsignedTxn = - blockchain_txn_validator_heartbeat_v1:new(Address, Height, CBMod:Callback()), + blockchain_txn_validator_heartbeat_v1:new(Address, Height, CBMod:Callback(), EmpKeyHashes), Txn = blockchain_txn_validator_heartbeat_v1:sign(UnsignedTxn, SigFun), lager:info("submitting txn ~p for val ~p ~p ~p", [Txn, Val, N, HBInterval]), Self = self(), @@ -132,3 +135,53 @@ code_change(_OldVsn, State, _Extra) -> %%%=================================================================== %%% Internal functions %%%=================================================================== +-spec generate_poc_keys(blockchain:ledger()) -> + {[#{secret => libp2p_crypto:privkey(), public => libp2p_crypto:pubkey()}], [binary()]}. +generate_poc_keys(Ledger) -> + case blockchain_ledger_v1:config(?poc_challenger_type, Ledger) of + {ok, validator} -> + %% if a val is in the ignore list then dont generate poc keys for it + %% TODO: this is a temp hack. remove when testing finished + IgnoreVals = application:get_env(sibyl, validator_ignore_list, []), + SelfPubKeyBin = blockchain_swarm:pubkey_bin(), + case not lists:member(SelfPubKeyBin, IgnoreVals) of + true -> + %% generate a set of ephemeral keys for POC usage + %% count is based on the num of active validators and the + %% target challenge rate + %% we also have to consider that key proposals are + %% submitted by validators as part of their heartbeats + %% which are only submitted periodically + %% so we need to ensure we have sufficient count of + %% key proposals submitted per HB + %% to help with this we reduce the number of val count + %% by 20% so that we have surplus keys being submitted + EphemeralKeyCount = + case sibyl_mgr:validator_count() of + NumVals when NumVals > 0 -> + {ok, ChallengeRate} = blockchain_ledger_v1:config(?poc_challenge_rate, Ledger), + {ok, HBInterval} = blockchain_ledger_v1:config(?validator_liveness_interval, Ledger), + round((ChallengeRate / (NumVals * 0.8 )) * HBInterval); + _ -> + 0 + end, + lager:info("heartbeat ephemeral key count ~p", [EphemeralKeyCount]), + generate_ephemeral_keys(EphemeralKeyCount); + false -> + {[], []} + end; + _ -> + {[], []} + + end. + +-spec generate_ephemeral_keys(pos_integer()) -> {[#{secret => libp2p_crypto:privkey(), public => libp2p_crypto:pubkey()}], [binary()]}. +generate_ephemeral_keys(NumKeys) -> + lists:foldl( + fun(_N, {AccKeys, AccHashes})-> + Keys = libp2p_crypto:generate_keys(ecc_compact), + #{public := OnionCompactKey} = Keys, + OnionHash = crypto:hash(sha256, libp2p_crypto:pubkey_to_bin(OnionCompactKey)), + {[Keys | AccKeys], [OnionHash | AccHashes]} + end, + {[], []}, lists:seq(1, NumKeys)). diff --git a/src/poc/miner_poc_mgr.erl b/src/poc/miner_poc_mgr.erl index b620d6c4f..67cddd4aa 100644 --- a/src/poc/miner_poc_mgr.erl +++ b/src/poc/miner_poc_mgr.erl @@ -14,18 +14,15 @@ -define(ACTIVE_POCS, active_pocs). -define(KEYS, keys). +-define(KEY_PROPOSALS, key_proposals). -define(ADDR_HASH_FP_RATE, 1.0e-9). -define(POC_DB_CF, {?MODULE, poc_db_cf_handle}). -ifdef(TEST). %% lifespan of a POC, after which we will %% submit the receipts txn and delete the local poc data -define(POC_TIMEOUT, 4). -%% timeout after which we will GC the public poc data, -%% we expect the receipt txn to be absorbed before this --define(POC_RECEIPTS_ABSORB_TIMEOUT, 15). -else. -define(POC_TIMEOUT, 10). --define(POC_RECEIPTS_ABSORB_TIMEOUT, 200). -endif. @@ -35,13 +32,17 @@ -export([ start_link/1, make_ets_table/0, - cached_poc_key/1, - save_poc_keys/2, + cached_local_poc_key/1, + save_local_poc_keys/2, check_target/3, report/4, active_pocs/0, local_poc_key/1, - local_poc/1 + local_poc/1, + save_poc_key_proposals/3, + delete_cached_local_poc_key_proposal/1, + get_random_poc_key_proposals/2, + cached_local_poc_key_proposals/0 ]). %% ------------------------------------------------------------------ %% gen_server exports @@ -59,11 +60,17 @@ bloom :: bloom_nif:bloom() }). --record(poc_key_data, { +-record(poc_local_key_data, { receive_height :: non_neg_integer(), keys :: keys() }). +-record(poc_key_proposal, { + receive_height :: non_neg_integer(), + key :: key_proposal(), + address :: libp2p_crypto:pubkey_bin() +}). + -record(local_poc, { onion_key_hash :: binary(), block_hash :: binary() | undefined, @@ -84,21 +91,22 @@ ledger :: undefined | blockchain:ledger(), sig_fun :: undefined | libp2p_crypto:sig_fun(), pub_key = undefined :: undefined | libp2p_crypto:pubkey_bin(), - addr_hash_filter :: undefined | #addr_hash_filter{}, - poc_timeout :: pos_integer() | undefined, - poc_receipts_absorb_timeout :: pos_integer() | undefined + addr_hash_filter :: undefined | #addr_hash_filter{} }). -type state() :: #state{}. -type keys() :: #{secret => libp2p_crypto:privkey(), public => libp2p_crypto:pubkey()}. -type poc_key() :: binary(). --type cached_poc_key_data() :: #poc_key_data{}. --type cached_poc_key_type() :: {POCKey :: poc_key(), POCKeyData :: #poc_key_data{}}. +-type cached_local_poc_local_key_data() :: #poc_local_key_data{}. +-type cached_local_poc_key_type() :: {POCKey :: poc_key(), POCKeyData :: #poc_local_key_data{}}. +-type key_proposals() :: [key_proposal()]. +-type key_proposal() :: binary(). +-type cached_key_proposal() :: #poc_key_proposal{}. -type local_poc() :: #local_poc{}. -type local_pocs() :: [local_poc()]. -type local_poc_key() :: binary(). --export_type([keys/0, local_poc_key/0, cached_poc_key_data/0, cached_poc_key_type/0, local_poc/0, local_pocs/0]). +-export_type([keys/0, local_poc_key/0, cached_local_poc_local_key_data/0, cached_local_poc_key_type/0, local_poc/0, local_pocs/0]). %% ------------------------------------------------------------------ %% API functions @@ -120,12 +128,18 @@ start_link(Args) when is_map(Args) -> {ok, Tab1} -> true = ets:give_away(Tab1, Pid, undefined) end, + case maps:find(tab2, Args) of + error -> + ok; + {ok, Tab2} -> + true = ets:give_away(Tab2, Pid, undefined) + end, {ok, Pid}; Other -> Other end. --spec make_ets_table() -> ok. +-spec make_ets_table() -> [atom()]. make_ets_table() -> Tab1 = ets:new( ?KEYS, @@ -135,13 +149,20 @@ make_ets_table() -> {heir, self(), undefined} ] ), - Tab1. + Tab2 = ets:new( + ?KEY_PROPOSALS, + [ + named_table, + public, + {heir, self(), undefined} + ] + ), + [Tab1, Tab2]. --spec save_poc_keys(CurHeight :: non_neg_integer(), [keys()]) -> ok. -save_poc_keys(CurHeight, KeyList) -> - %% these are the keys generated by this validator and submitted in the block - %% either none or a subset of these will actually make it to the block - %% we dont obviously know which may make it so we cache them all here +-spec save_local_poc_keys(CurHeight :: non_neg_integer(), [keys()]) -> ok. +save_local_poc_keys(CurHeight, KeyList) -> + %% these are the keys ( public & private ) generated by this validator + %% as part of submitting a new heartbeat %% push each key set to ets with a hash of the public key as key %% each new block we will then check if any of our cached keys made it into the block %% and if so retrieve the private key for each @@ -149,16 +170,16 @@ save_poc_keys(CurHeight, KeyList) -> begin #{public := PubKey} = Keys, OnionKeyHash = crypto:hash(sha256, libp2p_crypto:pubkey_to_bin(PubKey)), - POCKeyRec = #poc_key_data{receive_height = CurHeight, keys = Keys}, + POCKeyRec = #poc_local_key_data{receive_height = CurHeight, keys = Keys}, lager:info("caching local poc keys with hash ~p", [OnionKeyHash]), - _ = cache_poc_key(OnionKeyHash, POCKeyRec) + _ = catch cache_poc_key(OnionKeyHash, POCKeyRec) end || Keys <- KeyList ], ok. --spec cached_poc_key(poc_key()) -> {ok, cached_poc_key_type()} | false. -cached_poc_key(ID) -> +-spec cached_local_poc_key(poc_key()) -> {ok, cached_local_poc_key_type()} | false. +cached_local_poc_key(ID) -> case ets:lookup(?KEYS, ID) of [Res] -> {ok, Res}; _ -> false @@ -185,18 +206,20 @@ check_target(Challengee, BlockHash, OnionKeyHash) -> Res = case LocalPOC of {error, not_found} -> - %% if the cache returns not found it could be it hasnt yet been initialized - %% so check if we have a cached POC key. these are added at the point - %% a block is proposed and then before the block has been gossiped - %% if such a key exists its a strong indication its not yet initialized + %% if the cache returns not found it could be the poc has not yet been initialized + %% so check if we have a cached local POC key. + %% These are added when a val HB is submitted by the local node + %% if such a key exists its an indication the POC may not yet have been initialized %% OR the e2qc cache was called before the POC was initialised and it %% has cached the {error, not_found} term %% so if we have the key then check rocks again, %% if still not available then its likely the POC hasnt been initialized %% if found then invalidate the e2qc cache - case cached_poc_key(OnionKeyHash) of + %% TODO: do the assumptions above still hold true with the val pool generating challenges + %% rather than the CG generating challenges ? + case cached_local_poc_key(OnionKeyHash) of {ok, {_KeyHash, _POCData}} -> - %% we do know this key + %% the submitted key is one of this nodes local keys lager:info("*** ~p is a known key ~p", [OnionKeyHash]), case ?MODULE:local_poc(OnionKeyHash) of {error, _} -> @@ -254,6 +277,40 @@ local_poc(OnionKeyHash) -> end end. +-spec save_poc_key_proposals(libp2p_crypto:pubkey_bin(), key_proposals(), pos_integer()) -> ok. +save_poc_key_proposals(Address, KeyProposals, Height) -> + %% these are key proposals submitted by *any* validator via their heartbeat + %% save_poc_key_proposals/3 is called when absorbing a heartbeat + %% we add the proposed keys to this local cache + %% and from this cache a random set of keys will be selected as part of + %% block proposals by the consensus group + [ + begin + POCKeyProposalRec = #poc_key_proposal{ + receive_height = Height, + address = Address, + key = KeyProposal + }, + lager:debug("caching poc key proposal ~p", [KeyProposal]), + _ = catch cache_poc_key_proposal(KeyProposal, POCKeyProposalRec) + end + || KeyProposal <- KeyProposals + ], + ok. + +-spec delete_cached_local_poc_key_proposal(key_proposal()) -> ok. +delete_cached_local_poc_key_proposal(KeyProposal) -> + true = ets:delete(?KEY_PROPOSALS, KeyProposal), + ok. + +-spec get_random_poc_key_proposals(pos_integer(), blockchain:ledger()) -> + [{libp2p_crypto:pubkey_bin(), key_proposal()}]. +get_random_poc_key_proposals(NumKeys, Ledger) -> + Keys = cached_local_poc_key_proposals(), + ShuffledKeys = blockchain_utils:shuffle(Keys), + {ok, CGMembers} = blockchain_ledger_v1:consensus_members(Ledger), + do_get_random_poc_key_proposals(NumKeys, CGMembers, ShuffledKeys). + %% ------------------------------------------------------------------ %% gen_server functions %% ------------------------------------------------------------------ @@ -295,22 +352,10 @@ handle_info(init, #state{chain = undefined} = State) -> Ledger = blockchain:ledger(Chain), ok = miner_poc:add_stream_handler(blockchain_swarm:tid(), miner_poc_report_handler), SelfPubKeyBin = blockchain_swarm:pubkey_bin(), - POCTimeout = - case blockchain:config(?poc_timeout, Ledger) of - {ok, T1} -> T1; - _ -> ?POC_TIMEOUT - end, - POCReceiptsAborbTimeout = - case blockchain:config(?poc_receipts_absorb_timeout, Ledger) of - {ok, T2} -> T2; - _ -> ?POC_RECEIPTS_ABSORB_TIMEOUT - end, {noreply, State#state{ chain = Chain, ledger = Ledger, - pub_key = SelfPubKeyBin, - poc_timeout = POCTimeout, - poc_receipts_absorb_timeout = POCReceiptsAborbTimeout + pub_key = SelfPubKeyBin }} end; handle_info(init, State) -> @@ -332,13 +377,6 @@ handle_info( State1 = maybe_init_addr_hash(State), ok = handle_add_block_event(CurPOCChallengerType, BlockHash, Chain, State1), {noreply, State1}; -%% TODO: review approach to syc blocks again -%%handle_info( -%% {blockchain_event, {add_block, _BlockHash, Sync, _Ledger} = _Event}, -%% #state{chain = _Chain} = State -%%) when Sync =:= true -> -%% lager:info("ignoring add block event, sync is ~p", [Sync]), -%% {noreply, State}; handle_info(_Info, State = #state{}) -> {noreply, State}. @@ -366,13 +404,22 @@ handle_add_block_event(POCChallengeType, BlockHash, Chain, State) when POCChalle %% take care of GC ok = purge_local_pocs(Block, State), BlockHeight = blockchain_block:height(Block), + Ledger = blockchain:ledger(Chain), %% GC local pocs keys every 50 blocks - %% NOTE, we dont need to GC the public POCs on the ledger here - %% that GC is handled elsewhere via blockchain_ledger_v1:maybe_gc_pocs/2 case BlockHeight rem 50 == 0 of - true -> ok = purge_pocs_keys(Block, State); - false -> ok + true -> + ok = purge_local_poc_keys(BlockHeight, Ledger); + false -> + ok + end, + %% GC pocs key proposals every 60 blocks + case BlockHeight rem 60 == 0 of + true -> + ok = purge_pocs_key_proposals(BlockHeight, Ledger); + false -> + ok end; + _ -> %% err what? ok @@ -609,10 +656,10 @@ process_block_pocs( [ begin %% the published key is a hash of the public key, aka the onion key hash - %% use this to check our local cache containing the secret keys of POCs owned by this validator + %% use this to check our local cache containing the keys of POCs owned by this validator %% if it is one of this local validators POCs, then kick it off - case cached_poc_key(OnionKeyHash) of - {ok, {_KeyHash, #poc_key_data{keys = Keys}}} -> + case cached_local_poc_key(OnionKeyHash) of + {ok, {_KeyHash, #poc_local_key_data{keys = Keys}}} -> lager:info("found local poc key, starting a poc for ~p", [OnionKeyHash]), %% its a locally owned POC key, so kick off a new POC Vars = blockchain_utils:vars_binary_keys_to_atoms(maps:from_list(blockchain_ledger_v1:snapshot_vars(Ledger))), @@ -620,7 +667,10 @@ process_block_pocs( _ -> lager:info("failed to find local poc key for ~p", [OnionKeyHash]), noop - end + end, + %% GC the block key from the key proposals cache + %% dont want to have it reused + _ = delete_cached_local_poc_key_proposal(OnionKeyHash) end || {_CGPos, OnionKeyHash} <- BlockPocEphemeralKeys ], @@ -632,17 +682,23 @@ process_block_pocs( ) -> ok. purge_local_pocs( Block, - #state{chain = Chain, pub_key = SelfPubKeyBin, sig_fun = SigFun, poc_timeout = POCTimeout} = State + #state{chain = Chain, pub_key = SelfPubKeyBin, sig_fun = SigFun} = State ) -> %% iterate over the local POCs in our rocksdb %% end and clean up any which have exceeded their life span - %% these are POCs which were initiated by this node + %% these are active POCs which were initiated by this node %% and the data is known only to this node + Ledger = blockchain:ledger(Chain), + Timeout = + case blockchain:config(?poc_timeout, Ledger) of + {ok, N} -> N; + _ -> ?POC_TIMEOUT + end, BlockHeight = blockchain_block:height(Block), LocalPOCs = local_pocs(State), lists:foreach( fun([#local_poc{start_height = POCStartHeight, onion_key_hash = OnionKeyHash} = POC]) -> - case (BlockHeight - POCStartHeight) > POCTimeout of + case (BlockHeight - POCStartHeight) > Timeout of true -> lager:info("*** purging local poc with key ~p", [OnionKeyHash]), %% this POC's time is up, submit receipts we have received @@ -659,33 +715,36 @@ purge_local_pocs( ), ok. --spec purge_pocs_keys( - Block :: blockchain_block:block(), - State :: state() +-spec purge_local_poc_keys( + BlockHeight :: pos_integer(), + Ledger :: blockchain_ledger_v1:ledger() ) -> ok. -purge_pocs_keys( - Block, - #state{poc_timeout = POCTimeout} = _State +purge_local_poc_keys( + BlockHeight, + Ledger ) -> %% iterate over the poc keys in our ets cache - %% these are a copy of the keys generated by this node - %% as part of its block creation ( whilst it is in the CG) - %% and submitted as part of the block metadata - %% one or more of these keys *may* make it into the block - %% we cache all our locally generated keys - %% and then each new block check if each mined key + %% and purge any which are deemed to be passed due + %% these keys are generated by *this* node + %% as part of its heartbeat submission + %% and added to the poc_mgr cache + %% each new block check if each mined key %% for that block is one of our own %% if it is then we initiate a new local POC %% the keys are purged periodically - BlockHeight = blockchain_block:height(Block), + Timeout = + case blockchain:config(?poc_timeout, Ledger) of + {ok, N} -> N; + _ -> ?POC_TIMEOUT + end, %% iterate over the cached POC keys, delete any which are beyond the lifespan of when the active POC would have ended - CachedPOCKeys = cached_poc_keys(), + CachedPOCKeys = cached_local_poc_keys(), lists:foreach( - fun({Key, #poc_key_data{receive_height = POCHeight}}) -> - case (BlockHeight - POCHeight) > POCTimeout of + fun({Key, #poc_local_key_data{receive_height = ReceiveHeight}}) -> + case (BlockHeight - ReceiveHeight) > Timeout of true -> %% the lifespan of any POC for this key has passed, we can GC - ok = delete_cached_poc_key(Key); + ok = delete_cached_local_poc_key(Key); _ -> ok end @@ -694,6 +753,45 @@ purge_pocs_keys( ), ok. +-spec purge_pocs_key_proposals( + BlockHeight :: pos_integer(), + Ledger :: blockchain_ledger_v1:ledger() +) -> ok. +purge_pocs_key_proposals( + BlockHeight, + Ledger +) -> + %% iterate over the poc key proposals in our ets cache + %% and purge any which are deemed to be passed due + %% these proposed keys are those generated by any validator + %% and cached on this node when absorbing validator heartbeats + %% when blocks are proposed, a random subset of keys + %% from this cache will be selected and included + %% in the local block proposal ( assuming the node is in the CG ) + %% one or more of these proposed keys *may* make it into the block + %% in order to prevent an unbounded cache we will GC + %% keys in this cache periodically + %% NOTE: a key will also be removed from the cache should it make it into a block + Timeout = + case blockchain:config(?poc_validator_ephemeral_key_timeout, Ledger) of + {ok, N} -> N; + _ -> 200 + end, + CachedPOCKeyProposals = cached_local_poc_key_proposals(), + lists:foreach( + fun({Key, #poc_key_proposal{receive_height = ReceiveHeight}}) -> + case (BlockHeight - ReceiveHeight) > Timeout of + true -> + %% the lifespan of any POC for this key has passed, we can GC + ok = delete_cached_local_poc_key_proposal(Key); + _ -> + ok + end + end, + CachedPOCKeyProposals + ), + ok. + -spec submit_receipts(local_poc(), libp2p_crypto:pubkey_bin(), libp2p_crypto:sig_fun(), blockchain:blockchain()) -> ok. submit_receipts( #local_poc{ @@ -748,19 +846,27 @@ submit_receipts( ok. --spec cache_poc_key(poc_key(), cached_poc_key_data()) -> true. +-spec cache_poc_key(poc_key(), cached_local_poc_local_key_data()) -> true. cache_poc_key(ID, Keys) -> true = ets:insert(?KEYS, {ID, Keys}). --spec cached_poc_keys() -> [cached_poc_key_type()]. -cached_poc_keys() -> +-spec cached_local_poc_keys() -> [cached_local_poc_key_type()]. +cached_local_poc_keys() -> ets:tab2list(?KEYS). --spec delete_cached_poc_key(poc_key()) -> ok. -delete_cached_poc_key(Key) -> +-spec delete_cached_local_poc_key(poc_key()) -> ok. +delete_cached_local_poc_key(Key) -> true = ets:delete(?KEYS, Key), ok. +-spec cache_poc_key_proposal(key_proposal(), cached_key_proposal()) -> true. +cache_poc_key_proposal(KeyProposal, Rec) -> + true = ets:insert(?KEY_PROPOSALS, {KeyProposal, Rec}). + +-spec cached_local_poc_key_proposals() -> [cached_key_proposal()]. +cached_local_poc_key_proposals() -> + ets:tab2list(?KEY_PROPOSALS). + -spec validate_witness(blockchain_poc_witness_v1:witness(), blockchain_ledger_v1:ledger()) -> boolean(). validate_witness(Witness, Ledger) -> @@ -938,6 +1044,27 @@ update_addr_hash(Bloom, Element) -> end end. +-spec do_get_random_poc_key_proposals(pos_integer(), [libp2p_crypto:pubkey_bin()], + [cached_key_proposal()]) -> + [{libp2p_crypto:pubkey_bin(), key_proposal()}]. +do_get_random_poc_key_proposals(NumKeys, CGMembers, Keys) -> + do_get_random_poc_key_proposals(NumKeys, CGMembers, Keys, []). +-spec do_get_random_poc_key_proposals(pos_integer(), [libp2p_crypto:pubkey_bin()], + [cached_key_proposal()], [{libp2p_crypto:pubkey_bin(), key_proposal()}]) -> + [{libp2p_crypto:pubkey_bin(), key_proposal()}]. +do_get_random_poc_key_proposals(0, _CGMembers, _Keys, Acc) -> + Acc; +do_get_random_poc_key_proposals(_, _CGMembers, [] = _Keys, Acc) -> + Acc; +do_get_random_poc_key_proposals(NumKeys, CGMembers, + [{_, #poc_key_proposal{key = Key, address = Address}} | T] = _Keys, Acc) -> + case lists:member(Address, CGMembers) of + true -> + do_get_random_poc_key_proposals(NumKeys, CGMembers, T, Acc); + false -> + do_get_random_poc_key_proposals(NumKeys-1, CGMembers, T, [{Address, Key} | Acc]) + end. + %% ------------------------------------------------------------------ %% DB functions %% ------------------------------------------------------------------ diff --git a/test/miner_ct_utils.erl b/test/miner_ct_utils.erl index 21f80e1a7..0e6536dba 100644 --- a/test/miner_ct_utils.erl +++ b/test/miner_ct_utils.erl @@ -770,9 +770,10 @@ init_per_testcase(Mod, TestCase, Config0) -> LogDir = ?config(log_dir, Config), SplitMiners = proplists:get_value(split_miners_vals_and_gateways, Config, false), NumValidators = proplists:get_value(num_validators, Config, 0), + NumGateways = proplists:get_value(num_gateways, Config, get_config("T", 8)), + NumConsensusMembers = proplists:get_value(num_consensus_members, Config, get_config("N", 7)), LoadChainOnGateways = proplists:get_value(gateways_run_chain, Config, true), - os:cmd(os:find_executable("epmd")++" -daemon"), {ok, Hostname} = inet:gethostname(), case net_kernel:start([list_to_atom("runner-miner" ++ @@ -783,33 +784,7 @@ init_per_testcase(Mod, TestCase, Config0) -> {error, {{already_started, _},_}} -> ok end, - %% Miner configuration, can be input from os env - TotalMiners = - case TestCase of - poc_grpc_dist_v11_test -> - 11; %% 5 vals, 6 gateways - poc_grpc_dist_v11_cn_test -> - 13; %% 5 vals, 8 gateways - poc_grpc_dist_v11_partitioned_test -> - 13; %% 5 vals, 8 gateways - poc_grpc_dist_v11_partitioned_lying_test -> - 13; %% 5 vals, 8 gateways - _ -> - get_config("T", 8) - end, - NumConsensusMembers = - case TestCase of - poc_grpc_dist_v11_test -> - NumValidators; - poc_grpc_dist_v11_cn_test -> - NumValidators; - poc_grpc_dist_v11_partitioned_test -> - NumValidators; - poc_grpc_dist_v11_partitioned_lying_test -> - NumValidators; - _ -> - get_config("N", 7) - end, + TotalMiners = NumValidators + NumGateways, SeedNodes = [], JsonRpcBase = 4486, Port = get_config("PORT", 0), diff --git a/test/miner_poc_grpc_SUITE.erl b/test/miner_poc_grpc_SUITE.erl index 8414f683c..ad77b71f1 100644 --- a/test/miner_poc_grpc_SUITE.erl +++ b/test/miner_poc_grpc_SUITE.erl @@ -69,12 +69,16 @@ test_cases() -> init_per_group(poc_grpc_with_chain, Config) -> [ {split_miners_vals_and_gateways, true}, - {num_validators, 5}, + {num_validators, 10}, + {num_gateways, 6}, + {num_consensus_members, 4}, {gateways_run_chain, true} | Config]; init_per_group(poc_grpc_no_chain, Config) -> [ {split_miners_vals_and_gateways, true}, - {num_validators, 5}, + {num_validators, 10}, + {num_gateways, 6}, + {num_consensus_members, 4}, {gateways_run_chain, false} | Config]. init_per_testcase(TestCase, Config) -> @@ -141,9 +145,8 @@ exec_dist_test(_TestCase, Config, VarMap, Status) -> case Status of %% expect failure and exit false -> - ?assert(check_validators_are_creating_poc_keys(Validators)); + ok; true -> - ?assert(check_validators_are_creating_poc_keys(Validators)), %% Check that the receipts are growing case maps:get(?poc_version, VarMap, 11) of V when V >= 10 -> @@ -151,14 +154,15 @@ exec_dist_test(_TestCase, Config, VarMap, Status) -> %% the checks for both poc-v10 and poc-v11 here true = miner_ct_utils:wait_until( fun() -> + C1 = check_validators_are_creating_poc_keys(Validators), %% Check if we have some receipts C2 = maps:size(challenger_receipts_map(find_receipts(Validators))) > 0, %% Check there are some poc rewards RewardsMD = get_rewards_md(Config), ct:pal("RewardsMD: ~p", [RewardsMD]), C3 = check_non_empty_poc_rewards(take_poc_challengee_and_witness_rewards(RewardsMD)), - ct:pal("C2: ~p, C3: ~p", [C2, C3]), - C2 andalso C3 + ct:pal("C1: ~p C2: ~p, C3: ~p", [C1, C2, C3]), + C1 andalso C2 andalso C3 end, 25, 5000), FinalRewards = get_rewards(Config), @@ -538,7 +542,8 @@ extra_vars(grpc) -> ?poc_challenge_rate => 1, ?poc_challenger_type => validator, ?poc_timeout => 4, - ?poc_receipts_absorb_timeout => 2 + ?poc_receipts_absorb_timeout => 2, + ?poc_validator_ephemeral_key_timeout => 50 }, maps:merge(extra_vars(poc_v11), GrpcVars); extra_vars(poc_v11) -> @@ -557,10 +562,6 @@ extra_vars(poc_v10) -> ?securities_percent => 0.34, ?reward_version => 5, ?rewards_txn_version => 2, - ?poc_challenge_rate => 1, - ?poc_challenger_type => validator, - ?poc_timeout => 4, - ?poc_receipts_absorb_timeout => 2, ?election_interval => 10, ?block_time => 5000 });