diff --git a/deps/rabbit/Makefile b/deps/rabbit/Makefile
index aad618f4211e..da1d32fe52ca 100644
--- a/deps/rabbit/Makefile
+++ b/deps/rabbit/Makefile
@@ -239,22 +239,16 @@ define ct_master.erl
peer:call(Pid2, persistent_term, put, [rabbit_ct_tcp_port_base, 25000]),
peer:call(Pid3, persistent_term, put, [rabbit_ct_tcp_port_base, 27000]),
peer:call(Pid4, persistent_term, put, [rabbit_ct_tcp_port_base, 29000]),
- ct_master_fork:run("$1"),
- Fail1 = peer:call(Pid1, cth_parallel_ct_detect_failure, has_failures, []),
- Fail2 = peer:call(Pid2, cth_parallel_ct_detect_failure, has_failures, []),
- Fail3 = peer:call(Pid3, cth_parallel_ct_detect_failure, has_failures, []),
- Fail4 = peer:call(Pid4, cth_parallel_ct_detect_failure, has_failures, []),
+ [{[_], {ok, Results}}] = ct_master_fork:run("$1"),
peer:stop(Pid4),
peer:stop(Pid3),
peer:stop(Pid2),
peer:stop(Pid1),
- if
- Fail1 -> halt(1);
- Fail2 -> halt(2);
- Fail3 -> halt(3);
- Fail4 -> halt(4);
- true -> halt(0)
- end
+ lists:foldl(fun
+ ({_, {_, 0, {_, 0}}}, Err) -> Err + 1;
+ (What, Peer) -> halt(Peer)
+ end, 1, Results),
+ halt(0)
endef
PARALLEL_CT_SET_1_A = amqp_client unit_cluster_formation_locking_mocks unit_cluster_formation_sort_nodes unit_collections unit_config_value_encryption unit_connection_tracking
@@ -293,6 +287,7 @@ define tpl_parallel_ct_test_spec
{logdir, "$(CT_LOGS_DIR)"}.
{logdir, master, "$(CT_LOGS_DIR)"}.
{create_priv_dir, all_nodes, auto_per_run}.
+{auto_compile, false}.
{node, shard1, 'rabbit_shard1@localhost'}.
{node, shard2, 'rabbit_shard2@localhost'}.
diff --git a/deps/rabbitmq_ct_helpers/app.bzl b/deps/rabbitmq_ct_helpers/app.bzl
index a2f85973d675..5754e5f4c8aa 100644
--- a/deps/rabbitmq_ct_helpers/app.bzl
+++ b/deps/rabbitmq_ct_helpers/app.bzl
@@ -11,9 +11,10 @@ def all_beam_files(name = "all_beam_files"):
name = "other_beam",
testonly = True,
srcs = [
+ "src/ct_master_event_fork.erl",
"src/ct_master_fork.erl",
+ "src/ct_master_logs_fork.erl",
"src/cth_log_redirect_any_domains.erl",
- "src/cth_parallel_ct_detect_failure.erl",
"src/rabbit_control_helper.erl",
"src/rabbit_ct_broker_helpers.erl",
"src/rabbit_ct_config_schema.erl",
@@ -39,9 +40,10 @@ def all_test_beam_files(name = "all_test_beam_files"):
name = "test_other_beam",
testonly = True,
srcs = [
+ "src/ct_master_event_fork.erl",
"src/ct_master_fork.erl",
+ "src/ct_master_logs_fork.erl",
"src/cth_log_redirect_any_domains.erl",
- "src/cth_parallel_ct_detect_failure.erl",
"src/rabbit_control_helper.erl",
"src/rabbit_ct_broker_helpers.erl",
"src/rabbit_ct_config_schema.erl",
@@ -103,9 +105,10 @@ def all_srcs(name = "all_srcs"):
name = "srcs",
testonly = True,
srcs = [
+ "src/ct_master_event_fork.erl",
"src/ct_master_fork.erl",
+ "src/ct_master_logs_fork.erl",
"src/cth_log_redirect_any_domains.erl",
- "src/cth_parallel_ct_detect_failure.erl",
"src/rabbit_control_helper.erl",
"src/rabbit_ct_broker_helpers.erl",
"src/rabbit_ct_config_schema.erl",
diff --git a/deps/rabbitmq_ct_helpers/src/ct_master_event_fork.erl b/deps/rabbitmq_ct_helpers/src/ct_master_event_fork.erl
new file mode 100644
index 000000000000..2ac634840849
--- /dev/null
+++ b/deps/rabbitmq_ct_helpers/src/ct_master_event_fork.erl
@@ -0,0 +1,217 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2006-2024. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%% Common Test Framework Event Handler
+%%%
+%%% This module implements an event handler that the CT Master
+%%% uses to handle status and progress notifications sent to the
+%%% master node during test runs. It also keeps track of the
+%%% details of failures which are used by the CT Master to print
+%%% a summary at the end of its run. This module may be used as a
+%%% template for other event handlers that can be plugged in to
+%%% handle logging and reporting on the master node.
+-module(ct_master_event_fork).
+-moduledoc false.
+
+-behaviour(gen_event).
+
+%% API
+-export([start_link/0, add_handler/0, add_handler/1, stop/0]).
+-export([notify/1, sync_notify/1, get_results/0]).
+
+%% gen_event callbacks
+-export([init/1, handle_event/2, handle_call/2,
+ handle_info/2, terminate/2, code_change/3]).
+
+-include_lib("common_test/include/ct_event.hrl").
+-include_lib("common_test/src/ct_util.hrl").
+
+
+-record(state, {auto_skipped=[], failed=[]}).
+
+%%====================================================================
+%% gen_event callbacks
+%%====================================================================
+%%--------------------------------------------------------------------
+%% Function: start_link() -> {ok,Pid} | {error,Error}
+%% Description: Creates an event manager.
+%%--------------------------------------------------------------------
+start_link() ->
+ gen_event:start_link({local,?CT_MEVMGR}).
+
+%%--------------------------------------------------------------------
+%% Function: add_handler() -> ok | {'EXIT',Reason} | term()
+%% Description: Adds an event handler
+%%--------------------------------------------------------------------
+add_handler() ->
+ gen_event:add_handler(?CT_MEVMGR_REF,?MODULE,[]).
+add_handler(Args) ->
+ gen_event:add_handler(?CT_MEVMGR_REF,?MODULE,Args).
+
+%%--------------------------------------------------------------------
+%% Function: stop() -> ok
+%% Description: Stops the event manager
+%%--------------------------------------------------------------------
+stop() ->
+ case flush() of
+ {error,Reason} ->
+ ct_master_logs_fork:log("Error",
+ "No response from CT Master Event.\n"
+ "Reason = ~tp\n"
+ "Terminating now!\n",[Reason]),
+ %% communication with event manager fails, kill it
+ catch exit(whereis(?CT_MEVMGR_REF), kill);
+ _ ->
+ gen_event:stop(?CT_MEVMGR_REF)
+ end.
+
+flush() ->
+ try gen_event:call(?CT_MEVMGR_REF,?MODULE,flush,1800000) of
+ flushing ->
+ timer:sleep(1),
+ flush();
+ done ->
+ ok;
+ Error = {error,_} ->
+ Error
+ catch
+ _:Reason ->
+ {error,Reason}
+ end.
+
+%%--------------------------------------------------------------------
+%% Function: notify(Event) -> ok
+%% Description: Asynchronous notification to event manager.
+%%--------------------------------------------------------------------
+notify(Event) ->
+ gen_event:notify(?CT_MEVMGR_REF,Event).
+
+%%--------------------------------------------------------------------
+%% Function: sync_notify(Event) -> ok
+%% Description: Synchronous notification to event manager.
+%%--------------------------------------------------------------------
+sync_notify(Event) ->
+ gen_event:sync_notify(?CT_MEVMGR_REF,Event).
+
+%%--------------------------------------------------------------------
+%% Function: sync_notify(Event) -> Results
+%% Description: Get the results for auto-skipped and failed test cases.
+%%--------------------------------------------------------------------
+get_results() ->
+ gen_event:call(?CT_MEVMGR_REF,?MODULE,get_results).
+
+%%====================================================================
+%% gen_event callbacks
+%%====================================================================
+%%--------------------------------------------------------------------
+%% Function: init(Args) -> {ok, State}
+%% Description: Whenever a new event handler is added to an event manager,
+%% this function is called to initialize the event handler.
+%%--------------------------------------------------------------------
+init(_) ->
+ ct_util:mark_process(),
+ ct_master_logs_fork:log("CT Master Event Handler started","",[]),
+ {ok,#state{}}.
+
+%%--------------------------------------------------------------------
+%% Function:
+%% handle_event(Event, State) -> {ok, State} |
+%% {swap_handler, Args1, State1, Mod2, Args2} |
+%% remove_handler
+%% Description:Whenever an event manager receives an event sent using
+%% gen_event:notify/2 or gen_event:sync_notify/2, this function is called for
+%% each installed event handler to handle the event.
+%%--------------------------------------------------------------------
+handle_event(#event{name=start_logging,node=Node,data=RunDir},State) ->
+ ct_master_logs_fork:log("CT Master Event Handler","Got ~ts from ~w",[RunDir,Node]),
+ ct_master_logs_fork:nodedir(Node,RunDir),
+ {ok,State};
+
+handle_event(Event=#event{name=Name,node=Node,data=Data},State) ->
+ print("~n=== ~w ===~n", [?MODULE]),
+ print("~tw on ~w: ~tp~n", [Name,Node,Data]),
+ {ok,maybe_store_event(Event,State)}.
+
+%%--------------------------------------------------------------------
+%% Function:
+%% handle_call(Request, State) -> {ok, Reply, State} |
+%% {swap_handler, Reply, Args1, State1,
+%% Mod2, Args2} |
+%% {remove_handler, Reply}
+%% Description: Whenever an event manager receives a request sent using
+%% gen_event:call/3,4, this function is called for the specified event
+%% handler to handle the request.
+%%--------------------------------------------------------------------
+handle_call(get_results,State=#state{auto_skipped=AutoSkipped,failed=Failed}) ->
+ {ok,#{
+ auto_skipped => lists:sort(AutoSkipped),
+ failed => lists:sort(Failed)
+ },State};
+handle_call(flush,State) ->
+ case process_info(self(),message_queue_len) of
+ {message_queue_len,0} ->
+ {ok,done,State};
+ _ ->
+ {ok,flushing,State}
+ end.
+
+%%--------------------------------------------------------------------
+%% Function:
+%% handle_info(Info, State) -> {ok, State} |
+%% {swap_handler, Args1, State1, Mod2, Args2} |
+%% remove_handler
+%% Description: This function is called for each installed event handler when
+%% an event manager receives any other message than an event or a synchronous
+%% request (or a system message).
+%%--------------------------------------------------------------------
+handle_info(_Info,State) ->
+ {ok,State}.
+
+%%--------------------------------------------------------------------
+%% Function: terminate(Reason, State) -> ok
+%% Description:Whenever an event handler is deleted from an event manager,
+%% this function is called. It should be the opposite of Module:init/1 and
+%% do any necessary cleaning up.
+%%--------------------------------------------------------------------
+terminate(_Reason,_State) ->
+ ct_master_logs_fork:log("CT Master Event Handler stopping","",[]),
+ ok.
+
+%%--------------------------------------------------------------------
+%% Function: code_change(OldVsn, State, Extra) -> {ok, NewState}
+%% Description: Convert process state when code is changed
+%%--------------------------------------------------------------------
+code_change(_OldVsn,State,_Extra) ->
+ {ok,State}.
+
+%%--------------------------------------------------------------------
+%%% Internal functions
+%%--------------------------------------------------------------------
+
+print(_Str,_Args) ->
+% io:format(_Str,_Args),
+ ok.
+
+maybe_store_event(#event{name=tc_done,node=Node,data={Suite,FuncOrGroup,{auto_skipped,Reason}}},State=#state{auto_skipped=Acc}) ->
+ State#state{auto_skipped=[{Node,Suite,FuncOrGroup,Reason}|Acc]};
+maybe_store_event(#event{name=tc_done,node=Node,data={Suite,FuncOrGroup,{failed,Reason}}},State=#state{failed=Acc}) ->
+ State#state{failed=[{Node,Suite,FuncOrGroup,Reason}|Acc]};
+maybe_store_event(_Event,State) ->
+ State.
diff --git a/deps/rabbitmq_ct_helpers/src/ct_master_fork.erl b/deps/rabbitmq_ct_helpers/src/ct_master_fork.erl
index 443635fe912a..a698ca9e1613 100644
--- a/deps/rabbitmq_ct_helpers/src/ct_master_fork.erl
+++ b/deps/rabbitmq_ct_helpers/src/ct_master_fork.erl
@@ -177,23 +177,14 @@ run([TS|TestSpecs],AllowUserTerms,InclNodes,ExclNodes) when is_list(TS),
Tests ->
RunResult =
lists:map(
- fun({Specs,TSRec=#testspec{logdir=AllLogDirs,
- config=StdCfgFiles,
- userconfig=UserCfgFiles,
- include=AllIncludes,
- init=AllInitOpts,
- event_handler=AllEvHs}}) ->
- AllCfgFiles =
- {StdCfgFiles,UserCfgFiles},
+ fun({Specs,TSRec=#testspec{}}) ->
RunSkipPerNode =
ct_testspec:prepare_tests(TSRec),
RunSkipPerNode2 =
exclude_nodes(ExclNodes,RunSkipPerNode),
TSList = if is_integer(hd(TS)) -> [TS];
true -> TS end,
- {Specs,run_all(RunSkipPerNode2,AllLogDirs,
- AllCfgFiles,AllEvHs,
- AllIncludes,[],[],AllInitOpts,TSList)}
+ {Specs,run_all(RunSkipPerNode2,TSRec,[],[],TSList)}
end, Tests),
RunResult ++ run(TestSpecs,AllowUserTerms,InclNodes,ExclNodes)
end;
@@ -258,19 +249,11 @@ run_on_node([TS|TestSpecs],AllowUserTerms,Node) when is_list(TS),is_atom(Node) -
Tests ->
RunResult =
lists:map(
- fun({Specs,TSRec=#testspec{logdir=AllLogDirs,
- config=StdCfgFiles,
- init=AllInitOpts,
- include=AllIncludes,
- userconfig=UserCfgFiles,
- event_handler=AllEvHs}}) ->
- AllCfgFiles = {StdCfgFiles,UserCfgFiles},
+ fun({Specs,TSRec=#testspec{}}) ->
{Run,Skip} = ct_testspec:prepare_tests(TSRec,Node),
TSList = if is_integer(hd(TS)) -> [TS];
true -> TS end,
- {Specs,run_all([{Node,Run,Skip}],AllLogDirs,
- AllCfgFiles,AllEvHs,
- AllIncludes, [],[],AllInitOpts,TSList)}
+ {Specs,run_all([{Node,Run,Skip}],TSRec,[],[],TSList)}
end, Tests),
RunResult ++ run_on_node(TestSpecs,AllowUserTerms,Node)
end;
@@ -291,54 +274,117 @@ run_on_node(TestSpecs,Node) ->
-run_all([{Node,Run,Skip}|Rest],AllLogDirs,
- {AllStdCfgFiles, AllUserCfgFiles}=AllCfgFiles,
- AllEvHs,AllIncludes,NodeOpts,LogDirs,InitOptions,Specs) ->
- LogDir =
- lists:foldl(fun({N,Dir},_Found) when N == Node ->
- Dir;
- ({_N,_Dir},Found) ->
- Found;
- (Dir,".") ->
- Dir;
- (_Dir,Found) ->
- Found
- end,".",AllLogDirs),
-
- StdCfgFiles =
- lists:foldr(fun({N,F},Fs) when N == Node -> [F|Fs];
- ({_N,_F},Fs) -> Fs;
- (F,Fs) -> [F|Fs]
- end,[],AllStdCfgFiles),
- UserCfgFiles =
+run_all([{Node,Run,Skip}|Rest],TSRec=#testspec{label = Labels,
+% profile = Profiles,
+ logdir = LogDirs,
+ logopts = LogOptsList,
+ basic_html = BHs,
+ esc_chars = EscChs,
+ stylesheet = SSs,
+ verbosity = VLvls,
+ silent_connections = SilentConnsList,
+ cover = CoverFs,
+ cover_stop = CoverStops,
+ config = Cfgs,
+ userconfig = UsrCfgs,
+ event_handler = EvHs,
+ ct_hooks = CTHooks,
+ %% Not available in OTP-26. We don't use it so leave commented for now.
+% ct_hooks_order = CTHooksOrder0,
+ enable_builtin_hooks = EnableBuiltinHooks0,
+ auto_compile = ACs,
+ abort_if_missing_suites = AiMSs,
+ include = Incl,
+ multiply_timetraps = MTs,
+ scale_timetraps = STs,
+ create_priv_dir = PDs},
+ NodeOpts,LogDirsRun,Specs) ->
+ %% We mirror ct_run:get_data_for_node to retrieve data from #testspec,
+ %% but set the default values where appropriate.
+ Label = proplists:get_value(Node, Labels),
+% Profile = proplists:get_value(Node, Profiles),
+ LogDir = case proplists:get_value(Node, LogDirs) of
+ undefined -> ".";
+ Dir -> Dir
+ end,
+ LogOpts = case proplists:get_value(Node, LogOptsList) of
+ undefined -> [];
+ LOs -> LOs
+ end,
+ BasicHtml = proplists:get_value(Node, BHs, false),
+ EscChars = proplists:get_value(Node, EscChs, true),
+ Stylesheet = proplists:get_value(Node, SSs),
+ Verbosity = case proplists:get_value(Node, VLvls) of
+ undefined -> [];
+ Lvls -> Lvls
+ end,
+ SilentConns = case proplists:get_value(Node, SilentConnsList) of
+ undefined -> [];
+ SCs -> SCs
+ end,
+ Cover = proplists:get_value(Node, CoverFs),
+ CoverStop = proplists:get_value(Node, CoverStops, true),
+ MT = proplists:get_value(Node, MTs, 1),
+ ST = proplists:get_value(Node, STs, false),
+ CreatePrivDir = proplists:get_value(Node, PDs, auto_per_run),
+ %% For these two values we can't exactly mirror get_data_for_node.
+ ConfigFiles =
+ lists:foldr(fun({N,F},Fs) when N == Node -> [F|Fs];
+ ({_N,_F},Fs) -> Fs;
+ (F,Fs) -> [F|Fs]
+ end,[],Cfgs),
+ UsrConfigFiles =
lists:foldr(fun({N,F},Fs) when N == Node -> [{userconfig, F}|Fs];
- ({_N,_F},Fs) -> Fs;
- (F,Fs) -> [{userconfig, F}|Fs]
- end,[],AllUserCfgFiles),
-
- Includes = lists:foldr(fun({N,I},Acc) when N =:= Node ->
- [I|Acc];
- ({_,_},Acc) ->
- Acc;
- (I,Acc) ->
- [I | Acc]
- end, [], AllIncludes),
- EvHs =
- lists:foldr(fun({N,H,A},Hs) when N == Node -> [{H,A}|Hs];
- ({_N,_H,_A},Hs) -> Hs;
- ({H,A},Hs) -> [{H,A}|Hs]
- end,[],AllEvHs),
-
- NO = {Node,[{prepared_tests,{Run,Skip},Specs},
- {ct_hooks, [cth_parallel_ct_detect_failure]},
- {logdir,LogDir},
- {include, Includes},
- {config,StdCfgFiles},
- {event_handler,EvHs}] ++ UserCfgFiles},
- run_all(Rest,AllLogDirs,AllCfgFiles,AllEvHs,AllIncludes,
- [NO|NodeOpts],[LogDir|LogDirs],InitOptions,Specs);
-run_all([],AllLogDirs,_,AllEvHs,_AllIncludes,
- NodeOpts,LogDirs,InitOptions,Specs) ->
+ ({_N,_F},Fs) -> Fs;
+ (F,Fs) -> [{userconfig, F}|Fs]
+ end,[],UsrCfgs),
+ EvHandlers = [{H,A} || {N,H,A} <- EvHs, N==Node],
+ FiltCTHooks = [Hook || {N,Hook} <- CTHooks, N==Node],
+% CTHooksOrder = case CTHooksOrder0 of
+% undefined -> test;
+% _ -> CTHooksOrder0
+% end,
+ EnableBuiltinHooks = case EnableBuiltinHooks0 of
+ undefined -> true;
+ _ -> EnableBuiltinHooks0
+ end,
+ AutoCompile = proplists:get_value(Node, ACs, true),
+ AbortIfMissing = proplists:get_value(Node, AiMSs, false),
+ Include = [I || {N,I} <- Incl, N==Node],
+ %% We then build the ct:run_test/1 options list.
+ RunTestOpts0 =
+ [{label, Label} || Label =/= undefined] ++
+ [{stylesheet, Stylesheet} || Stylesheet =/= undefined] ++
+ [{cover, Cover} || Cover =/= undefined] ++
+ UsrConfigFiles,
+ RunTestOpts = [
+% {profile, Profile},
+ {logdir, LogDir},
+ {logopts, LogOpts},
+ {basic_html, BasicHtml},
+ {esc_chars, EscChars},
+ {verbosity, Verbosity},
+ {silent_connections, SilentConns},
+ {cover_stop, CoverStop},
+ {config, ConfigFiles},
+ {event_handler, EvHandlers},
+ {ct_hooks, FiltCTHooks},
+% {ct_hooks_order, CTHooksOrder},
+ {enable_builtin_hooks, EnableBuiltinHooks},
+ {auto_compile, AutoCompile},
+ {abort_if_missing_suites, AbortIfMissing},
+ {include, Include},
+ {multiply_timetraps, MT},
+ {scale_timetraps, ST},
+ {create_priv_dir, CreatePrivDir}
+ |RunTestOpts0],
+ NO = {Node,[{prepared_tests,{Run,Skip},Specs}|RunTestOpts]},
+ run_all(Rest,TSRec,[NO|NodeOpts],[LogDir|LogDirsRun],Specs);
+run_all([],#testspec{
+ logdir=AllLogDirs,
+ init=InitOptions,
+ event_handler=AllEvHs},
+ NodeOpts,LogDirsRun,Specs) ->
Handlers = [{H,A} || {Master,H,A} <- AllEvHs, Master == master],
MasterLogDir = case lists:keysearch(master,1,AllLogDirs) of
{value,{_,Dir}} -> Dir;
@@ -346,8 +392,7 @@ run_all([],AllLogDirs,_,AllEvHs,_AllIncludes,
end,
log(tty,"Master Logdir","~ts",[MasterLogDir]),
start_master(lists:reverse(NodeOpts),Handlers,MasterLogDir,
- LogDirs,InitOptions,Specs),
- ok.
+ LogDirsRun,InitOptions,Specs).
%-doc """
@@ -446,7 +491,7 @@ init_master(Parent,NodeOptsList,EvHandlers,MasterLogDir,LogDirs,
end,
%% start master logger
- {MLPid,_} = ct_master_logs:start(MasterLogDir,
+ {MLPid,_} = ct_master_logs_fork:start(MasterLogDir,
[N || {N,_} <- NodeOptsList]),
log(all,"Master Logger process started","~w",[MLPid]),
@@ -456,13 +501,13 @@ init_master(Parent,NodeOptsList,EvHandlers,MasterLogDir,LogDirs,
SpecsStr = lists:map(fun(Name) ->
Name ++ " "
end,Specs),
- ct_master_logs:log("Test Specification file(s)","~ts",
+ ct_master_logs_fork:log("Test Specification file(s)","~ts",
[lists:flatten(SpecsStr)])
end,
%% start master event manager and add default handler
{ok, _} = start_ct_master_event(),
- ct_master_event:add_handler(),
+ ct_master_event_fork:add_handler(),
%% add user handlers for master event manager
Add = fun({H,Args}) ->
log(all,"Adding Event Handler","~w",[H]),
@@ -484,7 +529,7 @@ init_master(Parent,NodeOptsList,EvHandlers,MasterLogDir,LogDirs,
init_master1(Parent,NodeOptsList,InitOptions,LogDirs).
start_ct_master_event() ->
- case ct_master_event:start_link() of
+ case ct_master_event_fork:start_link() of
{error, {already_started, Pid}} ->
{ok, Pid};
Else ->
@@ -510,8 +555,8 @@ init_master1(Parent,NodeOptsList,InitOptions,LogDirs) ->
init_master1(Parent,NodeOptsList,InitOptions1,LogDirs);
_ ->
log(html,"Aborting Tests","",[]),
- ct_master_event:stop(),
- ct_master_logs:stop(),
+ ct_master_event_fork:stop(),
+ ct_master_logs_fork:stop(),
exit(aborted)
end
end.
@@ -535,20 +580,22 @@ init_master2(Parent,NodeOptsList,LogDirs) ->
Parent ! {self(),Result}.
master_loop(#state{node_ctrl_pids=[],
- logdirs=LogDirs,
- results=Finished}) ->
+ results=Finished0}) ->
+ Finished = lists:sort(Finished0),
Str =
lists:map(fun({Node,Result}) ->
io_lib:format("~-40.40.*ts~tp\n",
[$_,atom_to_list(Node),Result])
- end,lists:reverse(Finished)),
+ end,Finished),
log(all,"TEST RESULTS","~ts", [Str]),
log(all,"Info","Updating log files",[]),
- refresh_logs(LogDirs,[]),
-
- ct_master_event:stop(),
- ct_master_logs:stop(),
- ok;
+
+ %% Print the failed and auto skipped tests.
+ master_print_summary(),
+
+ ct_master_event_fork:stop(),
+ ct_master_logs_fork:stop(),
+ {ok, Finished};
master_loop(State=#state{node_ctrl_pids=NodeCtrlPids,
results=Results,
@@ -658,11 +705,31 @@ master_loop(State=#state{node_ctrl_pids=NodeCtrlPids,
blocked=Blocked1});
{cast,Event} when is_record(Event,event) ->
- ct_master_event:notify(Event),
+ ct_master_event_fork:notify(Event),
master_loop(State)
end.
+master_print_summary() ->
+ #{
+ auto_skipped := AutoSkipped,
+ failed := Failed
+ } = ct_master_event_fork:get_results(),
+ master_print_summary_for("Auto skipped test cases", AutoSkipped),
+ master_print_summary_for("Failed test cases", Failed),
+ ok.
+
+master_print_summary_for(Title,List) ->
+ _ = case List of
+ [] -> ok;
+ _ ->
+ Chars = [
+ io_lib:format("Node: ~w~nCase: ~w:~w~nReason: ~p~n~n",
+ [Node, Suite, FuncOrGroup, Reason])
+ || {Node, Suite, FuncOrGroup, Reason} <- List],
+ log(all,Title,Chars,[])
+ end,
+ ok.
update_queue(take,Node,From,Lock={Op,Resource},Locks,Blocked) ->
%% Locks: [{{Operation,Resource},Node},...]
@@ -740,34 +807,6 @@ master_progress(NodeCtrlPids,Results) ->
{Node,ongoing}
end,NodeCtrlPids).
-%% refresh those dirs where more than one node has written logs
-refresh_logs([D|Dirs],Refreshed) ->
- case lists:member(D,Dirs) of
- true ->
- case lists:keymember(D,1,Refreshed) of
- true ->
- refresh_logs(Dirs,Refreshed);
- false ->
- {ok,Cwd} = file:get_cwd(),
- case catch ct_run:refresh_logs(D, unknown) of
- {'EXIT',Reason} ->
- ok = file:set_cwd(Cwd),
- refresh_logs(Dirs,[{D,{error,Reason}}|Refreshed]);
- Result ->
- refresh_logs(Dirs,[{D,Result}|Refreshed])
- end
- end;
- false ->
- refresh_logs(Dirs,Refreshed)
- end;
-refresh_logs([],Refreshed) ->
- Str =
- lists:map(fun({D,Result}) ->
- io_lib:format("Refreshing logs in ~tp... ~tp",
- [D,Result])
- end,Refreshed),
- log(all,"Info","~ts", [Str]).
-
%%%-----------------------------------------------------------------
%%% NODE CONTROLLER, runs and controls tests on a test node.
%%%-----------------------------------------------------------------
@@ -851,7 +890,7 @@ log(To,Heading,Str,Args) ->
ok
end,
if To == all ; To == html ->
- ct_master_logs:log(Heading,Str,Args);
+ ct_master_logs_fork:log(Heading,Str,Args);
true ->
ok
end.
diff --git a/deps/rabbitmq_ct_helpers/src/ct_master_logs_fork.erl b/deps/rabbitmq_ct_helpers/src/ct_master_logs_fork.erl
new file mode 100644
index 000000000000..9541c941708b
--- /dev/null
+++ b/deps/rabbitmq_ct_helpers/src/ct_master_logs_fork.erl
@@ -0,0 +1,569 @@
+%%
+%% %CopyrightBegin%
+%%
+%% Copyright Ericsson AB 2006-2024. All Rights Reserved.
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+%%
+%% %CopyrightEnd%
+%%
+
+%%% Logging functionality for Common Test Master.
+%%%
+%%% This module implements a logger for the master
+%%% node.
+-module(ct_master_logs_fork).
+-moduledoc false.
+
+-export([start/2, make_all_runs_index/0, log/3, nodedir/2,
+ stop/0]).
+
+-include_lib("common_test/src/ct_util.hrl").
+
+-record(state, {log_fd, start_time, logdir, rundir,
+ nodedir_ix_fd, nodes, nodedirs=[]}).
+
+-define(ct_master_log_name, "ct_master_log.html").
+-define(all_runs_name, "master_runs.html").
+-define(nodedir_index_name, "index.html").
+-define(details_file_name,"details.info").
+-define(table_color,"lightblue").
+
+-define(now, os:timestamp()).
+
+%%%--------------------------------------------------------------------
+%%% API
+%%%--------------------------------------------------------------------
+
+start(LogDir,Nodes) ->
+ Self = self(),
+ Pid = spawn_link(fun() -> init(Self,LogDir,Nodes) end),
+ MRef = erlang:monitor(process,Pid),
+ receive
+ {started,Pid,Result} ->
+ erlang:demonitor(MRef, [flush]),
+ {Pid,Result};
+ {'DOWN',MRef,process,_,Reason} ->
+ exit({could_not_start_process,?MODULE,Reason})
+ end.
+
+log(Heading,Format,Args) ->
+ cast({log,self(),[{int_header(),[log_timestamp(?now),Heading]},
+ {Format,Args},
+ {int_footer(),[]}]}),
+ ok.
+
+make_all_runs_index() ->
+ call(make_all_runs_index).
+
+nodedir(Node,RunDir) ->
+ call({nodedir,Node,RunDir}).
+
+stop() ->
+ case whereis(?MODULE) of
+ Pid when is_pid(Pid) ->
+ MRef = erlang:monitor(process,Pid),
+ ?MODULE ! stop,
+ receive
+ {'DOWN',MRef,process,_,_} ->
+ ok
+ end;
+ undefined ->
+ ok
+ end,
+ ok.
+
+%%%--------------------------------------------------------------------
+%%% Logger process
+%%%--------------------------------------------------------------------
+
+init(Parent,LogDir,Nodes) ->
+ register(?MODULE,self()),
+ ct_util:mark_process(),
+ Time = calendar:local_time(),
+ RunDir = make_dirname(Time),
+ RunDirAbs = filename:join(LogDir,RunDir),
+ ok = make_dir(RunDirAbs),
+ _ = write_details_file(RunDirAbs,{node(),Nodes}),
+
+ case basic_html() of
+ true ->
+ put(basic_html, true);
+ BasicHtml ->
+ put(basic_html, BasicHtml),
+ %% copy priv files to log dir (both top dir and test run
+ %% dir) so logs are independent of Common Test installation
+ CTPath = code:lib_dir(common_test),
+ PrivFiles = [?css_default,?jquery_script,?tablesorter_script],
+ PrivFilesSrc = [filename:join(filename:join(CTPath, "priv"), F) ||
+ F <- PrivFiles],
+ PrivFilesDestTop = [filename:join(LogDir, F) || F <- PrivFiles],
+ PrivFilesDestRun = [filename:join(RunDirAbs, F) || F <- PrivFiles],
+ case copy_priv_files(PrivFilesSrc, PrivFilesDestTop) of
+ {error,Src1,Dest1,Reason1} ->
+ io:format(user, "ERROR! "++
+ "Priv file ~tp could not be copied to ~tp. "++
+ "Reason: ~tp~n",
+ [Src1,Dest1,Reason1]),
+ exit({priv_file_error,Dest1});
+ ok ->
+ case copy_priv_files(PrivFilesSrc, PrivFilesDestRun) of
+ {error,Src2,Dest2,Reason2} ->
+ io:format(user, "ERROR! "++
+ "Priv file ~tp could not be copied to ~tp. "++
+ "Reason: ~tp~n",
+ [Src2,Dest2,Reason2]),
+ exit({priv_file_error,Dest2});
+ ok ->
+ ok
+ end
+ end
+ end,
+
+ {ok,Cwd} = file:get_cwd(),
+ ok = file:set_cwd(LogDir),
+ _ = make_all_runs_index(LogDir),
+ CtLogFd = open_ct_master_log(RunDirAbs),
+ ok = file:set_cwd(Cwd),
+
+ NodeStr =
+ lists:flatten(lists:map(fun(N) ->
+ atom_to_list(N) ++ " "
+ end,Nodes)),
+
+ io:format(CtLogFd,int_header(),[log_timestamp(?now),"Test Nodes\n"]),
+ io:format(CtLogFd,"~ts\n",[NodeStr]),
+ io:put_chars(CtLogFd,[int_footer(),"\n"]),
+
+ NodeDirIxFd = open_nodedir_index(RunDirAbs,Time),
+ Parent ! {started,self(),{Time,RunDirAbs}},
+ loop(#state{log_fd=CtLogFd,
+ start_time=Time,
+ logdir=LogDir,
+ rundir=RunDirAbs,
+ nodedir_ix_fd=NodeDirIxFd,
+ nodes=Nodes,
+ nodedirs=lists:map(fun(N) ->
+ {N,""}
+ end,Nodes)}).
+
+copy_priv_files([SrcF | SrcFs], [DestF | DestFs]) ->
+ case file:copy(SrcF, DestF) of
+ {error,Reason} ->
+ {error,SrcF,DestF,Reason};
+ _ ->
+ copy_priv_files(SrcFs, DestFs)
+ end;
+copy_priv_files([], []) ->
+ ok.
+
+loop(State) ->
+ receive
+ {log,_From,List} ->
+ Fd = State#state.log_fd,
+ Fun =
+ fun({Str,Args}) ->
+ case catch io:format(Fd,Str++"\n",Args) of
+ {'EXIT',Reason} ->
+ io:format(Fd,
+ "Logging fails! Str: ~tp, Args: ~tp~n",
+ [Str,Args]),
+ exit({logging_failed,Reason}),
+ ok;
+ _ ->
+ ok
+ end
+ end,
+ lists:foreach(Fun,List),
+ loop(State);
+ {make_all_runs_index,From} ->
+ {ok,Cwd} = file:get_cwd(),
+ ok = file:set_cwd(State#state.logdir),
+ _ = make_all_runs_index(State#state.logdir),
+ ok = file:set_cwd(Cwd),
+ return(From,State#state.logdir),
+ loop(State);
+ {{nodedir,Node,RunDir},From} ->
+ print_nodedir(Node,RunDir,State#state.nodedir_ix_fd),
+ return(From,ok),
+ loop(State);
+ stop ->
+ {ok,Cwd} = file:get_cwd(),
+ ok = file:set_cwd(State#state.logdir),
+ _ = make_all_runs_index(State#state.logdir),
+ ok = file:set_cwd(Cwd),
+ io:format(State#state.log_fd,
+ int_header()++int_footer(),
+ [log_timestamp(?now),"Finished!"]),
+ _ = close_ct_master_log(State#state.log_fd),
+ _ = close_nodedir_index(State#state.nodedir_ix_fd),
+ ok
+ end.
+
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%% Master Log functions %%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+open_ct_master_log(Dir) ->
+ FullName = filename:join(Dir,?ct_master_log_name),
+ {ok,Fd} = file:open(FullName,[write,{encoding,utf8}]),
+ io:put_chars(Fd,header("Common Test Master Log", {[],[1,2],[]})),
+ %% maybe add config info here later
+ io:put_chars(Fd,config_table([])),
+ io:put_chars(Fd,
+ "\n"),
+ io:put_chars(Fd,
+ xhtml("
Progress Log
\n\n",
+ "
Progress Log
\n\n")),
+ Fd.
+
+close_ct_master_log(Fd) ->
+ io:put_chars(Fd,["
",footer()]),
+ file:close(Fd).
+
+config_table(Vars) ->
+ [config_table_header()|config_table1(Vars)].
+
+config_table_header() ->
+ ["Configuration
\n",
+ xhtml(["
\n",
+ "\n"]),
+ "| Key | Value |
|---|
\n",
+ xhtml("", "\n\n")].
+
+config_table1([]) ->
+ ["\n
\n"].
+
+int_header() ->
+ "\n*** CT MASTER ~s *** ~ts".
+int_footer() ->
+ "
".
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%% NodeDir Index functions %%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+open_nodedir_index(Dir,StartTime) ->
+ FullName = filename:join(Dir,?nodedir_index_name),
+ {ok,Fd} = file:open(FullName,[write,{encoding,utf8}]),
+ io:put_chars(Fd,nodedir_index_header(StartTime)),
+ Fd.
+
+print_nodedir(Node,RunDir,Fd) ->
+ Index = filename:join(RunDir,"index.html"),
+ io:put_chars(Fd,
+ ["\n"
+ "| ",atom_to_list(Node)," | \n",
+ "",Index,
+ " | \n",
+ "
\n"]),
+ ok.
+
+close_nodedir_index(Fd) ->
+ io:put_chars(Fd,index_footer()),
+ file:close(Fd).
+
+nodedir_index_header(StartTime) ->
+ [header("Log Files " ++ format_time(StartTime), {[],[1,2],[]}) |
+ ["\n",
+ "Common Test Master Log
",
+ xhtml(["\n"],
+ ["\n",
+ "\n\n"]),
+ "| Node | \n",
+ "Log | \n",
+ xhtml("", "
\n\n\n")]].
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+%%% All Run Index functions %%%
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+make_all_runs_index(LogDir) ->
+ FullName = filename:join(LogDir,?all_runs_name),
+ Match = filename:join(LogDir,logdir_prefix()++"*.*"),
+ Dirs = filelib:wildcard(Match),
+ DirsSorted = (catch sort_all_runs(Dirs)),
+ Header = all_runs_header(),
+ Index = [runentry(Dir) || Dir <- DirsSorted],
+ Result = file:write_file(FullName,
+ unicode:characters_to_binary(
+ Header++Index++index_footer())),
+ Result.
+
+sort_all_runs(Dirs) ->
+ %% sort on time string, always last and on the format:
+ %% "YYYY-MM-DD_HH.MM.SS"
+ KeyList =
+ lists:map(fun(Dir) ->
+ case lists:reverse(string:lexemes(Dir,[$.,$_])) of
+ [SS,MM,HH,Date|_] ->
+ {{Date,HH,MM,SS},Dir};
+ _Other ->
+ throw(Dirs)
+ end
+ end,Dirs),
+ lists:reverse(lists:map(fun({_,Dir}) ->
+ Dir
+ end,lists:keysort(1,KeyList))).
+
+runentry(Dir) ->
+ {MasterStr,NodesStr} =
+ case read_details_file(Dir) of
+ {Master,Nodes} when is_list(Nodes) ->
+ [_,Host] = string:lexemes(atom_to_list(Master),"@"),
+ {Host,lists:concat(lists:join(", ",Nodes))};
+ _Error ->
+ {"unknown",""}
+ end,
+ Index = filename:join(Dir,?nodedir_index_name),
+ ["\n"
+ "| ",
+ timestamp(Dir)," | \n",
+ "",MasterStr," | \n",
+ "",NodesStr," | \n",
+ "
\n"].
+
+all_runs_header() ->
+ [header("Master Test Runs", {[1],[2,3],[]}) |
+ ["\n",
+ xhtml(["\n"],
+ ["\n",
+ "\n\n"]),
+ "| History | \n"
+ "Master Host | \n"
+ "Test Nodes | \n",
+ xhtml("", "
\n\n")]].
+
+timestamp(Dir) ->
+ [S,Min,H,D,M,Y|_] = lists:reverse(string:lexemes(Dir,".-_")),
+ [S1,Min1,H1,D1,M1,Y1] = [list_to_integer(N) || N <- [S,Min,H,D,M,Y]],
+ format_time({{Y1,M1,D1},{H1,Min1,S1}}).
+
+write_details_file(Dir,Details) ->
+ FullName = filename:join(Dir,?details_file_name),
+ force_write_file(FullName,term_to_binary(Details)).
+
+read_details_file(Dir) ->
+ FullName = filename:join(Dir,?details_file_name),
+ case file:read_file(FullName) of
+ {ok,Bin} ->
+ binary_to_term(Bin);
+ Error ->
+ Error
+ end.
+
+%%%--------------------------------------------------------------------
+%%% Internal functions
+%%%--------------------------------------------------------------------
+
+header(Title, TableCols) ->
+ CSSFile = xhtml(fun() -> "" end,
+ fun() -> make_relative(locate_priv_file(?css_default)) end),
+ JQueryFile =
+ xhtml(fun() -> "" end,
+ fun() -> make_relative(locate_priv_file(?jquery_script)) end),
+ TableSorterFile =
+ xhtml(fun() -> "" end,
+ fun() -> make_relative(locate_priv_file(?tablesorter_script)) end),
+
+ [xhtml(["\n",
+ "\n"],
+ ["\n",
+ "\n"]),
+ "\n",
+ "\n",
+ "" ++ Title ++ "\n",
+ "\n",
+ "\n",
+ xhtml("",
+ ["\n"]),
+ xhtml("",
+ ["\n"]),
+ xhtml("",
+ ["\n"]),
+ xhtml(fun() -> "" end,
+ fun() -> ct_logs:insert_javascript({tablesorter,
+ ?sortable_table_name,
+ TableCols}) end),
+ "\n",
+ body_tag(),
+ "\n",
+ "" ++ Title ++ "
\n",
+ "\n"].
+
+index_footer() ->
+ ["\n
\n"
+ "\n" | footer()].
+
+footer() ->
+ ["\n",
+ xhtml("
\n", "
\n"),
+ xhtml("\n", ""),
+ "Copyright © ", year(),
+ "
Open Telecom Platform",
+ xhtml("
\n", "
\n"),
+ "Updated: ", current_time(), "",
+ xhtml("
\n", "
\n"),
+ xhtml("\n", "
\n"
+ "\n"].
+
+body_tag() ->
+ xhtml("\n",
+ "\n").
+
+current_time() ->
+ format_time(calendar:local_time()).
+
+format_time({{Y, Mon, D}, {H, Min, S}}) ->
+ Weekday = weekday(calendar:day_of_the_week(Y, Mon, D)),
+ lists:flatten(io_lib:format("~s ~s ~2.2.0w ~w ~2.2.0w:~2.2.0w:~2.2.0w",
+ [Weekday, month(Mon), D, Y, H, Min, S])).
+
+weekday(1) -> "Mon";
+weekday(2) -> "Tue";
+weekday(3) -> "Wed";
+weekday(4) -> "Thu";
+weekday(5) -> "Fri";
+weekday(6) -> "Sat";
+weekday(7) -> "Sun".
+
+month(1) -> "Jan";
+month(2) -> "Feb";
+month(3) -> "Mar";
+month(4) -> "Apr";
+month(5) -> "May";
+month(6) -> "Jun";
+month(7) -> "Jul";
+month(8) -> "Aug";
+month(9) -> "Sep";
+month(10) -> "Oct";
+month(11) -> "Nov";
+month(12) -> "Dec".
+
+year() ->
+ {Y, _, _} = date(),
+ integer_to_list(Y).
+
+
+make_dirname({{YY,MM,DD},{H,M,S}}) ->
+ io_lib:format(logdir_prefix()++".~w-~2.2.0w-~2.2.0w_~2.2.0w.~2.2.0w.~2.2.0w",
+ [YY,MM,DD,H,M,S]).
+
+logdir_prefix() ->
+ "ct_master_run".
+
+log_timestamp(Now) ->
+ put(log_timestamp,Now),
+ {_,{H,M,S}} = calendar:now_to_local_time(Now),
+ lists:flatten(io_lib:format("~2.2.0w:~2.2.0w:~2.2.0w",
+ [H,M,S])).
+
+basic_html() ->
+ case application:get_env(common_test_master, basic_html) of
+ {ok,true} ->
+ true;
+ _ ->
+ false
+ end.
+
+xhtml(HTML, XHTML) ->
+ ct_logs:xhtml(HTML, XHTML).
+
+locate_priv_file(File) ->
+ ct_logs:locate_priv_file(File).
+
+make_relative(Dir) ->
+ ct_logs:make_relative(Dir).
+
+force_write_file(Name,Contents) ->
+ _ = force_delete(Name),
+ file:write_file(Name,Contents).
+
+force_delete(Name) ->
+ case file:delete(Name) of
+ {error,eacces} ->
+ force_rename(Name,Name++".old.",0);
+ Other ->
+ Other
+ end.
+
+force_rename(From,To,Number) ->
+ Dest = [To|integer_to_list(Number)],
+ case file:read_file_info(Dest) of
+ {ok,_} ->
+ force_rename(From,To,Number+1);
+ {error,_} ->
+ file:rename(From,Dest)
+ end.
+
+call(Msg) ->
+ case whereis(?MODULE) of
+ undefined ->
+ {error,does_not_exist};
+ Pid ->
+ MRef = erlang:monitor(process,Pid),
+ Ref = make_ref(),
+ ?MODULE ! {Msg,{self(),Ref}},
+ receive
+ {Ref, Result} ->
+ erlang:demonitor(MRef, [flush]),
+ Result;
+ {'DOWN',MRef,process,_,Reason} ->
+ {error,{process_down,?MODULE,Reason}}
+ end
+ end.
+
+return({To,Ref},Result) ->
+ To ! {Ref, Result},
+ ok.
+
+cast(Msg) ->
+ case whereis(?MODULE) of
+ undefined ->
+ io:format("Warning: ct_master_logs not started~n"),
+ {_,_,Content} = Msg,
+ FormatArgs = get_format_args(Content),
+ _ = [io:format(Format, Args) || {Format, Args} <- FormatArgs],
+ ok;
+ _Pid ->
+ ?MODULE ! Msg,
+ ok
+ end.
+
+get_format_args(Content) ->
+ lists:map(fun(C) ->
+ case C of
+ {_, FA, _} -> FA;
+ _ -> C
+ end
+ end, Content).
+
+make_dir(Dir) ->
+ case file:make_dir(Dir) of
+ {error, eexist} ->
+ ok;
+ Else ->
+ Else
+ end.
diff --git a/deps/rabbitmq_ct_helpers/src/cth_parallel_ct_detect_failure.erl b/deps/rabbitmq_ct_helpers/src/cth_parallel_ct_detect_failure.erl
deleted file mode 100644
index 428e37468bf4..000000000000
--- a/deps/rabbitmq_ct_helpers/src/cth_parallel_ct_detect_failure.erl
+++ /dev/null
@@ -1,23 +0,0 @@
--module(cth_parallel_ct_detect_failure).
-
--export([init/2]).
--export([on_tc_fail/4]).
--export([has_failures/0]).
-
-init(_Id, _Opts) ->
- {ok, undefined}.
-
-%% We silence failures in end_per_suite/end_per_group
-%% to mirror the default behavior. It should be modified
-%% so that they are configured failures as well, but can
-%% be done at a later time.
-on_tc_fail(_SuiteName, end_per_suite, _Reason, CTHState) ->
- CTHState;
-on_tc_fail(_SuiteName, {end_per_group, _GroupName}, _Reason, CTHState) ->
- CTHState;
-on_tc_fail(_SuiteName, _TestName, _Reason, CTHState) ->
- persistent_term:put(?MODULE, true),
- CTHState.
-
-has_failures() ->
- persistent_term:get(?MODULE, false).
diff --git a/deps/rabbitmq_ct_helpers/src/rabbit_ct_broker_helpers.erl b/deps/rabbitmq_ct_helpers/src/rabbit_ct_broker_helpers.erl
index b01ea002842e..77c78cc98ac5 100644
--- a/deps/rabbitmq_ct_helpers/src/rabbit_ct_broker_helpers.erl
+++ b/deps/rabbitmq_ct_helpers/src/rabbit_ct_broker_helpers.erl
@@ -275,7 +275,7 @@ run_make_dist(Config) ->
end;
_ ->
global:del_lock(LockId, [node()]),
- ct:pal(?LOW_IMPORTANCE, "(skip `$MAKE test-dist`)", []),
+ ct:log(?LOW_IMPORTANCE, "(skip `$MAKE test-dist`)", []),
Config
end.
@@ -819,7 +819,7 @@ query_node(Config, NodeConfig) ->
%% 3.7.x node. If this is the case, we can ignore
%% this and leave the `enabled_plugins_file` config
%% variable unset.
- ct:pal("NO RABBITMQ_FEATURE_FLAGS_FILE"),
+ ct:log("NO RABBITMQ_FEATURE_FLAGS_FILE"),
Vars0
end,
cover_add_node(Nodename),
@@ -883,7 +883,7 @@ handle_nodes_in_parallel(NodeConfigs, Fun) ->
T1 = erlang:monotonic_time(),
Ret = Fun(NodeConfig),
T2 = erlang:monotonic_time(),
- ct:pal(
+ ct:log(
?LOW_IMPORTANCE,
"Time to run ~tp for node ~ts: ~b us",
[Fun,
@@ -901,7 +901,7 @@ handle_nodes_in_parallel(NodeConfigs, Fun) ->
wait_for_node_handling([], Fun, T0, Results) ->
T3 = erlang:monotonic_time(),
- ct:pal(
+ ct:log(
?LOW_IMPORTANCE,
"Time to run ~tp for all nodes: ~b us",
[Fun, erlang:convert_time_unit(T3 - T0, native, microsecond)]),
@@ -956,7 +956,7 @@ configured_metadata_store(Config) ->
end.
configure_metadata_store(Config) ->
- ct:pal("Configuring metadata store..."),
+ ct:log("Configuring metadata store..."),
case configured_metadata_store(Config) of
{khepri, FFs0} ->
case enable_khepri_metadata_store(Config, FFs0) of
@@ -967,12 +967,12 @@ configure_metadata_store(Config) ->
Config1
end;
mnesia ->
- ct:pal("Enabling Mnesia metadata store"),
+ ct:log("Enabling Mnesia metadata store"),
Config
end.
enable_khepri_metadata_store(Config, FFs0) ->
- ct:pal("Enabling Khepri metadata store"),
+ ct:log("Enabling Khepri metadata store"),
FFs = [khepri_db | FFs0],
lists:foldl(fun(_FF, {skip, _Reason} = Skip) ->
Skip;
@@ -1143,7 +1143,7 @@ stop_rabbitmq_node(Config, NodeConfig) ->
NodeConfig.
find_crashes_in_logs(NodeConfigs, IgnoredCrashes) ->
- ct:pal(
+ ct:log(
"Looking up any crash reports in the nodes' log files. If we find "
"some, they will appear below:"),
CrashesCount = lists:foldl(
@@ -1152,7 +1152,11 @@ find_crashes_in_logs(NodeConfigs, IgnoredCrashes) ->
NodeConfig, IgnoredCrashes),
Total + Count
end, 0, NodeConfigs),
- ct:pal("Found ~b crash report(s)", [CrashesCount]),
+ LogFn = case CrashesCount of
+ 0 -> log;
+ _ -> pal
+ end,
+ ct:LogFn("Found ~b crash report(s)", [CrashesCount]),
?assertEqual(0, CrashesCount).
count_crashes_in_logs(NodeConfig, IgnoredCrashes) ->
diff --git a/deps/rabbitmq_ct_helpers/src/rabbit_ct_helpers.erl b/deps/rabbitmq_ct_helpers/src/rabbit_ct_helpers.erl
index d9e34cf38fa6..c9b351ddd6ab 100644
--- a/deps/rabbitmq_ct_helpers/src/rabbit_ct_helpers.erl
+++ b/deps/rabbitmq_ct_helpers/src/rabbit_ct_helpers.erl
@@ -66,7 +66,7 @@
log_environment() ->
Vars = lists:sort(fun(A, B) -> A =< B end, os:getenv()),
- ct:pal(?LOW_IMPORTANCE, "Environment variables:~n~ts",
+ ct:log(?LOW_IMPORTANCE, "Environment variables:~n~ts",
[[io_lib:format(" ~ts~n", [V]) || V <- Vars]]).
run_setup_steps(Config) ->
@@ -152,7 +152,7 @@ run_steps(Config, []) ->
Config.
redirect_logger_to_ct_logs(Config) ->
- ct:pal(
+ ct:log(
?LOW_IMPORTANCE,
"Configuring logger to send logs to common_test logs"),
ok = logger:set_handler_config(cth_log_redirect, level, debug),
@@ -172,7 +172,7 @@ redirect_logger_to_ct_logs(Config) ->
ok = logger:remove_handler(default),
- ct:pal(
+ ct:log(
?LOW_IMPORTANCE,
"Logger configured to send logs to common_test logs; you should see "
"a message below saying so"),
@@ -433,12 +433,12 @@ ensure_rabbitmqctl_cmd(Config) ->
false ->
find_script(Config, "rabbitmqctl");
R ->
- ct:pal(?LOW_IMPORTANCE,
+ ct:log(?LOW_IMPORTANCE,
"Using rabbitmqctl from RABBITMQCTL: ~tp~n", [R]),
R
end;
R ->
- ct:pal(?LOW_IMPORTANCE,
+ ct:log(?LOW_IMPORTANCE,
"Using rabbitmqctl from rabbitmqctl_cmd: ~tp~n", [R]),
R
end,
@@ -470,7 +470,7 @@ find_script(Config, Script) ->
filelib:is_file(File)],
case Locations of
[Location | _] ->
- ct:pal(?LOW_IMPORTANCE, "Using ~ts at ~tp~n", [Script, Location]),
+ ct:log(?LOW_IMPORTANCE, "Using ~ts at ~tp~n", [Script, Location]),
Location;
[] ->
false
@@ -555,7 +555,7 @@ ensure_rabbitmq_queues_cmd(Config) ->
R -> R
end;
R ->
- ct:pal(?LOW_IMPORTANCE,
+ ct:log(?LOW_IMPORTANCE,
"Using rabbitmq-queues from rabbitmq_queues_cmd: ~tp~n", [R]),
R
end,
@@ -654,12 +654,12 @@ symlink_priv_dir(Config) ->
Target = filename:join([SrcDir, "logs", Name]),
case exec(["ln", "-snf", PrivDir, Target]) of
{ok, _} -> ok;
- _ -> ct:pal(?LOW_IMPORTANCE,
+ _ -> ct:log(?LOW_IMPORTANCE,
"Failed to symlink private_log directory.")
end,
Config;
not_found ->
- ct:pal(?LOW_IMPORTANCE,
+ ct:log(?LOW_IMPORTANCE,
"Failed to symlink private_log directory."),
Config
end
@@ -684,7 +684,7 @@ load_elixir(Config) ->
{skip, _} = Skip ->
Skip;
ElixirLibDir ->
- ct:pal(?LOW_IMPORTANCE, "Elixir lib dir: ~ts~n", [ElixirLibDir]),
+ ct:log(?LOW_IMPORTANCE, "Elixir lib dir: ~ts~n", [ElixirLibDir]),
true = code:add_pathz(ElixirLibDir),
{ok, _} = application:ensure_all_started(elixir),
Config
@@ -720,14 +720,18 @@ long_running_testsuite_monitor(TimerRef, Testcases) ->
long_running_testsuite_monitor(TimerRef, Testcases1);
ping_ct ->
T1 = erlang:monotonic_time(seconds),
- ct:pal(?STD_IMPORTANCE, "Testcases still in progress:~ts",
- [[
+ InProgress = [
begin
TDiff = format_time_diff(T1, T0),
rabbit_misc:format("~n - ~ts (~ts)", [TC, TDiff])
end
|| {TC, T0} <- Testcases
- ]]),
+ ],
+ case InProgress of
+ [] -> ok;
+ _ -> ct:pal(?STD_IMPORTANCE, "Testcases still in progress:~ts",
+ [InProgress])
+ end,
long_running_testsuite_monitor(TimerRef, Testcases);
stop ->
timer:cancel(TimerRef)
@@ -905,7 +909,7 @@ exec([Cmd | Args], Options) when is_list(Cmd) orelse is_binary(Cmd) ->
%% Because Args1 may contain binaries, we don't use string:join().
%% Instead we do a list comprehension.
ArgsIoList = [Cmd1, [[$\s, Arg] || Arg <- Args1]],
- ct:pal(?LOW_IMPORTANCE, Log1, [ArgsIoList, self()]),
+ ct:log(?LOW_IMPORTANCE, Log1, [ArgsIoList, self()]),
try
Port = erlang:open_port(
{spawn_executable, Cmd1}, [
@@ -951,10 +955,10 @@ port_receive_loop(Port, Stdout, Options, Until, DumpTimer) ->
Stdout =:= "",
if
DropStdout ->
- ct:pal(?LOW_IMPORTANCE, "Exit code: ~tp (pid ~tp)",
+ ct:log(?LOW_IMPORTANCE, "Exit code: ~tp (pid ~tp)",
[X, self()]);
true ->
- ct:pal(?LOW_IMPORTANCE, "~ts~nExit code: ~tp (pid ~tp)",
+ ct:log(?LOW_IMPORTANCE, "~ts~nExit code: ~tp (pid ~tp)",
[Stdout, X, self()])
end,
case proplists:get_value(match_stdout, Options) of
@@ -976,7 +980,7 @@ port_receive_loop(Port, Stdout, Options, Until, DumpTimer) ->
DropStdout ->
ok;
true ->
- ct:pal(?LOW_IMPORTANCE, "~ts~n[Command still in progress] (pid ~tp)",
+ ct:log(?LOW_IMPORTANCE, "~ts~n[Command still in progress] (pid ~tp)",
[Stdout, self()])
end,
port_receive_loop(Port, Stdout, Options, Until, stdout_dump_timer());
@@ -1101,7 +1105,7 @@ eventually({Line, Assertion} = TestObj, PollInterval, PollCount)
ok ->
ok;
Err ->
- ct:pal(?LOW_IMPORTANCE,
+ ct:log(?LOW_IMPORTANCE,
"Retrying in ~bms for ~b more times due to failed assertion in line ~b: ~tp",
[PollInterval, PollCount - 1, Line, Err]),
timer:sleep(PollInterval),
diff --git a/deps/rabbitmq_mqtt/Makefile b/deps/rabbitmq_mqtt/Makefile
index 48dcca6c934f..6a74a6a80c97 100644
--- a/deps/rabbitmq_mqtt/Makefile
+++ b/deps/rabbitmq_mqtt/Makefile
@@ -82,22 +82,16 @@ define ct_master.erl
peer:call(Pid2, persistent_term, put, [rabbit_ct_tcp_port_base, 25000]),
peer:call(Pid3, persistent_term, put, [rabbit_ct_tcp_port_base, 27000]),
peer:call(Pid4, persistent_term, put, [rabbit_ct_tcp_port_base, 29000]),
- ct_master_fork:run("$1"),
- Fail1 = peer:call(Pid1, cth_parallel_ct_detect_failure, has_failures, []),
- Fail2 = peer:call(Pid2, cth_parallel_ct_detect_failure, has_failures, []),
- Fail3 = peer:call(Pid3, cth_parallel_ct_detect_failure, has_failures, []),
- Fail4 = peer:call(Pid4, cth_parallel_ct_detect_failure, has_failures, []),
+ [{[_], {ok, Results}}] = ct_master_fork:run("$1"),
peer:stop(Pid4),
peer:stop(Pid3),
peer:stop(Pid2),
peer:stop(Pid1),
- if
- Fail1 -> halt(1);
- Fail2 -> halt(2);
- Fail3 -> halt(3);
- Fail4 -> halt(4);
- true -> halt(0)
- end
+ lists:foldl(fun
+ ({_, {_, 0, {_, 0}}}, Err) -> Err + 1;
+ (What, Peer) -> halt(Peer)
+ end, 1, Results),
+ halt(0)
endef
PARALLEL_CT_SET_1_A = auth retainer
@@ -116,6 +110,7 @@ define tpl_parallel_ct_test_spec
{logdir, "$(CT_LOGS_DIR)"}.
{logdir, master, "$(CT_LOGS_DIR)"}.
{create_priv_dir, all_nodes, auto_per_run}.
+{auto_compile, false}.
{node, shard1, 'rabbit_shard1@localhost'}.
{node, shard2, 'rabbit_shard2@localhost'}.
diff --git a/moduleindex.yaml b/moduleindex.yaml
index 969c58a7ace3..298a9a8b1413 100755
--- a/moduleindex.yaml
+++ b/moduleindex.yaml
@@ -868,9 +868,10 @@ rabbitmq_consistent_hash_exchange:
rabbitmq_ct_client_helpers:
- rabbit_ct_client_helpers
rabbitmq_ct_helpers:
+- ct_master_event_fork
- ct_master_fork
+- ct_master_logs_fork
- cth_log_redirect_any_domains
-- cth_parallel_ct_detect_failure
- rabbit_control_helper
- rabbit_ct_broker_helpers
- rabbit_ct_config_schema