[chip-tool] Add an option to continue running tests even after a test…

[woody-apple]

… failure

Problem

It may be sometimes useful to continue running tests even after a test failure. This is in order to know which tests are failing from the test suite, instead of only seeing the first error.

Change overview

• Add a continueOnFailure optional argument to the test runner

Testing

It was tested running the Test_TC_OO_1 test suite:

$ ./out/debug/standalone/chip-tool tests Test_TC_OO_1_1 --continueOnFailure 0

This one stops on the first error.

$ ./out/debug/standalone/chip-tool tests Test_TC_OO_1_1 --continueOnFailure 1

This one continues and accumulates errors if any.

[github-actions]

PR #20945: Size comparison from 56db839 to 5661d0e

Increases (2 builds for bl602, linux)

platform	target	config	section	56db839	5661d0e	change	% change
bl602	lighting-app	bl602+rpc	(read/write)	1425194	1425202	8	0.0
			.text	1082100	1082104	4	0.0
linux	chip-tool-ipv6only	arm64	(read only)	10037948	10038876	928	0.0
			.rodata	479820	480012	192	0.0
			.text	7977764	7978500	736	0.0

Decreases (1 build for telink)

platform	target	config	section	56db839	5661d0e	change	% change
telink	lighting-app	tlsr9518adk80d	text	583348	583346	-2	-0.0

Full report (32 builds for bl602, cc13x2_26x2, cyw30739, efr32, esp32, k32w, linux, mbed, nrfconnect, p6, telink)

platform	target	config	section	56db839	5661d0e	change	% change
bl602	lighting-app	bl602	(read/write)	1379762	1379762	0	0.0
			.bss	116962	116962	0	0.0
			.data	4480	4480	0	0.0
			.text	1050416	1050416	0	0.0
		bl602+rpc	(read/write)	1425194	1425202	8	0.0
			.bss	124410	124410	0	0.0
			.data	4600	4600	0	0.0
			.text	1082100	1082104	4	0.0
cc13x2_26x2	all-clusters-app	LP_CC2652R7	(read only)	668399	668399	0	0.0
			(read/write)	182840	182840	0	0.0
			.bss	74132	74132	0	0.0
			.data	3356	3356	0	0.0
			.rodata	88319	88319	0	0.0
			.text	579764	579764	0	0.0
	all-clusters-minimal-app	LP_CC2652R7	(read only)	634015	634015	0	0.0
			(read/write)	157700	157700	0	0.0
			.bss	73428	73428	0	0.0
			.data	3356	3356	0	0.0
			.rodata	77551	77551	0	0.0
			.text	556140	556140	0	0.0
	lock-ftd	LP_CC2652R7	(read only)	671595	671595	0	0.0
			(read/write)	169820	169820	0	0.0
			.bss	71196	71196	0	0.0
			.data	3280	3280	0	0.0
			.rodata	76451	76451	0	0.0
			.text	594664	594664	0	0.0
	lock-mtd	LP_CC2652R7	(read only)	653887	653887	0	0.0
			(read/write)	183216	183216	0	0.0
			.bss	66884	66884	0	0.0
			.data	3280	3280	0	0.0
			.rodata	101199	101199	0	0.0
			.text	552208	552208	0	0.0
	pump-app	LP_CC2652R7	(read only)	680587	680587	0	0.0
			(read/write)	161676	161676	0	0.0
			.bss	71276	71276	0	0.0
			.data	3280	3280	0	0.0
			.rodata	88811	88811	0	0.0
			.text	591292	591292	0	0.0
	pump-controller-app	LP_CC2652R7	(read only)	666379	666379	0	0.0
			(read/write)	176004	176004	0	0.0
			.bss	71396	71396	0	0.0
			.data	3276	3276	0	0.0
			.rodata	84675	84675	0	0.0
			.text	581224	581224	0	0.0
	shell	LP_CC2652R7	(read only)	660890	660890	0	0.0
			(read/write)	185852	185852	0	0.0
			.bss	76436	76436	0	0.0
			.data	3360	3360	0	0.0
			.rodata	85098	85098	0	0.0
			.text	575476	575476	0	0.0
cyw30739	light	cyw930739m2evb_01	(read/write)	581926	581926	0	0.0
			.app_xip_area	460632	460632	0	0.0
			.bss	64232	64232	0	0.0
			.data	716	716	0	0.0
			.rodata	0	0	0	0.0
			.text	112	112	0	0.0
	lock	cyw930739m2evb_01	(read/write)	587842	587842	0	0.0
			.app_xip_area	461820	461820	0	0.0
			.bss	68960	68960	0	0.0
			.data	720	720	0	0.0
			.rodata	0	0	0	0.0
			.text	112	112	0	0.0
	ota-requestor-no-progress-logging	cyw930739m2evb_01	(read/write)	585266	585266	0	0.0
			.app_xip_area	464820	464820	0	0.0
			.bss	63440	63440	0	0.0
			.data	660	660	0	0.0
			.rodata	0	0	0	0.0
			.text	112	112	0	0.0
efr32	lighting-app	BRD4161A	(read/write)	1084792	1084792	0	0.0
			.bss	133044	133044	0	0.0
			.data	2048	2048	0	0.0
			.text	949680	949680	0	0.0
		BRD4161A+rpc	(read/write)	1139132	1139132	0	0.0
			.bss	149724	149724	0	0.0
			.data	2260	2260	0	0.0
			.text	987124	987124	0	0.0
		BRD4161A+rs911x	(read/write)	950160	950160	0	0.0
			.bss	140768	140768	0	0.0
			.data	2048	2048	0	0.0
			.text	807324	807324	0	0.0
	lock-app	BRD4161A+wf200	(read/write)	1128004	1128004	0	0.0
			.bss	144136	144136	0	0.0
			.data	2056	2056	0	0.0
			.text	981792	981792	0	0.0
	window-app	BRD4161A	(read/write)	1078284	1078284	0	0.0
			.bss	134516	134516	0	0.0
			.data	2076	2076	0	0.0
			.text	941672	941672	0	0.0
esp32	all-clusters-app	c3devkit	(read only)	1021764	1021764	0	0.0
			(read/write)	1486026	1486026	0	0.0
			.dram0.bss	70080	70080	0	0.0
			.dram0.data	14600	14600	0	0.0
			.flash.rodata	215912	215912	0	0.0
			.flash.text	1021764	1021764	0	0.0
			.iram0.text	62902	62902	0	0.0
		m5stack	(read only)	1075507	1075507	0	0.0
			(read/write)	488064	488064	0	0.0
			.dram0.bss	75600	75600	0	0.0
			.dram0.data	34144	34144	0	0.0
			.flash.rodata	246324	246324	0	0.0
			.flash.text	1070123	1070123	0	0.0
			.iram0.text	123267	123267	0	0.0
k32w	light	k32w061+release	(read/write)	660700	660700	0	0.0
			.bss	69540	69540	0	0.0
			.data	1992	1992	0	0.0
			.text	583368	583368	0	0.0
	lock	k32w061+release	(read/write)	687684	687684	0	0.0
			.bss	70004	70004	0	0.0
			.data	2004	2004	0	0.0
			.text	609876	609876	0	0.0
linux	chip-tool-ipv6only	arm64	(read only)	10037948	10038876	928	0.0
			(read/write)	694801	694801	0	0.0
			.bss	42993	42993	0	0.0
			.data	3304	3304	0	0.0
			.data.rel.ro	630920	630920	0	0.0
			.dynamic	528	528	0	0.0
			.got	13656	13656	0	0.0
			.init	24	24	0	0.0
			.init_array	192	192	0	0.0
			.rodata	479820	480012	192	0.0
			.text	7977764	7978500	736	0.0
	thermostat-no-ble	arm64	(read only)	2602988	2602988	0	0.0
			(read/write)	158481	158481	0	0.0
			.bss	65281	65281	0	0.0
			.data	1704	1704	0	0.0
			.data.rel.ro	83368	83368	0	0.0
			.dynamic	528	528	0	0.0
			.got	5104	5104	0	0.0
			.init	24	24	0	0.0
			.init_array	400	400	0	0.0
			.rodata	165748	165748	0	0.0
			.text	2196944	2196944	0	0.0
mbed	lock-app	CY8CPROTO_062_4343W+release	(read only)	6224	6224	0	0.0
			(read/write)	2449192	2449192	0	0.0
			.bss	213940	213940	0	0.0
			.data	5872	5872	0	0.0
			.text	1411836	1411836	0	0.0
nrfconnect	all-clusters-app	nrf52840dk_nrf52840	(read/write)	1176643	1176643	0	0.0
			bss	142900	142900	0	0.0
			rodata	142204	142204	0	0.0
			text	812636	812636	0	0.0
	all-clusters-minimal-app	nrf52840dk_nrf52840	(read/write)	1156675	1156675	0	0.0
			bss	142136	142136	0	0.0
			rodata	133748	133748	0	0.0
			text	801920	801920	0	0.0
p6	all-clusters-app	default	(read only)	882136	882136	0	0.0
			(read/write)	1685036	1685036	0	0.0
			.bss	148560	148560	0	0.0
			.data	2648	2648	0	0.0
			.text	1525440	1525440	0	0.0
	all-clusters-minimal-app	default	(read only)	882856	882856	0	0.0
			(read/write)	1629180	1629180	0	0.0
			.bss	147840	147840	0	0.0
			.data	2648	2648	0	0.0
			.text	1470304	1470304	0	0.0
	light-app	default	(read only)	891160	891160	0	0.0
			(read/write)	1549220	1549220	0	0.0
			.bss	139744	139744	0	0.0
			.data	2440	2440	0	0.0
			.text	1398648	1398648	0	0.0
	lock-app	default	(read only)	891288	891288	0	0.0
			(read/write)	1576220	1576220	0	0.0
			.bss	139600	139600	0	0.0
			.data	2456	2456	0	0.0
			.text	1425776	1425776	0	0.0
telink	light-switch-app	tlsr9518adk80d	(read/write)	798700	798700	0	0.0
			bss	70576	70576	0	0.0
			noinit	40416	40416	0	0.0
			text	566812	566812	0	0.0
	lighting-app	tlsr9518adk80d	(read/write)	818740	818740	0	0.0
			bss	71420	71420	0	0.0
			noinit	40416	40416	0	0.0
			text	583348	583346	-2	-0.0