Issue 18505 - Darwin: synchronous API part 1: basic structure and serial read

[woody-apple]

Problem

We are looking to add a more natural synchronous API for UI development, as the preferred interface for the Darwin Framework.

Given the titular issue #18505 requires a solution that touches a lot of the files the above, we decided to put that in as part of this effort. This draft patch is the first part that includes the serial work queue needed to support an underlying async/callback model.

This is a work in progress.

Change overview

• New MTRAsyncCallbackWorkQueue
• New MTRDevice shim layer
  • with synchronous read API
  • read cache
  • "expected value" semantics for a form of write cache
• New zap-generated MTRClusters
  • Internal cppCluster object is initialized asynchronously through the MTRDevice shim
  • Commands converted to wait for and use the cppCluster object when ready

Note this is a draft patch with overall structure but lot of logic missing.

Testing

• Unit test was added for the work queue - more needs to be written for coverage.
• Additional testing TBD

[github-actions]

PR #21566: Size comparison from 2fced51 to 0cb899f

Increases (2 builds for telink)

platform	target	config	section	2fced51	0cb899f	change	% change
telink	light-switch-app	tlsr9518adk80d	text	567956	567960	4	0.0
	lighting-app	tlsr9518adk80d	text	584818	584820	2	0.0

Decreases (1 build for esp32)

platform	target	config	section	2fced51	0cb899f	change	% change
esp32	all-clusters-app	c3devkit	(read only)	1024430	1024428	-2	-0.0
			.flash.text	1024430	1024428	-2	-0.0

Full report (43 builds for bl602, cc13x2_26x2, cyw30739, efr32, esp32, k32w, linux, mbed, nrfconnect, p6, telink)

platform	target	config	section	2fced51	0cb899f	change	% change
bl602	lighting-app	bl602	(read/write)	1382282	1382282	0	0.0
			.bss	117626	117626	0	0.0
			.data	4480	4480	0	0.0
			.text	1052396	1052396	0	0.0
		bl602+rpc	(read/write)	1427714	1427714	0	0.0
			.bss	125066	125066	0	0.0
			.data	4600	4600	0	0.0
			.text	1084092	1084092	0	0.0
cc13x2_26x2	all-clusters-app	LP_CC2652R7	(read only)	669723	669723	0	0.0
			(read/write)	181660	181660	0	0.0
			.bss	74276	74276	0	0.0
			.data	3372	3372	0	0.0
			.rodata	88467	88467	0	0.0
			.text	580940	580940	0	0.0
	all-clusters-minimal-app	LP_CC2652R7	(read only)	635235	635235	0	0.0
			(read/write)	157844	157844	0	0.0
			.bss	73556	73556	0	0.0
			.data	3372	3372	0	0.0
			.rodata	77691	77691	0	0.0
			.text	557220	557220	0	0.0
	lock-ftd	LP_CC2652R7	(read only)	672575	672575	0	0.0
			(read/write)	168992	168992	0	0.0
			.bss	71348	71348	0	0.0
			.data	3296	3296	0	0.0
			.rodata	76535	76535	0	0.0
			.text	595560	595560	0	0.0
	lock-mtd	LP_CC2652R7	(read only)	655139	655139	0	0.0
			(read/write)	182116	182116	0	0.0
			.bss	67036	67036	0	0.0
			.data	3296	3296	0	0.0
			.rodata	101531	101531	0	0.0
			.text	553128	553128	0	0.0
	pump-app	LP_CC2652R7	(read only)	681071	681071	0	0.0
			(read/write)	161304	161304	0	0.0
			.bss	71388	71388	0	0.0
			.data	3296	3296	0	0.0
			.rodata	88919	88919	0	0.0
			.text	591668	591668	0	0.0
	pump-controller-app	LP_CC2652R7	(read only)	666647	666647	0	0.0
			(read/write)	175864	175864	0	0.0
			.bss	71524	71524	0	0.0
			.data	3292	3292	0	0.0
			.rodata	84743	84743	0	0.0
			.text	581424	581424	0	0.0
	shell	LP_CC2652R7	(read only)	662414	662414	0	0.0
			(read/write)	184488	184488	0	0.0
			.bss	76596	76596	0	0.0
			.data	3376	3376	0	0.0
			.rodata	85446	85446	0	0.0
			.text	576652	576652	0	0.0
cyw30739	light	cyw930739m2evb_01	(read/write)	583750	583750	0	0.0
			.app_xip_area	460704	460704	0	0.0
			.bss	65632	65632	0	0.0
			.data	744	744	0	0.0
			.rodata	0	0	0	0.0
			.text	112	112	0	0.0
	lock	cyw930739m2evb_01	(read/write)	590230	590230	0	0.0
			.app_xip_area	462400	462400	0	0.0
			.bss	70416	70416	0	0.0
			.data	748	748	0	0.0
			.rodata	0	0	0	0.0
			.text	112	112	0	0.0
	ota-requestor-no-progress-logging	cyw930739m2evb_01	(read/write)	589466	589466	0	0.0
			.app_xip_area	467228	467228	0	0.0
			.bss	64880	64880	0	0.0
			.data	688	688	0	0.0
			.rodata	0	0	0	0.0
			.text	112	112	0	0.0
efr32	lighting-app	BRD4161A	(read/write)	1088992	1088992	0	0.0
			.bss	133268	133268	0	0.0
			.data	2064	2064	0	0.0
			.text	953640	953640	0	0.0
		BRD4161A+rpc	(read/write)	1143348	1143348	0	0.0
			.bss	149948	149948	0	0.0
			.data	2276	2276	0	0.0
			.text	991104	991104	0	0.0
		BRD4161A+rs911x	(read/write)	974288	974288	0	0.0
			.bss	161744	161744	0	0.0
			.data	2048	2048	0	0.0
			.text	810476	810476	0	0.0
	lock-app	BRD4161A+wf200	(read/write)	1130112	1130112	0	0.0
			.bss	144400	144400	0	0.0
			.data	2056	2056	0	0.0
			.text	983636	983636	0	0.0
	window-app	BRD4161A	(read/write)	1082284	1082284	0	0.0
			.bss	134748	134748	0	0.0
			.data	2092	2092	0	0.0
			.text	945420	945420	0	0.0
esp32	all-clusters-app	c3devkit	(read only)	1024430	1024428	-2	-0.0
			(read/write)	1487042	1487042	0	0.0
			.dram0.bss	70336	70336	0	0.0
			.dram0.data	14600	14600	0	0.0
			.flash.rodata	216672	216672	0	0.0
			.flash.text	1024430	1024428	-2	-0.0
			.iram0.text	62902	62902	0	0.0
		m5stack	(read only)	1077871	1077871	0	0.0
			(read/write)	489064	489064	0	0.0
			.dram0.bss	75840	75840	0	0.0
			.dram0.data	34144	34144	0	0.0
			.flash.rodata	247084	247084	0	0.0
			.flash.text	1072487	1072487	0	0.0
			.iram0.text	123267	123267	0	0.0
k32w	light	k32w0+release	(read/write)	643368	643368	0	0.0
			.bss	69720	69720	0	0.0
			.data	2044	2044	0	0.0
			.text	568876	568876	0	0.0
	lock	k32w0+release	(read/write)	701512	701512	0	0.0
			.bss	70184	70184	0	0.0
			.data	2052	2052	0	0.0
			.text	626548	626548	0	0.0
linux	all-clusters-app	debug	(read only)	3013617	3013617	0	0.0
			(read/write)	155720	155720	0	0.0
			.bss	61888	61888	0	0.0
			.data	2096	2096	0	0.0
			.data.rel.ro	85384	85384	0	0.0
			.dynamic	608	608	0	0.0
			.got	4568	4568	0	0.0
			.init	27	27	0	0.0
			.init_array	1152	1152	0	0.0
			.rodata	271659	271659	0	0.0
			.text	2563074	2563074	0	0.0
	all-clusters-minimal-app	debug	(read only)	2855385	2855385	0	0.0
			(read/write)	147488	147488	0	0.0
			.bss	61152	61152	0	0.0
			.data	2064	2064	0	0.0
			.data.rel.ro	77992	77992	0	0.0
			.dynamic	608	608	0	0.0
			.got	4488	4488	0	0.0
			.init	27	27	0	0.0
			.init_array	1144	1144	0	0.0
			.rodata	271659	271659	0	0.0
			.text	2407474	2407474	0	0.0
	bridge-app	debug+rpc	(read only)	2360793	2360793	0	0.0
			(read/write)	127584	127584	0	0.0
			.bss	50592	50592	0	0.0
			.data	3600	3600	0	0.0
			.data.rel.ro	67544	67544	0	0.0
			.dynamic	608	608	0	0.0
			.got	4392	4392	0	0.0
			.init	27	27	0	0.0
			.init_array	824	824	0	0.0
			.rodata	202504	202504	0	0.0
			.text	1995282	1995282	0	0.0
	chip-tool	debug	(read only)	10505745	10505745	0	0.0
			(read/write)	633120	633120	0	0.0
			.bss	24856	24856	0	0.0
			.data	3266	3266	0	0.0
			.data.rel.ro	598496	598496	0	0.0
			.dynamic	608	608	0	0.0
			.got	5088	5088	0	0.0
			.init	27	27	0	0.0
			.init_array	768	768	0	0.0
			.rodata	538581	538581	0	0.0
			.text	8510788	8510788	0	0.0
	chip-tool-ipv6only	arm64	(read only)	9909564	9909564	0	0.0
			(read/write)	680721	680721	0	0.0
			.bss	32897	32897	0	0.0
			.data	3272	3272	0	0.0
			.data.rel.ro	626032	626032	0	0.0
			.dynamic	560	560	0	0.0
			.got	13568	13568	0	0.0
			.init	24	24	0	0.0
			.init_array	192	192	0	0.0
			.rodata	470364	470364	0	0.0
			.text	7852148	7852148	0	0.0
	lighting-app	debug+rpc	(read only)	2583121	2583121	0	0.0
			(read/write)	130232	130232	0	0.0
			.bss	49728	49728	0	0.0
			.data	2096	2096	0	0.0
			.data.rel.ro	72456	72456	0	0.0
			.dynamic	608	608	0	0.0
			.got	4392	4392	0	0.0
			.init	27	27	0	0.0
			.init_array	912	912	0	0.0
			.rodata	218096	218096	0	0.0
			.text	2193954	2193954	0	0.0
	lock-app	debug	(read only)	2571929	2571929	0	0.0
			(read/write)	125568	125568	0	0.0
			.bss	48256	48256	0	0.0
			.data	1712	1712	0	0.0
			.data.rel.ro	69592	69592	0	0.0
			.dynamic	608	608	0	0.0
			.got	4464	4464	0	0.0
			.init	27	27	0	0.0
			.init_array	888	888	0	0.0
			.rodata	235696	235696	0	0.0
			.text	2169218	2169218	0	0.0
	ota-provider-app	debug	(read only)	2351825	2351825	0	0.0
			(read/write)	119048	119048	0	0.0
			.bss	47808	47808	0	0.0
			.data	1936	1936	0	0.0
			.data.rel.ro	63416	63416	0	0.0
			.dynamic	608	608	0	0.0
			.got	4488	4488	0	0.0
			.init	27	27	0	0.0
			.init_array	768	768	0	0.0
			.rodata	208664	208664	0	0.0
			.text	1979282	1979282	0	0.0
	ota-requestor-app	debug	(read only)	2472401	2472401	0	0.0
			(read/write)	126440	126440	0	0.0
			.bss	50176	50176	0	0.0
			.data	2240	2240	0	0.0
			.data.rel.ro	68072	68072	0	0.0
			.dynamic	608	608	0	0.0
			.got	4480	4480	0	0.0
			.init	27	27	0	0.0
			.init_array	832	832	0	0.0
			.rodata	212192	212192	0	0.0
			.text	2087714	2087714	0	0.0
	shell	debug	(read only)	2584841	2584841	0	0.0
			(read/write)	141872	141872	0	0.0
			.bss	57800	57800	0	0.0
			.data	1264	1264	0	0.0
			.data.rel.ro	77008	77008	0	0.0
			.dynamic	608	608	0	0.0
			.got	4136	4136	0	0.0
			.init	27	27	0	0.0
			.init_array	1024	1024	0	0.0
			.rodata	232658	232658	0	0.0
			.text	2193954	2193954	0	0.0
	thermostat-no-ble	arm64	(read only)	2349548	2349548	0	0.0
			(read/write)	141761	141761	0	0.0
			.bss	55329	55329	0	0.0
			.data	1672	1672	0	0.0
			.data.rel.ro	75976	75976	0	0.0
			.dynamic	560	560	0	0.0
			.got	5016	5016	0	0.0
			.init	24	24	0	0.0
			.init_array	408	408	0	0.0
			.rodata	139316	139316	0	0.0
			.text	1972720	1972720	0	0.0
	tv-app	debug	(read only)	3136753	3136753	0	0.0
			(read/write)	257512	257512	0	0.0
			.bss	167288	167288	0	0.0
			.data	4736	4736	0	0.0
			.data.rel.ro	78960	78960	0	0.0
			.dynamic	608	608	0	0.0
			.got	4848	4848	0	0.0
			.init	27	27	0	0.0
			.init_array	1064	1064	0	0.0
			.rodata	255496	255496	0	0.0
			.text	2692018	2692018	0	0.0
	tv-casting-app	debug	(read only)	5391665	5391665	0	0.0
			(read/write)	158728	158728	0	0.0
			.bss	51384	51384	0	0.0
			.data	2432	2432	0	0.0
			.data.rel.ro	98504	98504	0	0.0
			.dynamic	608	608	0	0.0
			.got	4736	4736	0	0.0
			.init	27	27	0	0.0
			.init_array	1032	1032	0	0.0
			.rodata	339569	339569	0	0.0
			.text	4784802	4784802	0	0.0
mbed	lock-app	CY8CPROTO_062_4343W+release	(read only)	6224	6224	0	0.0
			(read/write)	2453224	2453224	0	0.0
			.bss	214548	214548	0	0.0
			.data	5872	5872	0	0.0
			.text	1415868	1415868	0	0.0
nrfconnect	all-clusters-app	nrf52840dk_nrf52840	(read/write)	1180935	1180935	0	0.0
			bss	143224	143224	0	0.0
			rodata	143344	143344	0	0.0
			text	815520	815520	0	0.0
	all-clusters-minimal-app	nrf52840dk_nrf52840	(read/write)	1160827	1160827	0	0.0
			bss	142460	142460	0	0.0
			rodata	134876	134876	0	0.0
			text	804668	804668	0	0.0
p6	all-clusters-app	default	(read only)	881528	881528	0	0.0
			(read/write)	1689300	1689300	0	0.0
			.bss	149168	149168	0	0.0
			.data	2648	2648	0	0.0
			.text	1529096	1529096	0	0.0
	all-clusters-minimal-app	default	(read only)	882248	882248	0	0.0
			(read/write)	1633332	1633332	0	0.0
			.bss	148448	148448	0	0.0
			.data	2648	2648	0	0.0
			.text	1473848	1473848	0	0.0
	light-app	default	(read only)	890568	890568	0	0.0
			(read/write)	1553380	1553380	0	0.0
			.bss	140336	140336	0	0.0
			.data	2440	2440	0	0.0
			.text	1402216	1402216	0	0.0
	lock-app	default	(read only)	886072	886072	0	0.0
			(read/write)	1592020	1592020	0	0.0
			.bss	144816	144816	0	0.0
			.data	2456	2456	0	0.0
			.text	1436360	1436360	0	0.0
telink	light-switch-app	tlsr9518adk80d	(read/write)	800528	800528	0	0.0
			bss	70808	70808	0	0.0
			noinit	40416	40416	0	0.0
			text	567956	567960	4	0.0
	lighting-app	tlsr9518adk80d	(read/write)	820944	820944	0	0.0
			bss	71652	71652	0	0.0
			noinit	40416	40416	0	0.0
			text	584818	584820	2	0.0