diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py
index b638aa70905..469bc710337 100644
--- a/xarray/tests/test_backends.py
+++ b/xarray/tests/test_backends.py
@@ -2275,13 +2275,10 @@ def create_zarr_target(self):
         raise NotImplementedError
 
     @contextlib.contextmanager
-    def create_store(self, cache_members: bool = True):
+    def create_store(self):
         with self.create_zarr_target() as store_target:
             yield backends.ZarrStore.open_group(
-                store_target,
-                mode="w",
-                cache_members=cache_members,
-                **self.version_kwargs,
+                store_target, mode="w", **self.version_kwargs
             )
 
     def save(self, dataset, store_target, **kwargs):  # type: ignore[override]
@@ -2575,7 +2572,7 @@ def test_hidden_zarr_keys(self) -> None:
         skip_if_zarr_format_3("This test is unnecessary; no hidden Zarr keys")
 
         expected = create_test_data()
-        with self.create_store(cache_members=False) as store:
+        with self.create_store() as store:
             expected.dump_to_store(store)
             zarr_group = store.ds
 
@@ -2597,7 +2594,6 @@ def test_hidden_zarr_keys(self) -> None:
 
             # put it back and try removing from a variable
             del zarr_group["var2"].attrs[self.DIMENSION_KEY]
-
             with pytest.raises(KeyError):
                 with xr.decode_cf(store):
                     pass
@@ -3262,18 +3258,19 @@ def test_chunked_cftime_datetime(self) -> None:
                 assert original[name].chunks == actual_var.chunks
             assert original.chunks == actual.chunks
 
-    def test_cache_members(self) -> None:
+    @pytest.mark.parametrize("cache_array_keys", [True, False])
+    def test_get_array_keys(self, cache_array_keys: bool) -> None:
         """
-        Ensure that if `ZarrStore` is created with `cache_members` set to `True`,
-        a `ZarrStore` only inspects the underlying zarr group once,
-        and that the results of that inspection are cached.
+        Ensure that if `ZarrStore` is created with `cache_array_keys` set to `True`,
+        a `ZarrStore.get_array_keys` only invokes the `array_keys` function on the
+        `ZarrStore.zarr_group` instance once, and that the results of that call are cached.
 
-        Otherwise, `ZarrStore.members` should inspect the underlying zarr group each time it is
-        invoked
+        Otherwise, `ZarrStore.get_array_keys` instance should invoke the `array_keys`
+        each time it is called.
         """
         with self.create_zarr_target() as store_target:
-            zstore_mut = backends.ZarrStore.open_group(
-                store_target, mode="w", cache_members=False
+            zstore = backends.ZarrStore.open_group(
+                store_target, mode="w", cache_array_keys=cache_array_keys
             )
 
             # ensure that the keys are sorted
@@ -3281,24 +3278,20 @@ def test_cache_members(self) -> None:
 
             # create some arrays
             for ak in array_keys:
-                zstore_mut.zarr_group.create(name=ak, shape=(1,), dtype="uint8")
-
-            zstore_stat = backends.ZarrStore.open_group(
-                store_target, mode="r", cache_members=True
-            )
+                zstore.zarr_group.create(name=ak, shape=(1,), dtype="uint8")
 
-            observed_keys_0 = sorted(zstore_stat.array_keys())
+            observed_keys_0 = sorted(zstore.get_array_keys())
             assert observed_keys_0 == array_keys
 
             # create a new array
             new_key = "baz"
-            zstore_mut.zarr_group.create(name=new_key, shape=(1,), dtype="uint8")
-
-            observed_keys_1 = sorted(zstore_stat.array_keys())
-            assert observed_keys_1 == array_keys
+            zstore.zarr_group.create(name=new_key, shape=(1,), dtype="uint8")
+            observed_keys_1 = sorted(zstore.get_array_keys())
 
-            observed_keys_2 = sorted(zstore_mut.array_keys())
-            assert observed_keys_2 == sorted(array_keys + [new_key])
+            if cache_array_keys:
+                assert observed_keys_1 == array_keys
+            else:
+                assert observed_keys_1 == sorted(array_keys + [new_key])
 
 
 @requires_zarr
@@ -3378,11 +3371,11 @@ def test_append(self) -> None:
                 }
             else:
                 expected = {
-                    "iter": 2,
+                    "iter": 3,
                     "contains": 18,
                     "setitem": 10,
                     "getitem": 13,
-                    "listdir": 1,
+                    "listdir": 2,
                     "list_prefix": 2,
                 }
 
@@ -3480,11 +3473,11 @@ def test_region_write(self) -> None:
                 }
             else:
                 expected = {
-                    "iter": 1,
+                    "iter": 2,
                     "contains": 4,
                     "setitem": 1,
-                    "getitem": 5,
-                    "listdir": 1,
+                    "getitem": 4,
+                    "listdir": 2,
                     "list_prefix": 0,
                 }
 
@@ -3563,11 +3556,9 @@ def create_zarr_target(self):
             yield tmp
 
     @contextlib.contextmanager
-    def create_store(self, cache_members: bool = True):
+    def create_store(self):
         with self.create_zarr_target() as store_target:
-            group = backends.ZarrStore.open_group(
-                store_target, mode="a", cache_members=cache_members
-            )
+            group = backends.ZarrStore.open_group(store_target, mode="a")
             yield group