Merge branch 'master' into spec-updates

.github/workflows/tests.yaml

@@ -546,6 +546,22 @@ jobs:
                   scripts/run_in_python_env.sh out/venv 'src/python_testing/execute_python_tests.py --env-file /tmp/test_env.yaml --search-directory src/python_testing'
                   scripts/run_in_python_env.sh out/venv 'scripts/tests/TestTimeSyncTrustedTimeSourceRunner.py --all-clusters out/linux-x64-all-clusters-ipv6only-no-ble-no-wifi-tsan-clang-test/chip-all-clusters-app'
 
+            - name: Execute Jupyter Notebooks
+              run: |
+                  scripts/run_in_build_env.sh './scripts/build_python.sh --jupyter-lab --install_virtual_env out/venv'
+                  ./scripts/run_in_build_env.sh \
+                   "./scripts/build/build_examples.py \
+                      --target linux-x64-all-clusters-ipv6only-no-ble-no-wifi-clang-test \
+                      build \
+                      --copy-artifacts-to objdir-clone \
+                   "
+                  scripts/run_in_python_env.sh out/venv \
+                  "jupyter execute docs/development_controllers/chip-repl/Matter_REPL_Intro.ipynb \
+                  docs/development_controllers/chip-repl/Matter_Basic_Interactions.ipynb \
+                  docs/development_controllers/chip-repl/Matter_Access_Control.ipynb \
+                  docs/development_controllers/chip-repl/Matter_Multi_Fabric_Commissioning.ipynb \
+                  "
+
             - name: Uploading core files
               uses: actions/upload-artifact@v4
               if: ${{ failure() && !env.ACT }}

docs/development_controllers/chip-repl/python_chip_controller_building.md

@@ -103,6 +103,8 @@ To build and run the Python CHIP controller:
     chip-repl
     ```
 
+NOTE: To get more verbose logs, pass the debug flag: `chip-repl --debug`
+
 <hr>
 
 ## Using Python CHIP Controller REPL for Matter accessory testing
@@ -133,8 +135,8 @@ An uncommissioned accessory device advertises over Bluetooth LE or via mDNS if
 already on the network. Run the following command to scan all advertised Matter
 devices:
 
-```
-devCtrl.DiscoverCommissionableNodes()
+```python
+await devCtrl.DiscoverCommissionableNodes()
 ```
 
 ### Step 4: Set network pairing credentials
@@ -173,7 +175,7 @@ network interface, such as Thread or Wi-Fi.
 2. Set the previously obtained Active Operational Dataset as a byte array using
    the following command:
 
-    ```
+    ```python
     thread_dataset = bytes.fromhex("0e080000000000010000000300001335060004001fffe002084fe76e9a8b5edaf50708fde46f999f0698e20510d47f5027a414ffeebaefa92285cc84fa030f4f70656e5468726561642d653439630102e49c0410b92f8c7fbb4f9f3e08492ee3915fbd2f0c0402a0fff8")
     devCtrl.SetThreadOperationalDataset(thread_dataset)
     ```
@@ -183,7 +185,7 @@ network interface, such as Thread or Wi-Fi.
 Assuming your Wi-Fi SSID is _TESTSSID_, and your Wi-Fi password is _P455W4RD_,
 set the credentials to the controller by executing the following command:
 
-```
+```python
 devCtrl.SetWiFiCredentials(<ssid>, <password>)
 ```
 
@@ -213,8 +215,8 @@ with the following assumptions for the Matter accessory device:
 -   The setup pin code of the device is _20202021_
 -   The temporary Node ID is _1234_
 
-```
-devCtrl.ConnectBLE(3840, 20202021, 1234)
+```python
+await devCtrl.ConnectBLE(3840, 20202021, 1234)
 ```
 
 You can skip the last parameter, the Node ID, in the command. If you skip it,
@@ -230,8 +232,8 @@ CHIP:SVR: SetupQRCode: [MT:-24J0AFN00KA0648G00]
 
 Use the following command to commission the device with the QR code:
 
-```
-devCtrl.CommissionWithCode("MT:-24J0AFN00KA0648G00", 1234)
+```python
+await devCtrl.CommissionWithCode("MT:-24J0AFN00KA0648G00", 1234)
 ```
 
 After connecting the device over Bluetooth LE, the controller will go through
@@ -262,14 +264,14 @@ the following stages:
 For the light bulb example, execute the following command to toggle the LED
 state:
 
-```
+```python
 await devCtrl.SendCommand(1234, 1, Clusters.OnOff.Commands.Toggle())
 ```
 
 To change the brightness of the LED, use the following command, with the level
 value somewhere between 0 and 255.
 
-```
+```python
 commandToSend = LevelControl.Commands.MoveToLevel(level=50, transitionTime=Null, optionsMask=0, optionsOverride=0)
 await devCtrl.SendCommand(1234, 1, commandToSend)
 ```
@@ -281,7 +283,7 @@ maintains collection of attributes that a controller can obtain from a device,
 such as the vendor name, the product name, or software version. Use
 `ReadAttribute()` command to read those values from the device:
 
-```
+```python
 attributes = [
     (0, Clusters.BasicInformation.Attributes.VendorName),
     (0, Clusters.BasicInformation.Attributes.ProductName),
@@ -309,7 +311,7 @@ the full list of available commands.
 Provides the controller with Thread network credentials that will be used in the
 device commissioning procedure to configure the device with a Thread interface.
 
-```
+```python
 thread_dataset = bytes.fromhex("0e080000000000010000000300001335060004001fffe002084fe76e9a8b5edaf50708fde46f999f0698e20510d47f5027a414ffeebaefa92285cc84fa030f4f70656e5468726561642d653439630102e49c0410b92f8c7fbb4f9f3e08492ee3915fbd2f0c0402a0fff8")
 devCtrl.SetThreadOperationalDataset(thread_dataset)
 ```
@@ -319,7 +321,7 @@ devCtrl.SetThreadOperationalDataset(thread_dataset)
 Provides the controller with Wi-Fi network credentials that will be used in the
 device commissioning procedure to configure the device with a Wi-Fi interface.
 
-```
+```python
 devCtrl.SetWiFiCredentials('TESTSSID', 'P455W4RD')
 ```
 
@@ -328,8 +330,8 @@ devCtrl.SetWiFiCredentials('TESTSSID', 'P455W4RD')
 Commission with the given nodeid from the setupPayload. setupPayload may be a QR
 or the manual setup code.
 
-```
-devCtrl.CommissionWithCode("MT:-24J0AFN00KA0648G00", 1234)
+```python
+await devCtrl.CommissionWithCode("MT:-24J0AFN00KA0648G00", 1234)
 ```
 
 ### `SendCommand(<nodeid>: int, <endpoint>: int, Clusters.<cluster>.Commands.<command>(<arguments>))`

docs/platforms/esp32/config_options.md

@@ -15,14 +15,12 @@ CONFIG_LWIP_IPV4=n
 ### Executable component not in "main" component
 
 The ESP-IDF framework allows renaming the main component, which can be useful if
-you want to place the app_main() function in a different component.
+you want to place the `app_main()` function in a different component.
 
 For required changes in the executable component, please refer to the
 [esp-idf documentation](https://docs.espressif.com/projects/esp-idf/en/stable/esp32/api-guides/build-system.html#renaming-main-component).
 
-If you're building applications that support Matter and want to place app_main()
-in a component other than main, use the following command:
-
-```
-idf.py -DEXECUTABLE_COMPONENT_NAME="your_component" build
-```
+You need to list the required components in `idf_component_register()`. If this
+module contains Matter related code, you may need to include
+`chip, app_update, spi_flash, and nvs_flash` as `PRIV_REQUIRES`, along with any
+other necessary dependencies.

integrations/docker/images/base/chip-build/version

@@ -1 +1 @@
-105 : Upgrade android docker with new kotlin/gradle
+106 : Upgrade android docker with java 17 and adjust the location for android cmdline tool

integrations/docker/images/stage-2/chip-build-java/Dockerfile

@@ -6,7 +6,7 @@ LABEL org.opencontainers.image.source https://github.com/project-chip/connectedh
 RUN set -x \
     && apt-get update \
     && DEBIAN_FRONTEND=noninteractive apt-get install -fy \
-    openjdk-11-jdk \
+    openjdk-17-jdk \
     && rm -rf /var/lib/apt/lists/ \
     && : # last line
 
@@ -20,4 +20,4 @@ RUN set -x \
     && : # last line
 
 ENV PATH $PATH:/usr/lib/kotlinc/bin
-ENV JAVA_PATH=/usr/lib/jvm/java-11-openjdk-amd64
+ENV JAVA_PATH=/usr/lib/jvm/java-17-openjdk-amd64

integrations/docker/images/stage-3/chip-build-android/Dockerfile

@@ -27,11 +27,16 @@ RUN set -x \
     && : # last line
 
 # Download and install android command line tool (for installing `sdkmanager`)
+# We need create latest folder inide cmdline-tools, since latest android commandline tool looks for this latest folder
+# when running sdkmanager --licenses
 RUN set -x \
     && wget -O /tmp/cmdline-tools.zip https://dl.google.com/android/repository/commandlinetools-linux-11076708_latest.zip \
     && cd /opt/android/sdk \
-    && unzip /tmp/cmdline-tools.zip \
-    && rm -f /tmp/cmdline-tools.zip \
+    && mkdir -p temp \
+    && unzip /tmp/cmdline-tools.zip -d temp \
+    && mkdir -p cmdline-tools/latest \
+    && cp -rf temp/cmdline-tools/* cmdline-tools/latest \
+    && rm -rf temp \
     && test -d /opt/android/sdk/cmdline-tools \
     && : # last line
 

scripts/jupyterlab_requirements.txt

@@ -4,14 +4,15 @@
 # jupyter-lab
 #
 # --------------------------------------
-# import chip.native
-# import pkgutil
-# module = pkgutil.get_loader('chip.ChipReplStartup')
-# %run {module.path}
+# %reset -f
+# import importlib.util
+# spec = importlib.util.find_spec('chip.ChipReplStartup')
+# %run {spec.origin}
 # --------------------------------------
 #
 
 jupyterlab
 ipykernel
 jupyterlab-lsp
 python-lsp-server
+jupyterlab-git

src/app/data-model-provider/MetadataList.cpp

@@ -82,10 +82,11 @@ CHIP_ERROR GenericAppendOnlyBuffer::EnsureAppendCapacity(size_t numElements)
 
     if (mBuffer == nullptr)
     {
-        mBuffer            = static_cast<uint8_t *>(Platform::MemoryCalloc(numElements, mElementSize));
+        mBuffer = static_cast<uint8_t *>(Platform::MemoryCalloc(numElements, mElementSize));
+        VerifyOrReturnError(mBuffer != nullptr, CHIP_ERROR_NO_MEMORY);
         mCapacity          = numElements;
         mBufferIsAllocated = true;
-        return mBuffer != nullptr ? CHIP_NO_ERROR : CHIP_ERROR_NO_MEMORY;
+        return CHIP_NO_ERROR;
     }
 
     // we already have the data in buffer. we have two choices:
@@ -100,9 +101,9 @@ CHIP_ERROR GenericAppendOnlyBuffer::EnsureAppendCapacity(size_t numElements)
     else
     {
         // this is NOT an allocated buffer, but it should become one
-        auto new_buffer    = static_cast<uint8_t *>(Platform::MemoryCalloc(mElementCount + numElements, mElementSize));
-        mBufferIsAllocated = true;
+        auto new_buffer = static_cast<uint8_t *>(Platform::MemoryCalloc(mElementCount + numElements, mElementSize));
         VerifyOrReturnError(new_buffer != nullptr, CHIP_ERROR_NO_MEMORY);
+        mBufferIsAllocated = true;
         memcpy(new_buffer, mBuffer, mElementCount * mElementSize);
         mBuffer = new_buffer;
     }
@@ -119,7 +120,7 @@ CHIP_ERROR GenericAppendOnlyBuffer::AppendSingleElementRaw(const void * buffer)
     return CHIP_NO_ERROR;
 }
 
-CHIP_ERROR GenericAppendOnlyBuffer::AppendElementArrayRaw(const void * buffer, size_t numElements)
+CHIP_ERROR GenericAppendOnlyBuffer::AppendElementArrayRaw(const void * __restrict__ buffer, size_t numElements)
 {
     ReturnErrorOnFailure(EnsureAppendCapacity(numElements));
 

src/app/data-model-provider/MetadataList.h

@@ -47,7 +47,7 @@ class GenericAppendOnlyBuffer
     /// Ensure that at least the specified number of elements
     /// can be appended to the internal buffer;
     ///
-    /// This will cause the internal buffer to become and allocated buffer
+    /// This will cause the internal buffer to become an allocated buffer
     CHIP_ERROR EnsureAppendCapacity(size_t numElements);
 
     bool IsEmpty() const { return mElementCount == 0; }
@@ -66,7 +66,10 @@ class GenericAppendOnlyBuffer
     ///
     /// This ALWAYS COPIES the elements internally.
     /// Additional capacity is AUTOMATICALLY ADDED.
-    CHIP_ERROR AppendElementArrayRaw(const void * buffer, size_t numElements);
+    ///
+    /// buffer MUST NOT point inside "own" buffer as mBuffer may be reallocated
+    /// as part of the appending.
+    CHIP_ERROR AppendElementArrayRaw(const void * __restrict__ buffer, size_t numElements);
 
     /// Appends a list of elements from a raw array.
     ///
@@ -93,7 +96,11 @@ class GenericAppendOnlyBuffer
 
 /// Represents a RAII instance owning a buffer.
 ///
-/// It auto-frees the owned buffer on destruction
+/// It auto-frees the owned buffer on destruction via `Platform::MemoryFree`.
+///
+/// This class is designed to be a storage class for `GenericAppendOnlyBuffer` and
+/// its subclasses (i.e. GenericAppendOnlyBuffer uses PlatformMemory and this class
+/// does the same. They MUST be kept in sync.)
 class ScopedBuffer
 {
 public:
@@ -165,6 +172,10 @@ class ListBuilder : public detail::GenericAppendOnlyBuffer
     ///
     /// Automatically attempts to allocate sufficient space to fulfill the element
     /// requirements.
+    ///
+    /// `span` MUST NOT point inside "own" buffer (and generally will not
+    /// as this class does not expose buffer access except by releasing ownership
+    /// via `Take`)
     CHIP_ERROR AppendElements(SpanType span) { return AppendElementArrayRaw(span.data(), span.size()); }
 
     /// Append a single element.

src/app/data-model-provider/ProviderMetadataTree.h

@@ -83,7 +83,11 @@ class ProviderMetadataTree
     virtual void Temporary_ReportAttributeChanged(const AttributePathParams & path) = 0;
 
     // "convenience" functions that just return the data and ignore the error
-    // This returns the builder as-is even after the error (e.g. not found would return empty data)
+    // This returns the `ListBuilder<..>::TakeBuffer` from their equivalent fuctions as-is,
+    // even after an error (e.g. not found would return empty data).
+    //
+    // Usage of these indicates no error handling (not even logging) and code should
+    // consider handling errors instead.
     ReadOnlyBuffer<EndpointEntry> EndpointsIgnoreError();
     ReadOnlyBuffer<ServerClusterEntry> ServerClustersIgnoreError(EndpointId endpointId);
     ReadOnlyBuffer<AttributeEntry> AttributesIgnoreError(const ConcreteClusterPath & path);

src/controller/java/AndroidLogDownloadFromNode.cpp

@@ -152,7 +152,7 @@ void AndroidLogDownloadFromNode::OnResponseRetrieveLogs(void * context,
     using namespace chip::app::Clusters::DiagnosticLogs;
     if (data.status == StatusEnum::kSuccess)
     {
-        ChipLogProgress(Controller, "Success. Will receive log from BDX protocol.")
+        ChipLogProgress(Controller, "Success. Will receive log from BDX protocol.");
     }
     else if (data.status == StatusEnum::kExhausted)
     {
@@ -210,8 +210,8 @@ void AndroidLogDownloadFromNode::FinishLogDownloadFromNode(void * context, CHIP_
     JniLocalReferenceScope scope(env);
 
     jobject jCallback   = self->mJavaCallback.ObjectRef();
-    jint jFabricIndex   = self->mController->GetFabricIndex();
-    jlong jremoteNodeId = self->mRemoteNodeId;
+    jint jFabricIndex   = static_cast<jint>(self->mController->GetFabricIndex());
+    jlong jremoteNodeId = static_cast<jlong>(self->mRemoteNodeId);
 
     VerifyOrExit(env != nullptr, ChipLogError(Controller, "Could not get JNIEnv for current thread"));
 
@@ -274,7 +274,8 @@ void AndroidLogDownloadFromNode::OnTransferCallback(FabricIndex fabricIndex, Nod
 
     if (ret != JNI_TRUE)
     {
-        ChipLogError(Controller, "Transfer will be rejected.") * errInfoOnFailure = CHIP_ERROR_INTERNAL;
+        ChipLogError(Controller, "Transfer will be rejected.");
+        *errInfoOnFailure = CHIP_ERROR_INTERNAL;
     }
 }
 

src/controller/java/CHIPP256KeypairBridge.cpp

@@ -108,14 +108,14 @@ CHIP_ERROR CHIPP256KeypairBridge::ECDSA_sign_msg(const uint8_t * msg, size_t msg
         return CHIP_ERROR_INCORRECT_STATE;
     }
 
-    VerifyOrReturnError(CanCastTo<uint32_t>(msg_length), CHIP_ERROR_INVALID_ARGUMENT);
+    VerifyOrReturnError(CanCastTo<jsize>(msg_length), CHIP_ERROR_INVALID_ARGUMENT);
 
     CHIP_ERROR err = CHIP_NO_ERROR;
     jbyteArray jniMsg;
     jobject signedResult = nullptr;
     JNIEnv * env         = JniReferences::GetInstance().GetEnvForCurrentThread();
     VerifyOrReturnError(env != nullptr, err = CHIP_JNI_ERROR_NO_ENV);
-    err = JniReferences::GetInstance().N2J_ByteArray(env, msg, static_cast<uint32_t>(msg_length), jniMsg);
+    err = JniReferences::GetInstance().N2J_ByteArray(env, msg, static_cast<jsize>(msg_length), jniMsg);
     VerifyOrReturnError(err == CHIP_NO_ERROR, err);
     VerifyOrReturnError(jniMsg != nullptr, err);
     VerifyOrReturnError(mDelegate.HasValidObjectRef(), CHIP_ERROR_INCORRECT_STATE);

src/controller/python/chip-repl.py

@@ -26,9 +26,9 @@
 def main():
     c = Config()
     c.InteractiveShellApp.exec_lines = [
-        "import pkgutil",
-        "module = pkgutil.get_loader('chip.ChipReplStartup')",
-        "%run {module.path} " + " ".join(sys.argv[1:])
+        "import importlib.util",
+        "spec = importlib.util.find_spec('chip.ChipReplStartup')",
+        "%run {spec.origin} " + " ".join(sys.argv[1:])
     ]
 
     sys.argv = [sys.argv[0]]

src/controller/python/chip/ChipReplStartup.py

@@ -6,6 +6,7 @@
 import pathlib
 
 import chip.CertificateAuthority
+import chip.clusters as Clusters  # noqa: F401
 import chip.FabricAdmin
 import chip.logging
 import chip.native
@@ -163,7 +164,7 @@ def main():
     console.print(
         '''\t[red]certificateAuthorityManager[blue]:\tManages a list of CertificateAuthority instances.
     \t[red]caList[blue]:\t\t\t\tThe list of CertificateAuthority instances.
-    \t[red]caList[n][m][blue]:\t\t\tA specific FabricAdmin object at index m for the nth CertificateAuthority instance.''')
+    \t[red]caList\[n].adminList\[m][blue]:\t\tA specific FabricAdmin object at index m for the nth CertificateAuthority instance.''')
 
     console.print(
         f'\n\n[blue]Default CHIP Device Controller (NodeId: {devCtrl.nodeId}): '

src/darwin/Framework/CHIP/MTRDeviceController.h

@@ -309,6 +309,13 @@ MTR_AVAILABLE(ios(16.1), macos(13.0), watchos(9.1), tvos(16.1))
  */
 - (void)resume MTR_AVAILABLE(ios(18.2), macos(15.2), watchos(11.2), tvos(18.2));
 
+/**
+ * Returns the list of node IDs for which this controller has stored
+ * information.  Returns empty list if the controller does not have any
+ * information stored.
+ */
+- (NSArray<NSNumber *> *)nodesWithStoredData MTR_AVAILABLE(ios(18.4), macos(15.4), watchos(11.4), tvos(18.4));
+
 /**
  * Shut down the controller. Calls to shutdown after the first one are NO-OPs.
  * This must be called, either directly or via shutting down the