RPC: Add tokenized log support to console (#16537)

- Update Pigwee to 67506a1a448 to bring in console fix.
- Hide Python Repl when opened in raw mode.
- Add tokenized log support to console.
- Cleanup py setup so install is cleaner, and also add
  chip-console as an entrypoint to make it cleaner to run.

examples/all-clusters-app/ameba/README.md

@@ -127,7 +127,7 @@ to be On or Off.
 
 -   Launch the chip-rpc console after resetting Ameba board
 
-            $ python3 -m chip_rpc.console --device /dev/tty<port connected to USB-TTL adapter> -b 115200
+            $ chip-console --device /dev/tty<port connected to USB-TTL adapter> -b 115200
 
 -   Get and Set lighting directly using the RPC console
 

examples/all-clusters-app/esp32/README.md

@@ -315,7 +315,7 @@ Build or install the [rpc console](../../common/pigweed/rpc_console/README.md)
 
 Start the console
 
-    python -m chip_rpc.console --device /dev/ttyUSB0
+    chip-console --device /dev/ttyUSB0
 
 From within the console you can then invoke rpcs:
 

examples/build_overrides/pigweed_environment.gni

@@ -0,0 +1 @@
+../../build_overrides/pigweed_environment.gni

examples/common/pigweed/rpc_console/README.md

@@ -39,7 +39,7 @@ files required for CHIP.
 
 To start the console provide the path to the device, for example:
 
-    $ python -m chip_rpc.console --device /dev/ttyUSB0
+    $ chip-console --device /dev/ttyUSB0
 
 An example RPC command:
 

examples/common/pigweed/rpc_console/py/BUILD.gn

@@ -20,16 +20,13 @@ import("$dir_pw_build/python_dist.gni")
 import("${chip_root}/examples/common/pigweed/pigweed_rpcs.gni")
 
 pw_python_package("chip_rpc") {
-  generate_setup = {
-    metadata = {
-      name = "chip_rpc"
-      version = "0.0.1"
-    }
-    options = {
-      install_requires = [ "ipython" ]
-    }
-  }
+  setup = [
+    "pyproject.toml",
+    "setup.cfg",
+    "setup.py",
+  ]
   sources = [
+    "chip_rpc/__init__.py",
     "chip_rpc/console.py",
     "chip_rpc/plugins/device_toolbar.py",
     "chip_rpc/plugins/helper_scripts.py",

examples/common/pigweed/rpc_console/py/chip_rpc/console.py

@@ -30,7 +30,6 @@
     rpcs   - used to invoke RPCs
     device - the serial device used for communication
     client - the pw_rpc.Client
-    scripts - helper scripts for working with chip devices
     protos - protocol buffer messages indexed by proto package
 
 An example RPC command:
@@ -47,7 +46,7 @@
 from concurrent.futures import ThreadPoolExecutor
 import sys
 import threading
-from typing import Any, BinaryIO
+from typing import Any, BinaryIO, Collection
 
 from chip_rpc.plugins.device_toolbar import DeviceToolbar
 from chip_rpc.plugins.helper_scripts import HelperScripts
@@ -59,6 +58,10 @@
 from pw_rpc import callback_client
 from pw_rpc.console_tools.console import ClientInfo, flattened_rpc_completions
 
+from pw_tokenizer.database import LoadTokenDatabases
+from pw_tokenizer.detokenize import Detokenizer, detokenize_base64
+from pw_tokenizer import tokens
+
 # Protos
 from attributes_service import attributes_service_pb2
 from button_service import button_service_pb2
@@ -112,6 +115,11 @@ def _parse_args():
         '--raw_serial',
         action="store_true",
         help=('Use raw serial instead of HDLC/RPC'))
+    parser.add_argument("--token-databases",
+                        metavar='elf_or_token_database',
+                        nargs="+",
+                        action=LoadTokenDatabases,
+                        help="Path to tokenizer database csv file(s).")
     group.add_argument('-s',
                        '--socket-addr',
                        type=str,
@@ -152,6 +160,7 @@ def _start_ipython_raw_terminal() -> None:
 
     interactive_console.hide_windows('Host Logs')
     interactive_console.hide_windows('Serial Debug')
+    interactive_console.hide_windows('Python Repl')
 
     # Setup Python logger propagation
     interactive_console.setup_python_logging()
@@ -236,7 +245,8 @@ def read(self, num_bytes: int = PW_RPC_MAX_PACKET_SIZE):
 
 
 def write_to_output(data: bytes,
-                    unused_output: BinaryIO = sys.stdout.buffer,):
+                    unused_output: BinaryIO = sys.stdout.buffer,
+                    detokenizer=None):
     log_line = data
     RegexStruct = namedtuple('RegexStruct', 'platform type regex match_num')
     LEVEL_MAPPING = {"I": logging.INFO, "W": logging.WARNING, "P": logging.INFO,
@@ -276,9 +286,17 @@ def write_to_output(data: bytes,
                 fields.update(match.groupdict())
                 if "level" in match.groupdict():
                     fields["level"] = LEVEL_MAPPING[fields["level"]]
+                if detokenizer:
+                    _LOG.warn(fields["msg"])
+                    if len(fields["msg"]) % 2:
+                        # TODO the msg likely wrapped, trim for now
+                        fields["msg"] = fields["msg"][:-1]
+                    fields["msg"] = detokenizer.detokenize(
+                        bytes.fromhex(fields["msg"]))
                 break
+
         _DEVICE_LOG.log(fields["level"], fields["msg"], extra={'extra_metadata_fields': {
-                        "time": fields["time"], "type": fields["type"], "mod": fields["mod"]}})
+                        "timestamp": fields["time"], "type": fields["type"], "mod": fields["mod"]}})
 
 
 def _read_raw_serial(read: Callable[[], bytes], output):
@@ -294,6 +312,7 @@ def _read_raw_serial(read: Callable[[], bytes], output):
 
 
 def console(device: str, baudrate: int,
+            token_databases: Collection[tokens.Database],
             socket_addr: str, output: Any, raw_serial: bool) -> int:
     """Starts an interactive RPC console for HDLC."""
     # argparse.FileType doesn't correctly handle '-' for binary files.
@@ -323,15 +342,22 @@ def read(): return serial_device.read(8192)
         default_stream_timeout_s=None,
     )
 
+    detokenizer = Detokenizer(tokens.Database.merged(*token_databases),
+                              show_errors=False) if token_databases else None
+
     if raw_serial:
         threading.Thread(target=_read_raw_serial,
                          daemon=True,
-                         args=(read, write_to_output)).start()
+                         args=(read,
+                               lambda data: write_to_output(
+                                   data, output, detokenizer),
+                               )).start()
         _start_ipython_raw_terminal()
     else:
         _start_ipython_hdlc_terminal(
             HdlcRpcClient(read, PROTOS, default_channels(write),
-                          lambda data: write_to_output(data, output),
+                          lambda data: write_to_output(
+                              data, output, detokenizer),
                           client_impl=callback_client_impl)
         )
     return 0

examples/common/pigweed/rpc_console/py/pyproject.toml

@@ -0,0 +1,16 @@
+# Copyright (c) 2022 Project CHIP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+[build-system]
+requires = ['setuptools', 'wheel']
+build-backend = 'setuptools.build_meta'

examples/common/pigweed/rpc_console/py/setup.cfg

@@ -0,0 +1,29 @@
+# Copyright (c) 2022 Project CHIP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+[metadata]
+name = chip_rpc
+version = 0.0.1
+
+[options]
+packages = find:
+zip_safe = False
+install_requires =
+    pw_console
+    pw_hdlc
+    pw_protobuf_compiler
+    pw_rpc
+    pw_tokenizer
+
+[options.entry_points]
+console_scripts = chip-console = chip_rpc.console:main

examples/common/pigweed/rpc_console/py/setup.py

@@ -0,0 +1,17 @@
+# Copyright (c) 2022 Project CHIP Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import setuptools  # type: ignore
+
+setuptools.setup()  # Package definition in setup.cfg

examples/ipv6only-app/esp32/README.md

@@ -96,7 +96,7 @@ Build or install the [rpc console](../../common/pigweed/rpc_console/README.md)
 
 Start the console:
 
-    $ python -m chip_rpc.console --device /dev/ttyUSB0 -b 115200
+    $ chip-console --device /dev/ttyUSB0 -b 115200
 
 An example flow of performing a scan, connecting, and getting the IPv6 address:
 

examples/light-switch-app/efr32/README.md

@@ -347,7 +347,7 @@ via 2002::2
 
 -   To use the chip-rpc console after it has been installed run:
 
-    `python3 -m chip_rpc.console --device /dev/tty.<SERIALDEVICE> -b 115200 -o /<YourFolder>/pw_log.out`
+    `chip-console --device /dev/tty.<SERIALDEVICE> -b 115200 -o /<YourFolder>/pw_log.out`
 
 -   Then you can simulate a button press or release using the following command
     where : idx = 0 or 1 for Button PB0 or PB1 action = 0 for PRESSED, 1 for

examples/lighting-app/efr32/README.md

@@ -290,7 +290,7 @@ via 2002::2
     `pip3 install out/debug/chip_rpc_console_wheels/*.whl`
 
 -   To use the chip-rpc console after it has been installed run:
-    `python3 -m chip_rpc.console --device /dev/tty.<SERIALDEVICE> -b 115200 -o /<YourFolder>/pw_log.out`
+    `chip-console --device /dev/tty.<SERIALDEVICE> -b 115200 -o /<YourFolder>/pw_log.out`
 
 -   Then you can simulate a button press or release using the following command
     where : idx = 0 or 1 for Button PB0 or PB1 action = 0 for PRESSED, 1 for

examples/lighting-app/linux/README.md

@@ -130,7 +130,7 @@ To cross-compile this example on x64 host and run on **NXP i.MX 8M Mini**
     `pip3 install out/debug/chip_rpc_console_wheels/*.whl`
 
 -   To use the chip-rpc console after it has been installed run:
-    `python3 -m chip_rpc.console -s localhost:33000 -o /<YourFolder>/pw_log.out`
+    `chip-console -s localhost:33000 -o /<YourFolder>/pw_log.out`
 
 -   Then you can Get and Set the light using the RPCs:
     `rpcs.chip.rpc.Lighting.Get()`

examples/lighting-app/mbed/README.md

@@ -266,7 +266,7 @@ parameters as arguments:
 
 Example:
 
-    python -m chip_rpc.console -d /dev/ttyUSB0 -b 115200 -o /tmp/pw_rpc.out
+    chip-console -d /dev/ttyUSB0 -b 115200 -o /tmp/pw_rpc.out
 
 To control the lighting type the following command, where you define if 'on'
 state is true or false:

examples/lighting-app/nrfconnect/README.md

@@ -563,7 +563,7 @@ Build or install the [rpc console](../../common/pigweed/rpc_console/README.md)
 
 Start the console
 
-        $ python -m chip_rpc.console --device /dev/ttyUSB0
+        $ chip-console --device /dev/ttyUSB0
 
 From within the console you can then invoke rpcs:
 

examples/lock-app/esp32/README.md

@@ -212,7 +212,7 @@ Build or install the [rpc console](../../common/pigweed/rpc_console/README.md)
 
 Start the console
 
-    python -m chip_rpc.console --device /dev/ttyUSB0
+    chip-console --device /dev/ttyUSB0
 
 From within the console you can then invoke rpcs:
 

examples/lock-app/mbed/README.md

@@ -256,7 +256,7 @@ parameters as arguments:
 
 Example:
 
-    python -m chip_rpc.console -d /dev/ttyUSB0 -b 115200 -o /tmp/pw_rpc.out
+    chip-console -d /dev/ttyUSB0 -b 115200 -o /tmp/pw_rpc.out
 
 To control the lock type the following command, where you define if 'on' state
 is true or false:

examples/pigweed-app/mbed/README.md

@@ -225,7 +225,7 @@ parameters as arguments:
 
 Example:
 
-    python -m chip_rpc.console -d /dev/ttyUSB0 -b 115200 -o /tmp/pw_rpc.out
+    chip-console -d /dev/ttyUSB0 -b 115200 -o /tmp/pw_rpc.out
 
 To send the echo message type the following command, where you define the
 message content: