From ba34705b70615075009c0c7f27310aed8795c344 Mon Sep 17 00:00:00 2001 From: Jon Oberheide <506986+jonoberheide@users.noreply.github.com> Date: Wed, 13 Nov 2024 20:10:36 -0500 Subject: [PATCH 1/4] use orjson instead of json, like home assistant does. pylutron-caseta spends a lot of time munging json and orjson is fast at that. --- pyproject.toml | 1 + src/pylutron_caseta/cli.py | 6 +++--- src/pylutron_caseta/leap.py | 4 ++-- src/pylutron_caseta/pairing.py | 4 ++-- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index cd2d200..01ae1ab 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,6 +18,7 @@ maintainers = [ dependencies = [ "async_timeout>=3.0.1;python_version<'3.11'", "cryptography", + "orjson", ] classifiers = [ "License :: OSI Approved :: Apache Software License", diff --git a/src/pylutron_caseta/cli.py b/src/pylutron_caseta/cli.py index 9483e7b..854c3af 100644 --- a/src/pylutron_caseta/cli.py +++ b/src/pylutron_caseta/cli.py @@ -2,7 +2,7 @@ import asyncio import functools -import json +import orjson as json import logging import socket import ssl @@ -271,7 +271,7 @@ async def _connect( @click.option( "-o", "--output", - type=click.File("w", encoding="utf8"), + type=click.File("wb"), default="-", help="Save the response into a file.", ) @@ -332,4 +332,4 @@ async def leap( else: output.write(json.dumps(response.Body)) - output.write("\n") + output.write(b"\n") diff --git a/src/pylutron_caseta/leap.py b/src/pylutron_caseta/leap.py index c133a4f..ce6a2d0 100644 --- a/src/pylutron_caseta/leap.py +++ b/src/pylutron_caseta/leap.py @@ -1,7 +1,7 @@ """LEAP protocol layer.""" import asyncio -import json +import orjson as json import logging import re import uuid @@ -68,7 +68,7 @@ def clean_up(future): future.add_done_callback(clean_up) try: - text = json.dumps(cmd).encode("UTF-8") + text = json.dumps(cmd) _LOG.debug("sending %s", text) self._writer.write(text + b"\r\n") diff --git a/src/pylutron_caseta/pairing.py b/src/pylutron_caseta/pairing.py index 13c6267..44fb5b0 100644 --- a/src/pylutron_caseta/pairing.py +++ b/src/pylutron_caseta/pairing.py @@ -1,7 +1,7 @@ """Guide the user through pairing and save the necessary files.""" import asyncio -import json +import orjson as json import logging import socket import ssl @@ -61,7 +61,7 @@ async def async_read_json(self, timeout): async def async_write_json(self, obj): """Write an object.""" - buffer = f"{json.dumps(obj)}\r\n".encode("ASCII") + buffer = json.dumps(obj) + b"\r\n" self._writer.write(buffer) LOGGER.debug("sent: %s", buffer) From 16896791bc18ed94e0dd935e4ad61fb578802433 Mon Sep 17 00:00:00 2001 From: Jon Oberheide Date: Wed, 13 Nov 2024 20:44:52 -0500 Subject: [PATCH 2/4] fix typing issue on py3.12 --- src/pylutron_caseta/cli.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pylutron_caseta/cli.py b/src/pylutron_caseta/cli.py index 854c3af..f1d0254 100644 --- a/src/pylutron_caseta/cli.py +++ b/src/pylutron_caseta/cli.py @@ -9,7 +9,7 @@ import urllib.parse from contextlib import asynccontextmanager from pathlib import Path -from typing import Any, AsyncIterator, List, Optional, TextIO +from typing import Any, AsyncIterator, List, Optional, TextIO, BinaryIO from urllib.parse import urlparse import click @@ -289,7 +289,7 @@ async def leap( data: Optional[str], paging: Optional[str], fail: bool, - output: TextIO, + output: BinaryIO, verbose: bool, ): """ From 2cf8f1d24febeefbdb077c19b6e0fb506b27361f Mon Sep 17 00:00:00 2001 From: Jon Oberheide Date: Thu, 14 Nov 2024 11:59:08 -0500 Subject: [PATCH 3/4] use orjson explicitly in the import instead of aliasing it as json. also use writelines() in a couple places. --- src/pylutron_caseta/cli.py | 10 +++++----- src/pylutron_caseta/leap.py | 9 +++++---- src/pylutron_caseta/pairing.py | 8 ++++---- tests/test_leap.py | 22 +++++++++++----------- tests/test_smartbridge.py | 4 ++-- 5 files changed, 27 insertions(+), 26 deletions(-) diff --git a/src/pylutron_caseta/cli.py b/src/pylutron_caseta/cli.py index f1d0254..abc537a 100644 --- a/src/pylutron_caseta/cli.py +++ b/src/pylutron_caseta/cli.py @@ -2,7 +2,6 @@ import asyncio import functools -import orjson as json import logging import socket import ssl @@ -12,6 +11,7 @@ from typing import Any, AsyncIterator, List, Optional, TextIO, BinaryIO from urllib.parse import urlparse +import orjson import click import xdg from zeroconf import DNSQuestionType, InterfaceChoice, ServiceListener @@ -298,8 +298,8 @@ async def leap( LEAP is similar to JSON over HTTP, and this tool is similar to Curl. """ async with _connect(resource, cacert, cert, key) as connection: - body = json.loads(data) if data is not None else None - paging_json = json.loads(paging) if paging is not None else None + body = orjson.loads(data) if data is not None else None + paging_json = orjson.loads(paging) if paging is not None else None res = resource.path if resource.query is not None and len(resource.query) > 0: @@ -328,8 +328,8 @@ async def leap( if response.Header.Paging: message["Header"]["Paging"] = response.Header.Paging - output.write(json.dumps(message)) + output.write(orjson.dumps(message)) else: - output.write(json.dumps(response.Body)) + output.write(orjson.dumps(response.Body)) output.write(b"\n") diff --git a/src/pylutron_caseta/leap.py b/src/pylutron_caseta/leap.py index ce6a2d0..98f92ab 100644 --- a/src/pylutron_caseta/leap.py +++ b/src/pylutron_caseta/leap.py @@ -1,12 +1,13 @@ """LEAP protocol layer.""" import asyncio -import orjson as json import logging import re import uuid from typing import Callable, Dict, List, Optional, Tuple +import orjson + from . import BridgeDisconnectedError from .messages import Response @@ -68,9 +69,9 @@ def clean_up(future): future.add_done_callback(clean_up) try: - text = json.dumps(cmd) + text = orjson.dumps(cmd) _LOG.debug("sending %s", text) - self._writer.write(text + b"\r\n") + self._writer.writelines((text, b"\r\n")) return await future finally: @@ -84,7 +85,7 @@ async def run(self): if received == b"": break - resp_json = json.loads(received.decode("UTF-8")) + resp_json = orjson.loads(received.decode("UTF-8")) if isinstance(resp_json, dict): tag = resp_json.get("Header", {}).pop("ClientTag", None) diff --git a/src/pylutron_caseta/pairing.py b/src/pylutron_caseta/pairing.py index 44fb5b0..064b23b 100644 --- a/src/pylutron_caseta/pairing.py +++ b/src/pylutron_caseta/pairing.py @@ -1,7 +1,6 @@ """Guide the user through pairing and save the necessary files.""" import asyncio -import orjson as json import logging import socket import ssl @@ -9,6 +8,7 @@ from typing import Callable, Optional, Tuple, TypedDict import os +import orjson from cryptography import x509 from cryptography.hazmat.backends import default_backend from cryptography.hazmat.primitives import hashes, serialization @@ -57,12 +57,12 @@ async def async_read_json(self, timeout): return None LOGGER.debug("received: %s", buffer) - return json.loads(buffer.decode("UTF-8")) + return orjson.loads(buffer.decode("UTF-8")) async def async_write_json(self, obj): """Write an object.""" - buffer = json.dumps(obj) + b"\r\n" - self._writer.write(buffer) + buffer = orjson.dumps(obj) + self._writer.writelines((buffer, b"\r\n")) LOGGER.debug("sent: %s", buffer) def __del__(self): diff --git a/tests/test_leap.py b/tests/test_leap.py index e0d120e..cd48a73 100644 --- a/tests/test_leap.py +++ b/tests/test_leap.py @@ -1,6 +1,6 @@ """Tests to validate low-level network interactions.""" import asyncio -import json +import orjson from typing import AsyncGenerator, Iterable, NamedTuple, Tuple import pytest @@ -109,7 +109,7 @@ async def test_call(pipe: Pipe): """Test basic call and response.""" task = asyncio.create_task(pipe.leap.request("ReadRequest", "/test")) - received = json.loads((await pipe.test_reader.readline()).decode("utf-8")) + received = orjson.loads((await pipe.test_reader.readline()).decode("utf-8")) # message should contain ClientTag tag = received.get("Header", {}).pop("ClientTag", None) @@ -123,7 +123,7 @@ async def test_call(pipe: Pipe): "Header": {"ClientTag": tag, "StatusCode": "200 OK", "Url": "/test"}, "Body": {"ok": True}, } - response_bytes = f"{json.dumps(response_obj)}\r\n".encode("utf-8") + response_bytes = orjson.dumps(response_obj) + b"\r\n" pipe.test_writer.write(response_bytes) result = await task @@ -148,7 +148,7 @@ async def test_read_invalid(pipe): """Test reading when invalid data is received.""" pipe.test_writer.write(b"?\r\n") - with pytest.raises(json.JSONDecodeError): + with pytest.raises(orjson.JSONDecodeError): await pipe.leap_loop @@ -190,7 +190,7 @@ def handler2(response): "Header": {"StatusCode": "200 OK", "Url": "/test"}, "Body": {"Index": 0}, } - response_bytes = f"{json.dumps(response_dict)}\r\n".encode("utf-8") + response_bytes = orjson.dumps(response_dict) + b"\r\n" pipe.test_writer.write(response_bytes) response = Response.from_json(response_dict) @@ -203,7 +203,7 @@ def handler2(response): pipe.leap.unsubscribe_unsolicited(handler1) response_dict["Body"]["Index"] = 1 - response_bytes = f"{json.dumps(response_dict)}\r\n".encode("utf-8") + response_bytes = orjson.dumps(response_dict) + b"\r\n" pipe.test_writer.write(response_bytes) response = Response.from_json(response_dict) @@ -232,7 +232,7 @@ def handler(response): task = asyncio.create_task(pipe.leap.subscribe("/test", handler)) - received = json.loads((await pipe.test_reader.readline()).decode("utf-8")) + received = orjson.loads((await pipe.test_reader.readline()).decode("utf-8")) # message should contain ClientTag tag = received.get("Header", {}).pop("ClientTag", None) @@ -249,7 +249,7 @@ def handler(response): "Header": {"ClientTag": tag, "StatusCode": "200 OK", "Url": "/test"}, "Body": {"ok": True}, } - response_bytes = f"{json.dumps(response_obj)}\r\n".encode("utf-8") + response_bytes = orjson.dumps(response_obj) + b"\r\n" pipe.test_writer.write(response_bytes) result, received_tag = await task @@ -267,7 +267,7 @@ def handler(response): "Header": {"ClientTag": tag, "StatusCode": "200 OK", "Url": "/test"}, "Body": {"ok": True}, } - response_bytes = f"{json.dumps(response_obj)}\r\n".encode("utf-8") + response_bytes = orjson.dumps(response_obj) + b"\r\n" pipe.test_writer.write(response_bytes) await asyncio.wait_for(handler_called.wait(), 1.0) @@ -287,14 +287,14 @@ def _handler(_: Response): task = asyncio.create_task(pipe.leap.subscribe("/test", _handler)) - received = json.loads((await pipe.test_reader.readline()).decode("utf-8")) + received = orjson.loads((await pipe.test_reader.readline()).decode("utf-8")) tag = received.get("Header", {}).pop("ClientTag", None) response_obj = { "CommuniqueType": "SubscribeResponse", "Header": {"ClientTag": tag, "StatusCode": "404 Not Found", "Url": "/test"}, } - response_bytes = f"{json.dumps(response_obj)}\r\n".encode("utf-8") + response_bytes = orjson.dumps(response_obj) + b"\r\n" pipe.test_writer.write(response_bytes) result, _ = await task diff --git a/tests/test_smartbridge.py b/tests/test_smartbridge.py index 250b672..87ee285 100644 --- a/tests/test_smartbridge.py +++ b/tests/test_smartbridge.py @@ -2,7 +2,7 @@ import asyncio from collections import defaultdict from datetime import timedelta -import json +import orjson import logging import os import re @@ -54,7 +54,7 @@ def response_from_json_file(filename: str) -> Response: """Fetch a response from a saved JSON file.""" responsedir = os.path.join(os.path.split(__file__)[0], "responses") with open(os.path.join(responsedir, filename), "r", encoding="utf-8") as ifh: - return Response.from_json(json.load(ifh)) + return Response.from_json(orjson.loads(ifh.read())) class Request(NamedTuple): From e4739d8c8e50b7d9ed9528e7bd6f672f71963b4e Mon Sep 17 00:00:00 2001 From: Jon Oberheide Date: Thu, 14 Nov 2024 12:10:21 -0500 Subject: [PATCH 4/4] unnecessary utf decode pre-loads --- src/pylutron_caseta/leap.py | 2 +- src/pylutron_caseta/pairing.py | 2 +- tests/test_leap.py | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/pylutron_caseta/leap.py b/src/pylutron_caseta/leap.py index 98f92ab..c413c43 100644 --- a/src/pylutron_caseta/leap.py +++ b/src/pylutron_caseta/leap.py @@ -85,7 +85,7 @@ async def run(self): if received == b"": break - resp_json = orjson.loads(received.decode("UTF-8")) + resp_json = orjson.loads(received) if isinstance(resp_json, dict): tag = resp_json.get("Header", {}).pop("ClientTag", None) diff --git a/src/pylutron_caseta/pairing.py b/src/pylutron_caseta/pairing.py index 064b23b..18b20d2 100644 --- a/src/pylutron_caseta/pairing.py +++ b/src/pylutron_caseta/pairing.py @@ -57,7 +57,7 @@ async def async_read_json(self, timeout): return None LOGGER.debug("received: %s", buffer) - return orjson.loads(buffer.decode("UTF-8")) + return orjson.loads(buffer) async def async_write_json(self, obj): """Write an object.""" diff --git a/tests/test_leap.py b/tests/test_leap.py index cd48a73..6056311 100644 --- a/tests/test_leap.py +++ b/tests/test_leap.py @@ -109,7 +109,7 @@ async def test_call(pipe: Pipe): """Test basic call and response.""" task = asyncio.create_task(pipe.leap.request("ReadRequest", "/test")) - received = orjson.loads((await pipe.test_reader.readline()).decode("utf-8")) + received = orjson.loads(await pipe.test_reader.readline()) # message should contain ClientTag tag = received.get("Header", {}).pop("ClientTag", None) @@ -232,7 +232,7 @@ def handler(response): task = asyncio.create_task(pipe.leap.subscribe("/test", handler)) - received = orjson.loads((await pipe.test_reader.readline()).decode("utf-8")) + received = orjson.loads(await pipe.test_reader.readline()) # message should contain ClientTag tag = received.get("Header", {}).pop("ClientTag", None) @@ -287,7 +287,7 @@ def _handler(_: Response): task = asyncio.create_task(pipe.leap.subscribe("/test", _handler)) - received = orjson.loads((await pipe.test_reader.readline()).decode("utf-8")) + received = orjson.loads(await pipe.test_reader.readline()) tag = received.get("Header", {}).pop("ClientTag", None) response_obj = {