From e4d8db475f9d0cb325580ceb2f9085c972766bcb Mon Sep 17 00:00:00 2001 From: Al Semjonovs Date: Mon, 21 Oct 2024 16:01:23 +0000 Subject: [PATCH] pw_tokenizer: Add prefix to AutoUpdatingDetokenizer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Allow AutoUpdatingDetokenizer to set the prefix in base Detokenizer class Change-Id: I34a4981037fe34f44d8a524b3d04647bd47e426f Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/243612 Presubmit-Verified: CQ Bot Account Commit-Queue: Auto-Submit Lint: Lint 🤖 Reviewed-by: Wyatt Hepler Docs-Not-Needed: Wyatt Hepler Pigweed-Auto-Submit: Al Semjonovs --- pw_tokenizer/py/pw_tokenizer/detokenize.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pw_tokenizer/py/pw_tokenizer/detokenize.py b/pw_tokenizer/py/pw_tokenizer/detokenize.py index 1f7f74502..cca5c617e 100755 --- a/pw_tokenizer/py/pw_tokenizer/detokenize.py +++ b/pw_tokenizer/py/pw_tokenizer/detokenize.py @@ -524,6 +524,7 @@ def __init__( *paths_or_files: Path | str, min_poll_period_s: float = 1.0, pool: Executor = ThreadPoolExecutor(max_workers=1), + prefix: str | bytes = encode.NESTED_TOKEN_PREFIX, ) -> None: self.paths = tuple(self._DatabasePath(path) for path in paths_or_files) self.min_poll_period_s = min_poll_period_s @@ -531,7 +532,7 @@ def __init__( # Thread pool to use for loading the databases. Limit to a single # worker since this is low volume and not time critical. self._pool = pool - super().__init__(*(path.load() for path in self.paths)) + super().__init__(*(path.load() for path in self.paths), prefix=prefix) def __del__(self) -> None: self._pool.shutdown(wait=False)