Merge pull request #81 from jap/bytes-helpers

add some helpers to deal with streams of bytes

.pre-commit-config.yaml

@@ -4,7 +4,7 @@ repos:
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
-  - repo: https://gitlab.com/pycqa/flake8
+  - repo: https://github.com/pycqa/flake8
     rev: 3.8.4
     hooks:
       - id: flake8

src/parsy/__init__.py

@@ -5,7 +5,7 @@
 import re
 from dataclasses import dataclass
 from functools import wraps
-from typing import Any, Callable, FrozenSet
+from typing import Any, AnyStr, Callable, FrozenSet
 
 __version__ = "2.1"
 
@@ -166,10 +166,23 @@ def combine_dict(self, combine_fn: Callable) -> Parser:
 
     def concat(self) -> Parser:
         """
-        Returns a parser that concatenates together (as a string) the previously
-        produced values.
+        Returns a parser that concatenates together the previously produced values.
+
+        This parser will join the values using the type of the input stream, so
+        when feeding bytes to the parser, the items to be joined must also be bytes.
         """
-        return self.map("".join)
+
+        @Parser
+        def parser(stream: bytes | str, index: int) -> Result:
+            joiner = type(stream)()
+            result = self(stream, index)
+            if result.status:
+                next_parser: Parser = success(joiner.join(result.value))
+                return next_parser(stream, result.index).aggregate(result)
+            else:
+                return result
+
+        return parser
 
     def then(self, other: Parser) -> Parser:
         """
@@ -516,13 +529,16 @@ def fail(expected: str) -> Parser:
     return Parser(lambda _, index: Result.failure(index, expected))
 
 
-def string(expected_string: str, transform: Callable[[str], str] = noop) -> Parser:
+def string(expected_string: AnyStr, transform: Callable[[AnyStr], AnyStr] = noop) -> Parser:
     """
     Returns a parser that expects the ``expected_string`` and produces
     that string value.
 
     Optionally, a transform function can be passed, which will be used on both
     the expected string and tested string.
+
+    This parser can also be instantiated with a bytes value, in which case it can
+    should be applied to a stream of bytes.
     """
 
     slen = len(expected_string)

tests/test_parsy.py

@@ -32,13 +32,13 @@
 
 
 class TestParser(unittest.TestCase):
-    def test_string(self):
+    def test_string_str(self):
         parser = string("x")
         self.assertEqual(parser.parse("x"), "x")
 
         self.assertRaises(ParseError, parser.parse, "y")
 
-    def test_string_transform(self):
+    def test_string_transform_str(self):
         parser = string("x", transform=lambda s: s.lower())
         self.assertEqual(parser.parse("x"), "x")
         self.assertEqual(parser.parse("X"), "x")
@@ -53,6 +53,19 @@ def test_string_transform_2(self):
 
         self.assertRaises(ParseError, parser.parse, "dog")
 
+    def test_string_bytes(self):
+        parser = string(b"x")
+        self.assertEqual(parser.parse(b"x"), b"x")
+
+        self.assertRaises(ParseError, parser.parse, b"y")
+
+    def test_string_transform_bytes(self):
+        parser = string(b"x", transform=lambda s: s.lower())
+        self.assertEqual(parser.parse(b"x"), b"x")
+        self.assertEqual(parser.parse(b"X"), b"x")
+
+        self.assertRaises(ParseError, parser.parse, b"y")
+
     def test_regex_str(self):
         parser = regex(r"[0-9]")
 
@@ -157,11 +170,16 @@ def test_combine_dict_skip_underscores(self):
         ).combine_dict(Pair)
         self.assertEqual(parser.parse("ABC   123"), Pair(word="ABC", number=123))
 
-    def test_concat(self):
+    def test_concat_str(self):
         parser = letter.many().concat()
         self.assertEqual(parser.parse(""), "")
         self.assertEqual(parser.parse("abc"), "abc")
 
+    def test_concat_bytes(self):
+        parser = any_char.many().concat()
+        self.assertEqual(parser.parse(b""), b"")
+        self.assertEqual(parser.parse(b"abc"), b"abc")
+
     def test_generate(self):
         x = y = None