ENH: Extract Text Enhancement (whitespaces)

PyPDF2/_cmap.py

@@ -35,10 +35,17 @@ def build_char_map(
         for x in int_entry:
             if x <= 255:
                 encoding[x] = chr(x)
-    if font_name in _default_fonts_space_width:
+    try:
         # override space_width with new params
-        space_width = _default_fonts_space_width[font_name]
-    sp_width = compute_space_width(ft, space_code, space_width)
+        space_width = _default_fonts_space_width[cast(str, ft["/BaseFont"])]
+    except Exception:
+        pass
+    # I conside the space_code is available on one byte
+    if isinstance(space_code, str):
+        sp = space_code.encode("charmap")[0]
+    else:
+        sp = space_code
+    sp_width = compute_space_width(ft, sp, space_width)
 
     return (
         font_type,
@@ -193,7 +200,7 @@ def parse_to_unicode(
     )
 
     for l in cm.split(b"\n"):
-        if l in (b"", b" "):
+        if l in (b"", b" ") or l[0] == 37:  # 37 = %
             continue
         if b"beginbfrange" in l:
             process_rg = True
@@ -224,7 +231,7 @@ def parse_to_unicode(
                     a += 1
             else:
                 c = int(lst[2], 16)
-                fmt2 = b"%%0%dX" % len(lst[2])
+                fmt2 = b"%%0%dX" % max(4, len(lst[2]))
                 while a <= b:
                     map_dict[
                         unhexlify(fmt % a).decode(
@@ -259,30 +266,40 @@ def compute_space_width(
 ) -> float:
     sp_width: float = space_width * 2  # default value
     w = []
+    w1 = {}
     st: int = 0
-    if "/W" in ft:
-        if "/DW" in ft:
-            sp_width = cast(float, ft["/DW"])
-        w = list(ft["/W"])  # type: ignore
+    if "/DescendantFonts" in ft:  # ft["/Subtype"].startswith("/CIDFontType"):
+        ft1 = ft["/DescendantFonts"][0].get_object()    # type: ignore
+        try:
+            w1[-1] = cast(float, ft1["/DW"])
+        except Exception:
+            w1[-1] = 1000.0
+        w = list(ft1["/W"])  # type: ignore
         while len(w) > 0:
             st = w[0]
             second = w[1]
-            if isinstance(int, second):
-                if st <= space_code and space_code <= second:
-                    sp_width = w[2]
-                    break
+            if isinstance(second, int):
+                for x in range(st, second):
+                    w1[x] = w[2]
                 w = w[3:]
-            if isinstance(list, second):
-                if st <= space_code and space_code <= st + len(second) - 1:
-                    sp_width = second[space_code - st]
+            elif isinstance(second, list):
+                for y in second:
+                    w1[st] = y
+                    st += 1
                 w = w[2:]
             else:
                 warnings.warn(
-                    "unknown widths : \n" + (ft["/W"]).__repr__(),
+                    "unknown widths : \n" + (ft1["/W"]).__repr__(),
                     PdfReadWarning,
                 )
                 break
-    if "/Widths" in ft:
+        try:
+            sp_width = w1[space_code]
+        except Exception:
+            sp_width = (
+                w1[-1] / 2.0
+            )  # if using default we consider space will be only half size
+    elif "/Widths" in ft:
         w = list(ft["/Widths"])  # type: ignore
         try:
             st = cast(int, ft["/FirstChar"])

PyPDF2/_page.py

@@ -1143,22 +1143,53 @@ def _extract_text(
         # are strings where the byte->string encoding was unknown, so adding
         # them to the text here would be gibberish.
 
+        cm_matrix: List[float] = [1.0, 0.0, 0.0, 1.0, 0.0, 0.0]
+        cm_stack = []
         tm_matrix: List[float] = [1.0, 0.0, 0.0, 1.0, 0.0, 0.0]
-        tm_prev: List[float] = [1.0, 0.0, 0.0, 1.0, 0.0, 0.0]
+        tm_prev: List[float] = [
+            1.0,
+            0.0,
+            0.0,
+            1.0,
+            0.0,
+            0.0,
+        ]  # will store cm_matrix * tm_matrix
         char_scale = 1.0
         space_scale = 1.0
         _space_width: float = 500.0  # will be set correctly at first Tf
         TL = 0.0
         font_size = 12.0  # init just in case of
 
-        # tm_matrix: Tuple = tm_matrix, output: str = output, text: str = text,
-        # char_scale: float = char_scale,space_scale : float = space_scale, _space_width: float = _space_width,
-        # TL: float = TL, font_size: float = font_size, cmap = cmap
+        def sign(x: float) -> float:
+            return 1 if x >= 0 else -1
+
+        def mult(m: List[float], n: List[float]) -> List[float]:
+            return [
+                m[0] * n[0] + m[1] * n[2],
+                m[0] * n[1] + m[1] * n[3],
+                m[2] * n[0] + m[3] * n[2],
+                m[2] * n[1] + m[3] * n[3],
+                m[4] * n[0] + m[5] * n[2] + n[4],
+                m[4] * n[1] + m[5] * n[3] + n[5],
+            ]
+
+        def orient(m: List[float]) -> int:
+            if m[3] > 1e-6:
+                return 0
+            elif m[3] < -1e-6:
+                return 180
+            elif m[1] > 0:
+                return 90
+            else:
+                return 270
+
+        def current_spacewidth() -> float:
+            # return space_scale * _space_width * char_scale
+            return _space_width / 1000.0
 
         def process_operation(operator: bytes, operands: List) -> None:
-            nonlocal tm_matrix, tm_prev, output, text, char_scale, space_scale, _space_width, TL, font_size, cmap
-            if tm_matrix[4] != 0 and tm_matrix[5] != 0:  # o reuse of the
-                tm_prev = list(tm_matrix)
+            nonlocal cm_matrix, cm_stack, tm_matrix, tm_prev, output, text, char_scale, space_scale, _space_width, TL, font_size, cmap
+            check_crlf_space: bool = False
             # Table 5.4 page 405
             if operator == b"BT":
                 tm_matrix = [1.0, 0.0, 0.0, 1.0, 0.0, 0.0]
@@ -1172,6 +1203,29 @@ def process_operation(operator: bytes, operands: List) -> None:
             elif operator == b"ET":
                 output += text
                 text = ""
+            # table 4.7, page 219
+            # cm_matrix calculation is a reserved for the moment
+            elif operator == b"q":
+                cm_stack.append(cm_matrix)
+            elif operator == b"Q":
+                try:
+                    cm_matrix = cm_stack.pop()
+                except Exception:
+                    cm_matrix = [1.0, 0.0, 0.0, 1.0, 0.0, 0.0]
+            elif operator == b"cm":
+                output += text
+                text = ""
+                cm_matrix = mult(
+                    [
+                        float(operands[0]),
+                        float(operands[1]),
+                        float(operands[2]),
+                        float(operands[3]),
+                        float(operands[4]),
+                        float(operands[5]),
+                    ],
+                    cm_matrix,
+                )
             # Table 5.2 page 398
             elif operator == b"Tz":
                 char_scale = float(operands[0]) / 100.0
@@ -1203,9 +1257,11 @@ def process_operation(operator: bytes, operands: List) -> None:
                     pass  # keep previous size
             # Table 5.5 page 406
             elif operator == b"Td":
-                tm_matrix[5] += float(operands[1])
+                check_crlf_space = True
                 tm_matrix[4] += float(operands[0])
+                tm_matrix[5] += float(operands[1])
             elif operator == b"Tm":
+                check_crlf_space = True
                 tm_matrix = [
                     float(operands[0]),
                     float(operands[1]),
@@ -1215,56 +1271,101 @@ def process_operation(operator: bytes, operands: List) -> None:
                     float(operands[5]),
                 ]
             elif operator == b"T*":
+                check_crlf_space = True
                 tm_matrix[5] -= TL
+
             elif operator == b"Tj":
-                t: str = ""
-                tt: bytes = (
-                    encode_pdfdocencoding(operands[0])
-                    if isinstance(operands[0], str)
-                    else operands[0]
-                )
-                if isinstance(cmap[0], str):
-                    try:
-                        t = tt.decode(cmap[0], "surrogatepass")  # apply str encoding
-                    except Exception:  # the data does not match the expectation, we use the alternative ; text extraction may not be good
-                        t = tt.decode(
-                            "utf-16-be" if cmap[0] == "charmap" else "charmap",
-                            "surrogatepass",
-                        )  # apply str encoding
-                else:  # apply dict encoding
-                    t = "".join(
-                        [
-                            cmap[0][x] if x in cmap[0] else bytes((x,)).decode()
-                            for x in tt
-                        ]
+                check_crlf_space = True
+                if isinstance(operands[0], str):
+                    text += operands[0]
+                else:
+                    t: str = ""
+                    tt: bytes = (
+                        encode_pdfdocencoding(operands[0])
+                        if isinstance(operands[0], str)
+                        else operands[0]
                     )
-
-                text += "".join([cmap[1][x] if x in cmap[1] else x for x in t])
+                    if isinstance(cmap[0], str):
+                        try:
+                            t = tt.decode(cmap[0], "surrogatepass")  # apply str encoding
+                        except Exception:  # the data does not match the expectation, we use the alternative ; text extraction may not be good
+                            t = tt.decode(
+                                "utf-16-be" if cmap[0] == "charmap" else "charmap",
+                                "surrogatepass",
+                            )  # apply str encoding
+                    else:  # apply dict encoding
+                        t = "".join(
+                            [
+                                cmap[0][x] if x in cmap[0] else bytes((x,)).decode()
+                                for x in tt
+                            ]
+                        )
+
+                    text += "".join([cmap[1][x] if x in cmap[1] else x for x in t])
             else:
                 return None
-            # process text changes due to positionchange: " "
-            if tm_matrix[5] <= (
-                tm_prev[5]
-                - font_size  # remove scaling * sqrt(tm_matrix[2] ** 2 + tm_matrix[3] ** 2)
-            ):  # it means that we are moving down by one line
-                output += text + "\n"  # .translate(cmap) + "\n"
-                text = ""
-            elif tm_matrix[4] >= (
-                tm_prev[4] + space_scale * _space_width * char_scale
-            ):  # it means that we are moving down by one line
-                text += " "
-            return None
-            # for clarity Operator in (b"g",b"G") : nothing to do
-            # end of process_operation ######
+            if check_crlf_space:
+                m = mult(tm_matrix, cm_matrix)
+                o = orient(m)
+                deltaX = m[4] - tm_prev[4]
+                deltaY = m[5] - tm_prev[5]
+                k = math.sqrt(abs(m[0] * m[3]) + abs(m[1] * m[2]))
+                f = font_size * k
+                tm_prev = m
+                try:
+                    if o == 0:
+                        if deltaY < -0.8 * f:
+                            if (output + text)[-1] != "\n":
+                                text += "\n"
+                        elif (
+                            abs(deltaY) < f * 0.3
+                            and abs(deltaX) > current_spacewidth() * f * 10
+                        ):
+                            if (output + text)[-1] != " ":
+                                text += " "
+                    elif o == 180:
+                        if deltaY > 0.8 * f:
+                            if (output + text)[-1] != "\n":
+                                text += "\n"
+                        elif (
+                            abs(deltaY) < f * 0.3
+                            and abs(deltaX) > current_spacewidth() * f * 10
+                        ):
+                            if (output + text)[-1] != " ":
+                                text += " "
+                    elif o == 90:
+                        if deltaX > 0.8 * f:
+                            if (output + text)[-1] != "\n":
+                                text += "\n"
+                        elif (
+                            abs(deltaX) < f * 0.3
+                            and abs(deltaY) > current_spacewidth() * f * 10
+                        ):
+                            if (output + text)[-1] != " ":
+                                text += " "
+                    elif o == 270:
+                        if deltaX < -0.8 * f:
+                            if (output + text)[-1] != "\n":
+                                text += "\n"
+                        elif (
+                            abs(deltaX) < f * 0.3
+                            and abs(deltaY) > current_spacewidth() * f * 10
+                        ):
+                            if (output + text)[-1] != " ":
+                                text += " "
+                except Exception:
+                    pass
 
         for operands, operator in content.operations:
             # multiple operators are defined in here ####
             if operator == b"'":
                 process_operation(b"T*", [])
                 process_operation(b"Tj", operands)
             elif operator == b'"':
+                process_operation(b"Tw", [operands[0]])
+                process_operation(b"Tc", [operands[1]])
                 process_operation(b"T*", [])
-                process_operation(b"TJ", operands)
+                process_operation(b"Tj", operands[2:])
             elif operator == b"TD":
                 process_operation(b"TL", [-operands[1]])
                 process_operation(b"Td", operands)
@@ -1273,15 +1374,23 @@ def process_operation(operator: bytes, operands: List) -> None:
                     if isinstance(op, (str, bytes)):
                         process_operation(b"Tj", [op])
                     if isinstance(op, (int, float, NumberObject, FloatObject)):
-                        process_operation(b"Td", [-op, 0.0])
+                        if (
+                            (abs(float(op)) >= _space_width)
+                            and (abs(float(op)) <= 8 * _space_width)
+                            and (text[-1] != " ")
+                        ):
+                            process_operation(b"Tj", [" "])
             elif operator == b"Do":
                 output += text
-                if output != "":
-                    output += "\n"
+                try:
+                    if output[-1] != "\n":
+                        output += "\n"
+                except IndexError:
+                    pass
                 try:
                     xobj = resources_dict["/XObject"]  # type: ignore
                     if xobj[operands[0]]["/Subtype"] != "/Image":  # type: ignore
-                        output += text
+                        # output += text
                         text = self.extract_xform_text(xobj[operands[0]], space_width)  # type: ignore
                         output += text
                 except Exception:

tests/test_page.py

@@ -35,6 +35,7 @@ def get_all_sample_files():
     [m for m in all_files_meta["data"] if not m["encrypted"]],
     ids=[m["path"] for m in all_files_meta["data"] if not m["encrypted"]],
 )
+@pytest.mark.filterwarnings("ignore::PyPDF2.errors.PdfReadWarning")
 def test_read(meta):
     pdf_path = os.path.join(EXTERNAL_ROOT, meta["path"])
     reader = PdfReader(pdf_path)