- Fix some cases where the creation of extremely large output buffers…

… (greater

  than 2GB, suspected to be buffers added via ``wsgi.file_wrapper``) might
  cause an OverflowError on Python 2.  See
  #47.

See #47

CHANGES.txt

@@ -1,3 +1,11 @@
+Next release
+------------
+
+- Fix some cases where the creation of extremely large output buffers (greater
+  than 2GB, suspected to be buffers added via ``wsgi.file_wrapper``) might
+  cause an OverflowError on Python 2.  See
+  https://github.com/Pylons/waitress/issues/47.
+
 0.8.7 (2013-08-29)
 ------------------
 

waitress/buffers.py

@@ -186,8 +186,8 @@ class OverflowableBuffer(object):
     """
     This buffer implementation has four stages:
     - No data
-    - String-based buffer
-    - StringIO-based buffer
+    - Bytes-based buffer
+    - BytesIO-based buffer
     - Temporary file storage
     The first two stages are fastest for simple transfers.
     """
@@ -203,11 +203,15 @@ def __init__(self, overflow):
     def __len__(self):
         buf = self.buf
         if buf is not None:
+            # use buf.__len__ rather than len(buf) FBO of not getting
+            # OverflowError on Python 2
             return buf.__len__()
         else:
             return self.strbuf.__len__()
 
     def __nonzero__(self):
+        # use self.__len__ rather than len(self) FBO of not getting
+        # OverflowError on Python 2
         return self.__len__() > 0
 
     __bool__ = __nonzero__ # py3
@@ -241,7 +245,9 @@ def append(self, s):
                 return
             buf = self._create_buffer()
         buf.append(s)
-        sz = len(buf)
+        # use buf.__len__ rather than len(buf) FBO of not getting
+        # OverflowError on Python 2
+        sz = buf.__len__()
         if not self.overflowed:
             if sz >= self.overflow:
                 self._set_large_buffer()
@@ -278,7 +284,9 @@ def prune(self):
             return
         buf.prune()
         if self.overflowed:
-            sz = len(buf)
+            # use buf.__len__ rather than len(buf) FBO of not getting
+            # OverflowError on Python 2
+            sz = buf.__len__()
             if sz < self.overflow:
                 # Revert to a faster buffer.
                 self._set_small_buffer()

waitress/channel.py

@@ -89,7 +89,10 @@ def any_outbuf_has_data(self):
         return False
 
     def total_outbufs_len(self):
-        return sum([len(b) for b in self.outbufs]) # genexpr == more funccalls
+        # genexpr == more funccalls
+        # use b.__len__ rather than len(b) FBO of not getting OverflowError
+        # on Python 2
+        return sum([b.__len__() for b in self.outbufs]) 
 
     def writable(self):
         # if there's data in the out buffer or we've been instructed to close
@@ -233,7 +236,9 @@ def _flush_some(self):
 
         while True:
             outbuf = self.outbufs[0]
-            outbuflen = len(outbuf)
+            # use outbuf.__len__ rather than len(outbuf) FBO of not getting
+            # OverflowError on Python 2
+            outbuflen = outbuf.__len__()
             if outbuflen <= 0:
                 # self.outbufs[-1] must always be a writable outbuf
                 if len(self.outbufs) > 1:

waitress/tests/test_buffers.py

@@ -284,6 +284,17 @@ def test__create_buffer_small(self):
         self.assertEqual(inst.buf.get(100), b'x' * 5)
         self.assertEqual(inst.strbuf, b'')
 
+    def test_append_with_len_more_than_max_int(self):
+        import sys
+        inst = self._makeOne()
+        inst.overflowed = True
+        buf = DummyBuffer(length=sys.maxint)
+        inst.buf = buf
+        result = inst.append(b'x')
+        # we don't want this to throw an OverflowError on Python 2 (see
+        # https://github.com/Pylons/waitress/issues/47)
+        self.assertEqual(result, None)
+
     def test_append_buf_None_not_longer_than_srtbuf_limit(self):
         inst = self._makeOne()
         inst.strbuf = b'x' * 5
@@ -373,6 +384,17 @@ def __len__(self):
         inst.prune()
         self.assertNotEqual(inst.buf, buf)
 
+    def test_prune_with_buflen_more_than_max_int(self):
+        import sys
+        inst = self._makeOne()
+        inst.overflowed = True
+        buf = DummyBuffer(length=sys.maxint+1)
+        inst.buf = buf
+        result = inst.prune()
+        # we don't want this to throw an OverflowError on Python 2 (see
+        # https://github.com/Pylons/waitress/issues/47)
+        self.assertEqual(result, None)
+
     def test_getfile_buf_None(self):
         inst = self._makeOne()
         f = inst.getfile()
@@ -417,3 +439,16 @@ def seek(self, v, whence=0):
     def tell(self):
         v = self.tellresults.pop(0)
         return v
+
+class DummyBuffer(object):
+    def __init__(self, length=0):
+        self.length = length
+
+    def __len__(self):
+        return self.length
+
+    def append(self, s):
+        self.length = self.length + len(s)
+
+    def prune(self):
+        pass

waitress/tests/test_channel.py

@@ -22,6 +22,18 @@ def test_ctor(self):
         self.assertEqual(inst.addr, '127.0.0.1')
         self.assertEqual(map[100], inst)
 
+    def test_total_outbufs_len_an_outbuf_size_gt_sys_maxint(self):
+        import sys
+        inst, _, map = self._makeOneWithMap()
+        class DummyHugeBuffer(object):
+            def __len__(self):
+                return sys.maxint + 1
+        inst.outbufs = [DummyHugeBuffer()]
+        result = inst.total_outbufs_len()
+        # we are testing that this method does not raise an OverflowError
+        # (see https://github.com/Pylons/waitress/issues/47)
+        self.assertEqual(result, sys.maxint+1)
+
     def test_writable_something_in_outbuf(self):
         inst, sock, map = self._makeOneWithMap()
         inst.outbufs[0].append(b'abc')
@@ -256,6 +268,27 @@ def doraise():
         self.assertEqual(inst.outbufs, [buffer])
         self.assertEqual(len(inst.logger.exceptions), 1)
 
+    def test__flush_some_outbuf_len_gt_sys_maxint(self):
+        import sys
+        inst, sock, map = self._makeOneWithMap()
+        class DummyHugeOutbuffer(object):
+            def __init__(self):
+                self.length = sys.maxint + 1
+            def __len__(self):
+                return self.length
+            def get(self, numbytes):
+                self.length = 0
+                return b'123'
+            def skip(self, *args):
+                pass
+        buf = DummyHugeOutbuffer()
+        inst.outbufs = [buf]
+        inst.send = lambda *arg: 0
+        result = inst._flush_some()
+        # we are testing that _flush_some doesn't raise an OverflowError
+        # when one of its outbufs has a __len__ that returns gt sys.maxint
+        self.assertEqual(result, False)
+
     def test_handle_close(self):
         inst, sock, map = self._makeOneWithMap()
         inst.handle_close()