Payload refactoring

Makefile

@@ -51,7 +51,7 @@ cov-dev-full: .develop
 	@echo "Run without extensions"
 	@AIOHTTP_NO_EXTENSIONS=1 py.test --cov=aiohttp tests
 	@echo "Run in debug mode"
-	@PYTHONASYNCIODEBUG=1 py.test --cov=aiohttp --cov-append tests
+	@PYTHONASYNCIODEBUG=1 py.test -s -v --cov=aiohttp --cov-append tests
 	@echo "Regular run"
 	@py.test --cov=aiohttp --cov-report=term --cov-report=html --cov-append tests
 	@echo "open file://`pwd`/coverage/index.html"

aiohttp/__init__.py

@@ -4,13 +4,16 @@
 
 from . import hdrs  # noqa
 from .client import *  # noqa
+from .formdata import *  # noqa
 from .helpers import *  # noqa
 from .http_message import HttpVersion, HttpVersion10, HttpVersion11  # noqa
 from .http_websocket import WSMsgType, WSCloseCode, WSMessage, WebSocketError  # noqa
 from .streams import *  # noqa
 from .multipart import *  # noqa
 from .file_sender import FileSender  # noqa
 from .cookiejar import CookieJar  # noqa
+from .payload import *  # noqa
+from .payload_streamer import *  # noqa
 from .resolver import *  # noqa
 
 # deprecated #1657
@@ -23,9 +26,12 @@
 
 
 __all__ = (client.__all__ +  # noqa
+           formdata.__all__ +  # noqa
            helpers.__all__ +  # noqa
-           streams.__all__ +  # noqa
            multipart.__all__ +  # noqa
+           payload.__all__ +  # noqa
+           payload_streamer.__all__ +  # noqa
+           streams.__all__ +  # noqa
            ('hdrs', 'FileSender',
             'HttpVersion', 'HttpVersion10', 'HttpVersion11',
             'WSMsgType', 'MsgType', 'WSCloseCode',

aiohttp/abc.py

@@ -129,3 +129,20 @@ def update_cookies(self, cookies, response_url=None):
     @abstractmethod
     def filter_cookies(self, request_url):
         """Return the jar's cookies filtered by their attributes."""
+
+
+class AbstractPayloadWriter(ABC):
+
+    @abstractmethod
+    def write(self, chunk):
+        """Write chunk into stream"""
+
+    @asyncio.coroutine  # pragma: no branch
+    @abstractmethod
+    def write_eof(self, chunk=b''):
+        """Write last chunk"""
+
+    @asyncio.coroutine  # pragma: no branch
+    @asyncio.coroutine
+    def drain(self):
+        """Flush the write buffer."""

aiohttp/client_reqrep.py

@@ -1,8 +1,6 @@
 import asyncio
 import io
 import json
-import mimetypes
-import os
 import sys
 import traceback
 import warnings
@@ -13,11 +11,11 @@
 
 import aiohttp
 
-from . import hdrs, helpers, http, streams
+from . import hdrs, helpers, http, payload
+from .formdata import FormData
 from .helpers import PY_35, HeadersMixin, SimpleCookie, _TimeServiceTimeoutNoop
 from .http import HttpMessage
 from .log import client_logger
-from .multipart import MultipartWriter
 from .streams import FlowControlStreamReader
 
 try:
@@ -219,86 +217,54 @@ def update_auth(self, auth):
 
         self.headers[hdrs.AUTHORIZATION] = auth.encode()
 
-    def update_body_from_data(self, data, skip_auto_headers):
-        if not data:
+    def update_body_from_data(self, body, skip_auto_headers):
+        if not body:
             return
 
-        if isinstance(data, str):
-            data = data.encode(self.encoding)
-
-        if isinstance(data, (bytes, bytearray)):
-            self.body = data
-            if (hdrs.CONTENT_TYPE not in self.headers and
-                    hdrs.CONTENT_TYPE not in skip_auto_headers):
-                self.headers[hdrs.CONTENT_TYPE] = 'application/octet-stream'
-            if hdrs.CONTENT_LENGTH not in self.headers and not self.chunked:
-                self.headers[hdrs.CONTENT_LENGTH] = str(len(self.body))
-
-        elif isinstance(data, (asyncio.StreamReader, streams.StreamReader,
-                               streams.DataQueue)):
-            self.body = data
+        if asyncio.iscoroutine(body):
+            warnings.warn(
+                'coroutine as data object is deprecated, '
+                'use aiohttp.streamer  #1664',
+                DeprecationWarning, stacklevel=2)
 
-        elif asyncio.iscoroutine(data):
-            self.body = data
+            self.body = body
             if (hdrs.CONTENT_LENGTH not in self.headers and
                     self.chunked is None):
                 self.chunked = True
 
-        elif isinstance(data, io.IOBase):
-            assert not isinstance(data, io.StringIO), \
-                'attempt to send text data instead of binary'
-            self.body = data
-            if not self.chunked and isinstance(data, io.BytesIO):
-                # Not chunking if content-length can be determined
-                size = len(data.getbuffer())
-                self.headers[hdrs.CONTENT_LENGTH] = str(size)
-                self.chunked = False
-            elif (not self.chunked and
-                  isinstance(data, (io.BufferedReader, io.BufferedRandom))):
-                # Not chunking if content-length can be determined
-                try:
-                    size = os.fstat(data.fileno()).st_size - data.tell()
-                    self.headers[hdrs.CONTENT_LENGTH] = str(size)
-                    self.chunked = False
-                except OSError:
-                    # data.fileno() is not supported, e.g.
-                    # io.BufferedReader(io.BytesIO(b'data'))
-                    self.chunked = True
-            else:
-                self.chunked = True
+            return
 
-            if hasattr(data, 'mode'):
-                if data.mode == 'r':
-                    raise ValueError('file {!r} should be open in binary mode'
-                                     ''.format(data))
-            if (hdrs.CONTENT_TYPE not in self.headers and
-                hdrs.CONTENT_TYPE not in skip_auto_headers and
-                    hasattr(data, 'name')):
-                mime = mimetypes.guess_type(data.name)[0]
-                mime = 'application/octet-stream' if mime is None else mime
-                self.headers[hdrs.CONTENT_TYPE] = mime
-
-        elif isinstance(data, MultipartWriter):
-            self.body = data.serialize()
-            self.headers.update(data.headers)
-            self.chunked = True
+        # FormData
+        if isinstance(body, FormData):
+            body = body(self.encoding)
 
-        else:
-            if not isinstance(data, helpers.FormData):
-                data = helpers.FormData(data)
+        try:
+            body = payload.PAYLOAD_REGISTRY.get(body)
+        except payload.LookupError:
+            body = FormData(body)(self.encoding)
 
-            self.body = data(self.encoding)
+        self.body = body
 
-            if (hdrs.CONTENT_TYPE not in self.headers and
-                    hdrs.CONTENT_TYPE not in skip_auto_headers):
-                self.headers[hdrs.CONTENT_TYPE] = data.content_type
+        # enable chunked encoding if needed
+        if not self.chunked:
+            if hdrs.CONTENT_LENGTH not in self.headers:
+                size = body.size
+                if size is None:
+                    self.chunked = True
+                else:
+                    if hdrs.CONTENT_LENGTH not in self.headers:
+                        self.headers[hdrs.CONTENT_LENGTH] = str(size)
 
-            if data.is_multipart:
-                self.chunked = True
-            else:
-                if (hdrs.CONTENT_LENGTH not in self.headers and
-                        not self.chunked):
-                    self.headers[hdrs.CONTENT_LENGTH] = str(len(self.body))
+        # set content-type
+        if (hdrs.CONTENT_TYPE not in self.headers and
+                hdrs.CONTENT_TYPE not in skip_auto_headers):
+            self.headers[hdrs.CONTENT_TYPE] = body.content_type
+
+        # copy payload headers
+        if body.headers:
+            for (key, value) in body.headers.items():
+                if key not in self.headers:
+                    self.headers[key] = value
 
     def update_transfer_encoding(self):
         """Analyze transfer-encoding header."""
@@ -344,7 +310,10 @@ def write_bytes(self, request, conn):
             yield from self._continue
 
         try:
-            if asyncio.iscoroutine(self.body):
+            if isinstance(self.body, payload.Payload):
+                yield from self.body.write(request)
+
+            elif asyncio.iscoroutine(self.body):
                 exc = None
                 value = None
                 stream = self.body
@@ -377,29 +346,6 @@ def write_bytes(self, request, conn):
                         raise ValueError(
                             'Bytes object is expected, got: %s.' %
                             type(result))
-
-            elif isinstance(self.body, (asyncio.StreamReader,
-                                        streams.StreamReader)):
-                chunk = yield from self.body.read(streams.DEFAULT_LIMIT)
-                while chunk:
-                    yield from request.write(chunk, drain=True)
-                    chunk = yield from self.body.read(streams.DEFAULT_LIMIT)
-
-            elif isinstance(self.body, streams.DataQueue):
-                while True:
-                    try:
-                        chunk = yield from self.body.read()
-                        if not chunk:
-                            break
-                        yield from request.write(chunk)
-                    except streams.EofStream:
-                        break
-
-            elif isinstance(self.body, io.IOBase):
-                chunk = self.body.read(streams.DEFAULT_LIMIT)
-                while chunk:
-                    request.write(chunk)
-                    chunk = self.body.read(self.chunked)
             else:
                 if isinstance(self.body, (bytes, bytearray)):
                     self.body = (self.body,)

aiohttp/formdata.py

@@ -0,0 +1,122 @@
+import io
+from urllib.parse import urlencode
+
+from multidict import MultiDict, MultiDictProxy
+
+from . import hdrs, multipart, payload
+from .helpers import guess_filename
+
+__all__ = ('FormData',)
+
+
+class FormData:
+    """Helper class for multipart/form-data and
+    application/x-www-form-urlencoded body generation."""
+
+    def __init__(self, fields=(), quote_fields=True):
+        self._writer = multipart.MultipartWriter('form-data')
+        self._fields = []
+        self._is_multipart = False
+        self._quote_fields = quote_fields
+
+        if isinstance(fields, dict):
+            fields = list(fields.items())
+        elif not isinstance(fields, (list, tuple)):
+            fields = (fields,)
+        self.add_fields(*fields)
+
+    def add_field(self, name, value, *, content_type=None, filename=None,
+                  content_transfer_encoding=None):
+
+        if isinstance(value, io.IOBase):
+            self._is_multipart = True
+        elif isinstance(value, (bytes, bytearray, memoryview)):
+            if filename is None and content_transfer_encoding is None:
+                filename = name
+
+        type_options = MultiDict({'name': name})
+        if filename is not None and not isinstance(filename, str):
+            raise TypeError('filename must be an instance of str. '
+                            'Got: %s' % filename)
+        if filename is None and isinstance(value, io.IOBase):
+            filename = guess_filename(value, name)
+        if filename is not None:
+            type_options['filename'] = filename
+            self._is_multipart = True
+
+        headers = {}
+        if content_type is not None:
+            if not isinstance(content_type, str):
+                raise TypeError('content_type must be an instance of str. '
+                                'Got: %s' % content_type)
+            headers[hdrs.CONTENT_TYPE] = content_type
+            self._is_multipart = True
+        if content_transfer_encoding is not None:
+            if not isinstance(content_transfer_encoding, str):
+                raise TypeError('content_transfer_encoding must be an instance'
+                                ' of str. Got: %s' % content_transfer_encoding)
+            headers[hdrs.CONTENT_TRANSFER_ENCODING] = content_transfer_encoding
+            self._is_multipart = True
+
+        self._fields.append((type_options, headers, value))
+
+    def add_fields(self, *fields):
+        to_add = list(fields)
+
+        while to_add:
+            rec = to_add.pop(0)
+
+            if isinstance(rec, io.IOBase):
+                k = guess_filename(rec, 'unknown')
+                self.add_field(k, rec)
+
+            elif isinstance(rec, (MultiDictProxy, MultiDict)):
+                to_add.extend(rec.items())
+
+            elif isinstance(rec, (list, tuple)) and len(rec) == 2:
+                k, fp = rec
+                self.add_field(k, fp)
+
+            else:
+                raise TypeError('Only io.IOBase, multidict and (name, file) '
+                                'pairs allowed, use .add_field() for passing '
+                                'more complex parameters, got {!r}'
+                                .format(rec))
+
+    def _gen_form_urlencoded(self, encoding):
+        # form data (x-www-form-urlencoded)
+        data = []
+        for type_options, _, value in self._fields:
+            data.append((type_options['name'], value))
+
+        return payload.BytesPayload(
+            urlencode(data, doseq=True).encode(encoding),
+            content_type='application/x-www-form-urlencoded')
+
+    def _gen_form_data(self, encoding):
+        """Encode a list of fields using the multipart/form-data MIME format"""
+        for dispparams, headers, value in self._fields:
+            if hdrs.CONTENT_TYPE in headers:
+                part = payload.get_payload(
+                    value, content_type=headers[hdrs.CONTENT_TYPE],
+                    headers=headers, encoding=encoding)
+            else:
+                part = payload.get_payload(
+                    value, headers=headers, encoding=encoding)
+            if dispparams:
+                part.set_content_disposition(
+                    'form-data', quote_fields=self._quote_fields, **dispparams
+                )
+                # FIXME cgi.FieldStorage doesn't likes body parts with
+                # Content-Length which were sent via chunked transfer encoding
+                part.headers.pop(hdrs.CONTENT_LENGTH, None)
+
+            self._writer.append_payload(part)
+
+        return self._writer
+
+    def __call__(self, encoding):
+        if self._is_multipart:
+            return self._gen_form_data(encoding)
+        else:
+            return self._gen_form_urlencoded(encoding)