Merge pull request #1382 from andrewleech/static_range_requests

Static range requests

CONTRIBUTORS.txt

@@ -21,6 +21,7 @@ Alex Lisovoy
 Amy Boyle
 Andrei Ursulenko
 Andrej Antonov
+Andrew Leech
 Andrew Svetlov
 Andrii Soldatenko
 Anton Kasyanov

aiohttp/file_sender.py

@@ -4,6 +4,8 @@
 
 from . import hdrs
 from .helpers import create_future
+from .web_exceptions import (HTTPNotModified, HTTPOk, HTTPPartialContent,
+                             HTTPRequestRangeNotSatisfiable)
 from .web_reqrep import StreamResponse
 
 
@@ -81,8 +83,8 @@ def write_eof():
         # See https://github.com/KeepSafe/aiohttp/issues/958 for details
 
         # send headers
-        headers = ['HTTP/{0.major}.{0.minor} 200 OK\r\n'.format(
-            request.version)]
+        headers = ['HTTP/{0.major}.{0.minor} {1} OK\r\n'.format(
+            request.version, resp.status)]
         for hdr, val in resp.headers.items():
             headers.append('{}: {}\r\n'.format(hdr, val))
         headers.append('\r\n')
@@ -91,6 +93,7 @@ def write_eof():
         out_socket.setblocking(False)
         out_fd = out_socket.fileno()
         in_fd = fobj.fileno()
+        offset = fobj.tell()
 
         bheaders = ''.join(headers).encode('utf-8')
         headers_length = len(bheaders)
@@ -100,7 +103,7 @@ def write_eof():
         try:
             yield from loop.sock_sendall(out_socket, bheaders)
             fut = create_future(loop)
-            self._sendfile_cb(fut, out_fd, in_fd, 0, count, loop, False)
+            self._sendfile_cb(fut, out_fd, in_fd, offset, count, loop, False)
 
             yield from fut
         finally:
@@ -145,23 +148,44 @@ def send(self, request, filepath):
 
         modsince = request.if_modified_since
         if modsince is not None and st.st_mtime <= modsince.timestamp():
-            from .web_exceptions import HTTPNotModified
             raise HTTPNotModified()
 
         ct, encoding = mimetypes.guess_type(str(filepath))
         if not ct:
             ct = 'application/octet-stream'
 
-        resp = self._response_factory()
+        status = HTTPOk.status_code
+        file_size = st.st_size
+        count = file_size
+
+        try:
+            start, end = request.http_range
+        except ValueError:
+            raise HTTPRequestRangeNotSatisfiable
+
+        # If a range request has been made, convert start, end slice notation
+        # into file pointer offset and count
+        if start is not None or end is not None:
+            status = HTTPPartialContent.status_code
+            if start is None and end < 0:  # return tail of file
+                start = file_size + end
+                count = -end
+            else:
+                count = (end or file_size) - start
+
+            if start + count > file_size:
+                raise HTTPRequestRangeNotSatisfiable
+
+        resp = self._response_factory(status=status)
         resp.content_type = ct
         if encoding:
             resp.headers[hdrs.CONTENT_ENCODING] = encoding
         resp.last_modified = st.st_mtime
 
-        file_size = st.st_size
-
-        resp.content_length = file_size
+        resp.content_length = count
         with filepath.open('rb') as f:
-            yield from self._sendfile(request, resp, f, file_size)
+            if start:
+                f.seek(start)
+            yield from self._sendfile(request, resp, f, count)
 
         return resp

aiohttp/web_reqrep.py

@@ -8,6 +8,7 @@
 import io
 import json
 import math
+import re
 import time
 import warnings
 from email.utils import parsedate
@@ -296,6 +297,40 @@ def cookies(self):
         return MappingProxyType(
             {key: val.value for key, val in parsed.items()})
 
+    @property
+    def http_range(self, *, _RANGE=hdrs.RANGE):
+        """
+        The content of Range HTTP header.
+        :returns tuple (start, end): values that can be used for slice
+                                     eg. content[start:end]
+       """
+        rng = self.headers.get(_RANGE)
+        start, end = None, None
+        if rng is not None:
+            try:
+                pattern = r'^bytes=(\d*)-(\d*)$'
+                start, end = re.findall(pattern, rng)[0]
+            except IndexError:  # pattern was not found in header
+                raise ValueError("range not in acceptible format")
+
+            end = int(end) if end else None
+            start = int(start) if start else None
+
+            if start is None and end is not None:
+                # end with no start is to return tail of content
+                end = -end
+
+            if start is not None and end is not None:
+                # end is inclusive in range header, exclusive for slice
+                end += 1
+
+                if start >= end:
+                    raise ValueError('start cannot be after end')
+
+            if start is end is None:  # No valid range supplied
+                raise ValueError('No start or end of range specified')
+        return start, end
+
     @property
     def content(self):
         """Return raw payload stream."""

tests/test_web_sendfile_functional.py

@@ -1,7 +1,6 @@
 import asyncio
 import os
 import pathlib
-
 import pytest
 
 import aiohttp
@@ -323,3 +322,128 @@ def test_static_file_huge(loop, test_client, tmpdir):
         off += len(chunk)
         cnt += 1
     f.close()
+
+
+@asyncio.coroutine
+def test_static_file_range(loop, test_client, sender):
+    filepath = (pathlib.Path(__file__).parent /
+                'software_development_in_picture.jpg')
+
+    @asyncio.coroutine
+    def handler(request):
+        resp = yield from sender(chunk_size=16).send(request, filepath)
+        return resp
+
+    app = web.Application(loop=loop)
+    app.router.add_get('/', handler)
+    client = yield from test_client(lambda loop: app)
+
+    with filepath.open('rb') as f:
+        content = f.read()
+
+    # Ensure the whole file requested in parts is correct
+    responses = yield from asyncio.gather(
+        client.get('/', headers={'Range': 'bytes=0-999'}),
+        client.get('/', headers={'Range': 'bytes=1000-1999'}),
+        client.get('/', headers={'Range': 'bytes=2000-'}),
+        loop=loop
+    )
+    assert len(responses) == 3
+    assert responses[0].status == 206, \
+        "failed 'bytes=0-999': %s" % responses[0].reason
+    assert responses[1].status == 206, \
+        "failed 'bytes=1000-1999': %s" % responses[1].reason
+    assert responses[2].status == 206, \
+        "failed 'bytes=2000-': %s" % responses[2].reason
+
+    body = yield from asyncio.gather(
+        *(resp.read() for resp in responses),
+        loop=loop
+    )
+
+    assert len(body[0]) == 1000, \
+        "failed 'bytes=0-999', received %d bytes" % len(body[0])
+    assert len(body[1]) == 1000, \
+        "failed 'bytes=1000-1999', received %d bytes" % len(body[1])
+    responses[0].close()
+    responses[1].close()
+    responses[2].close()
+
+    assert content == b"".join(body)
+
+
+@asyncio.coroutine
+def test_static_file_range_tail(loop, test_client, sender):
+    filepath = (pathlib.Path(__file__).parent /
+                'software_development_in_picture.jpg')
+
+    @asyncio.coroutine
+    def handler(request):
+        resp = yield from sender(chunk_size=16).send(request, filepath)
+        return resp
+
+    app = web.Application(loop=loop)
+    app.router.add_get('/', handler)
+    client = yield from test_client(lambda loop: app)
+
+    with filepath.open('rb') as f:
+        content = f.read()
+
+    # Ensure the tail of the file is correct
+    resp = yield from client.get('/', headers={'Range': 'bytes=-500'})
+    assert resp.status == 206, resp.reason
+    body4 = yield from resp.read()
+    resp.close()
+    assert content[-500:] == body4
+
+
+@asyncio.coroutine
+def test_static_file_invalid_range(loop, test_client, sender):
+    filepath = (pathlib.Path(__file__).parent /
+                'software_development_in_picture.jpg')
+
+    @asyncio.coroutine
+    def handler(request):
+        resp = yield from sender(chunk_size=16).send(request, filepath)
+        return resp
+
+    app = web.Application(loop=loop)
+    app.router.add_get('/', handler)
+    client = yield from test_client(lambda loop: app)
+
+    flen = filepath.stat().st_size
+
+    # range must be in bytes
+    resp = yield from client.get('/', headers={'Range': 'blocks=0-10'})
+    assert resp.status == 416, 'Range must be in bytes'
+    resp.close()
+
+    # Range end is inclusive
+    resp = yield from client.get('/', headers={'Range': 'bytes=0-%d' % flen})
+    assert resp.status == 416, 'Range end must be inclusive'
+    resp.close()
+
+    # start > end
+    resp = yield from client.get('/', headers={'Range': 'bytes=100-0'})
+    assert resp.status == 416, "Range start can't be greater than end"
+    resp.close()
+
+    # start > end
+    resp = yield from client.get('/', headers={'Range': 'bytes=10-9'})
+    assert resp.status == 416, "Range start can't be greater than end"
+    resp.close()
+
+    # non-number range
+    resp = yield from client.get('/', headers={'Range': 'bytes=a-f'})
+    assert resp.status == 416, 'Range must be integers'
+    resp.close()
+
+    # double dash range
+    resp = yield from client.get('/', headers={'Range': 'bytes=0--10'})
+    assert resp.status == 416, 'double dash in range'
+    resp.close()
+
+    # no range
+    resp = yield from client.get('/', headers={'Range': 'bytes=-'})
+    assert resp.status == 416, 'no range given'
+    resp.close()