Forwarded headers and otherwise improved proxy handling

sanic/config.py

@@ -26,6 +26,7 @@
     "WEBSOCKET_WRITE_LIMIT": 2 ** 16,
     "GRACEFUL_SHUTDOWN_TIMEOUT": 15.0,  # 15 sec
     "ACCESS_LOG": True,
+    "FORWARDED_SECRET": None,
     "PROXIES_COUNT": -1,
     "FORWARDED_FOR_HEADER": "X-Forwarded-For",
     "REAL_IP_HEADER": "X-Real-IP",

sanic/headers.py

@@ -4,9 +4,9 @@
 
 Options = typing.Dict[str, str]  # key=value fields in various headers
 
-token, quoted = r"([\w!#$%&'*+\-.^_`|~]+)", r'"([^"]*)"'
-parameter = re.compile(fr";\s*{token}=(?:{token}|{quoted})", re.ASCII)
-firefox_quote_escape = re.compile(r'\\"(?!; |\s*$)')
+_token, _quoted = r"([\w!#$%&'*+\-.^_`|~]+)", r'"([^"]*)"'
+_param = re.compile(fr";\s*{_token}=(?:{_token}|{_quoted})", re.ASCII)
+_firefox_quote_escape = re.compile(r'\\"(?!; |\s*$)')
 
 # RFC's quoted-pair escapes are mostly ignored by browsers. Chrome, Firefox and
 # curl all have different escaping, that we try to handle as well as possible,
@@ -24,14 +24,131 @@ def parse_content_header(value: str) -> typing.Tuple[str, Options]:
     Mostly identical to cgi.parse_header and werkzeug.parse_options_header
     but runs faster and handles special characters better. Unescapes quotes.
     """
-    value = firefox_quote_escape.sub("%22", value)
+    value = _firefox_quote_escape.sub("%22", value)
     pos = value.find(";")
     if pos == -1:
         options = {}
     else:
         options = {
             m.group(1).lower(): m.group(2) or m.group(3).replace("%22", '"')
-            for m in parameter.finditer(value[pos:])
+            for m in _param.finditer(value[pos:])
         }
         value = value[:pos]
     return value.strip().lower(), options
+
+
+# https://tools.ietf.org/html/rfc7230#section-3.2.6 and
+# https://tools.ietf.org/html/rfc7239#section-4
+# This regex is for *reversed* strings because that works much faster for
+# right-to-left matching than the other way around. Be wary that all things are
+# a bit backwards! _rparam matches forwarded pairs alike ";key=value"
+_rparam = re.compile(f"(?:{_token}|{_quoted})={_token}\\s*($|[;,])", re.ASCII)
+
+
+def parse_forwarded(headers, config):
+    """Parse HTTP Forwarded headers.
+    Accepts only the last element with secret=`config.FORWARDED_SECRET`
+    :return: dict with matching keys (lowercase) and values, or None.
+    """
+    header = headers.getall("forwarded", None)
+    secret = config.FORWARDED_SECRET
+    if header is None or not secret:
+        return None
+    header = ",".join(header)  # Join multiple header lines
+    if secret not in header:
+        return None
+    # Loop over <separator><key>=<value> elements from right to left
+    ret = sep = pos = None
+    for m in _rparam.finditer(header[::-1]):
+        # Start of new element? (on parser skips and non-semicolon right sep)
+        if m.start() != pos or sep != ";":
+            if secret is True:
+                return normalize(ret)
+            ret = {}
+        pos = m.end()
+        val_token, val_quoted, key, sep = m.groups()
+        key = key.lower()[::-1]
+        val = (val_token or val_quoted.replace('"\\', '"'))[::-1]
+        ret[key] = val
+        if secret is not True and key == "secret" and val == secret:
+            secret = True
+        if secret is True and sep != ";":
+            return normalize(ret)
+    return normalize(ret) if secret is True else None
+
+
+def parse_xforwarded(headers, config):
+    """Parse X-Real-IP, X-Scheme and X-Forwarded-* headers."""
+    proxies_count = config.PROXIES_COUNT
+    if not proxies_count:
+        return None
+    h1, h2 = config.REAL_IP_HEADER, config.FORWARDED_FOR_HEADER
+    addr = h1 and headers.get(h1)
+    forwarded_for = h2 and headers.getall(h2, None)
+    if not addr and forwarded_for:
+        assert proxies_count == -1 or proxies_count > 0, config.PROXIES_COUNT
+        # Combine, split and filter multiple headers' entries
+        proxies = (p.strip() for h in forwarded_for for p in h.split(","))
+        proxies = [p for p in proxies if p]
+        try:
+            addr = proxies[-proxies_count] if proxies_count > 0 else proxies[0]
+        except IndexError:
+            return None
+    if not addr:
+        return None
+    other = (
+        (key, headers.get(header))
+        for key, header in (
+            ("proto", "x-scheme"),
+            ("proto", "x-forwarded-proto"),  # Overrides X-Scheme if present
+            ("host", "x-forwarded-host"),
+            ("port", "x-forwarded-port"),
+            ("path", "x-forwarded-path"),
+        )
+    )
+    return normalize({"for": addr, **{k: v for k, v in other if v}})
+
+
+_ipv6 = r"(?:[0-9A-Fa-f]{0,4}:){2,7}[0-9A-Fa-f]{0,4}"
+_ipv6_re = re.compile(_ipv6)
+_host_re = re.compile(
+    r"((?:\[" + _ipv6 + r"\])|[a-zA-Z0-9.\-]{1,253})(?::(\d{1,5}))?"
+)
+
+
+def parse_host(host):
+    m = _host_re.fullmatch(host)
+    if not m:
+        return None, None
+    host, port = m.groups()
+    return host.lower(), port and int(port)
+
+
+def bracketv6(addr):
+    return f"[{addr}]" if _ipv6_re.fullmatch(addr) else addr
+
+
+def normalize(fwd: dict) -> dict:
+    """Normalize and convert values extracted from forwarded headers.
+    Modifies fwd in place and returns the same object.
+    """
+    if "proto" in fwd:
+        fwd["proto"] = fwd["proto"].lower()
+    if "by" in fwd:
+        fwd["by"] = bracketv6(fwd["by"]).lower()
+    if "for" in fwd:
+        fwd["for"] = bracketv6(fwd["for"]).lower()
+    if "port" in fwd:
+        try:
+            fwd["port"] = int(fwd["port"])
+        except ValueError:
+            fwd.pop("port", None)
+    if "host" in fwd:
+        host, port = parse_host(fwd["host"])
+        if host:
+            fwd["host"] = host
+            if port:
+                fwd["port"] = port
+        else:
+            del fwd["host"]
+    return fwd

sanic/request.py

@@ -11,7 +11,12 @@
 from httptools import parse_url
 
 from sanic.exceptions import InvalidUsage
-from sanic.headers import parse_content_header
+from sanic.headers import (
+    parse_content_header,
+    parse_forwarded,
+    parse_host,
+    parse_xforwarded,
+)
 from sanic.log import error_logger, logger
 
 
@@ -87,6 +92,7 @@ class Request(dict):
         "parsed_files",
         "parsed_form",
         "parsed_json",
+        "parsed_forwarded",
         "raw_url",
         "stream",
         "transport",
@@ -107,6 +113,7 @@ def __init__(self, url_bytes, headers, version, method, transport, app):
 
         # Init but do not inhale
         self.body_init()
+        self.parsed_forwarded = None
         self.parsed_json = None
         self.parsed_form = None
         self.parsed_files = None
@@ -372,86 +379,70 @@ def _get_address(self):
     def server_name(self):
         """
         Attempt to get the server's hostname in this order:
-        `config.SERVER_NAME`, `x-forwarded-host` header, :func:`Request.host`
+        `config.SERVER_NAME`, `forwarded` header, `x-forwarded-host` header,
+        :func:`Request.host`
 
         :return: the server name without port number
         :rtype: str
         """
-        return (
-            self.app.config.get("SERVER_NAME")
-            or self.headers.get("x-forwarded-host")
-            or self.host.split(":")[0]
-        )
+        server_name = self.app.config.get("SERVER_NAME")
+        if server_name:
+            host = server_name.split("//", 1)[-1].split("/", 1)[0]
+            return parse_host(host)[0]
+        return self.forwarded.get("host") or parse_host(self.host)[0]
+
+    @property
+    def forwarded(self):
+        if self.parsed_forwarded is None:
+            self.parsed_forwarded = (
+                parse_forwarded(self.headers, self.app.config)
+                or parse_xforwarded(self.headers, self.app.config)
+                or {}
+            )
+        return self.parsed_forwarded
 
     @property
     def server_port(self):
         """
         Attempt to get the server's port in this order:
-        `x-forwarded-port` header, :func:`Request.host`, actual port used by
-        the transport layer socket.
+        `forwarded` header, `x-forwarded-port` header, :func:`Request.host`,
+        actual port used by the transport layer socket.
         :return: server port
         :rtype: int
         """
-        forwarded_port = self.headers.get("x-forwarded-port") or (
-            self.host.split(":")[1] if ":" in self.host else None
+        if self.forwarded:
+            return self.forwarded.get("port") or (
+                80 if self.scheme in ("http", "ws") else 443
+            )
+        return (
+            parse_host(self.host)[1]
+            or self.transport.get_extra_info("sockname")[1]
         )
-        if forwarded_port:
-            return int(forwarded_port)
-        else:
-            port = self.transport.get_extra_info("sockname")[1]
-            return port
 
     @property
     def remote_addr(self):
-        """Attempt to return the original client ip based on X-Forwarded-For
-        or X-Real-IP. If HTTP headers are unavailable or untrusted, returns
-        an empty string.
+        """Attempt to return the original client ip based on `forwarded`,
+        `x-forwarded-for` or `x-real-ip`. If HTTP headers are unavailable or
+        untrusted, returns an empty string.
 
         :return: original client ip.
         """
         if not hasattr(self, "_remote_addr"):
-            if self.app.config.PROXIES_COUNT == 0:
-                self._remote_addr = ""
-            elif self.app.config.REAL_IP_HEADER and self.headers.get(
-                self.app.config.REAL_IP_HEADER
-            ):
-                self._remote_addr = self.headers[
-                    self.app.config.REAL_IP_HEADER
-                ]
-            elif self.app.config.FORWARDED_FOR_HEADER:
-                forwarded_for = self.headers.get(
-                    self.app.config.FORWARDED_FOR_HEADER, ""
-                ).split(",")
-                remote_addrs = [
-                    addr
-                    for addr in [addr.strip() for addr in forwarded_for]
-                    if addr
-                ]
-                if self.app.config.PROXIES_COUNT == -1:
-                    self._remote_addr = remote_addrs[0]
-                elif len(remote_addrs) >= self.app.config.PROXIES_COUNT:
-                    self._remote_addr = remote_addrs[
-                        -self.app.config.PROXIES_COUNT
-                    ]
-                else:
-                    self._remote_addr = ""
-            else:
-                self._remote_addr = ""
+            self._remote_addr = self.forwarded.get("for") or ""
         return self._remote_addr
 
     @property
     def scheme(self):
         """
         Attempt to get the request scheme.
         Seeking the value in this order:
-        `x-forwarded-proto` header, `x-scheme` header, the sanic app itself.
+        `forwarded` header, `x-forwarded-proto` header,
+        `x-scheme` header, the sanic app itself.
 
         :return: http|https|ws|wss or arbitrary value given by the headers.
         :rtype: str
         """
-        forwarded_proto = self.headers.get(
-            "x-forwarded-proto"
-        ) or self.headers.get("x-scheme")
+        forwarded_proto = self.forwarded.get("proto")
         if forwarded_proto:
             return forwarded_proto
 
@@ -471,7 +462,7 @@ def scheme(self):
     @property
     def host(self):
         """
-        :return: the Host specified in the header, may contains port number.
+        :return: the Host specified in the header, may contain a port number.
         """
         # it appears that httptools doesn't return the host
         # so pull it from the headers
@@ -514,6 +505,10 @@ def url_for(self, view_name, **kwargs):
         :return: an absolute url to the given view
         :rtype: str
         """
+        # Full URL SERVER_NAME can only be handled in app.url_for
+        if "//" in self.app.config.SERVER_NAME:
+            return self.app.url_for(view_name, _external=True, **kwargs)
+
         scheme = self.scheme
         host = self.server_name
         port = self.server_port