Python 3: Convert some unicode/bytes uses

changelog.d/3569.bugfix

@@ -0,0 +1 @@
+Unicode passwords are now normalised before hashing, preventing the instance where two different devices or browsers might send a different UTF-8 sequence for the password.

synapse/api/auth.py

@@ -251,10 +251,10 @@ def _get_appservice_user_id(self, request):
             if ip_address not in app_service.ip_range_whitelist:
                 defer.returnValue((None, None))
 
-        if "user_id" not in request.args:
+        if b"user_id" not in request.args:
             defer.returnValue((app_service.sender, app_service))
 
-        user_id = request.args["user_id"][0]
+        user_id = request.args[b"user_id"][0].decode('utf8')
         if app_service.sender == user_id:
             defer.returnValue((app_service.sender, app_service))
 

synapse/federation/transport/server.py

@@ -165,7 +165,7 @@ def _parse_auth_header(header_bytes):
         param_dict = dict(kv.split("=") for kv in params)
 
         def strip_quotes(value):
-            if value.startswith(b"\""):
+            if value.startswith("\""):
                 return value[1:-1]
             else:
                 return value

synapse/handlers/auth.py

@@ -15,6 +15,7 @@
 # limitations under the License.
 
 import logging
+import unicodedata
 
 import attr
 import bcrypt
@@ -855,8 +856,16 @@ def hash(self, password):
             Deferred(str): Hashed password.
         """
         def _do_hash():
-            return bcrypt.hashpw(password.encode('utf8') + self.hs.config.password_pepper,
-                                 bcrypt.gensalt(self.bcrypt_rounds))
+            # Ensure that we normalise the password
+            if isinstance(password, bytes):
+                pw = unicodedata.normalize("NFKC", password.decode('utf8'))
+            else:
+                pw = unicodedata.normalize("NFKC", password)
+
+            return bcrypt.hashpw(
+                pw.encode('utf8') + self.hs.config.password_pepper.encode("utf8"),
+                bcrypt.gensalt(self.bcrypt_rounds),
+            )
 
         return make_deferred_yieldable(
             threads.deferToThreadPool(
@@ -876,8 +885,12 @@ def validate_hash(self, password, stored_hash):
         """
 
         def _do_validate_hash():
+            if isinstance(password, bytes):
+                pw = unicodedata.normalize("NFKC", password.decode('utf8'))
+            else:
+                pw = unicodedata.normalize("NFKC", password)
             return bcrypt.checkpw(
-                password.encode('utf8') + self.hs.config.password_pepper,
+                pw.encode('utf8') + self.hs.config.password_pepper.encode("utf8"),
                 stored_hash.encode('utf8')
             )
 

synapse/http/server.py

@@ -13,12 +13,13 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+
 import cgi
 import collections
 import logging
-import urllib
 
-from six.moves import http_client
+from six import PY3
+from six.moves import http_client, urllib
 
 from canonicaljson import encode_canonical_json, encode_pretty_printed_json, json
 
@@ -264,6 +265,7 @@ def __init__(self, hs, canonical_json=True):
         self.hs = hs
 
     def register_paths(self, method, path_patterns, callback):
+        method = method.encode("utf-8")  # method is bytes on py3
         for path_pattern in path_patterns:
             logger.debug("Registering for %s %s", method, path_pattern.pattern)
             self.path_regexs.setdefault(method, []).append(
@@ -296,8 +298,14 @@ def _async_render(self, request):
         # here. If it throws an exception, that is handled by the wrapper
         # installed by @request_handler.
 
+        def _parse(s):
+            if PY3:
+                return urllib.parse.unquote(s)
+            else:
+                return urllib.parse.unquote(s.encode('utf8')).decode('utf8')
+
         kwargs = intern_dict({
-            name: urllib.unquote(value).decode("UTF-8") if value else value
+            name: _parse(value) if value else value
             for name, value in group_dict.items()
         })
 
@@ -327,7 +335,7 @@ def _get_handler_for_request(self, request):
         # Loop through all the registered callbacks to check if the method
         # and path regex match
         for path_entry in self.path_regexs.get(request.method, []):
-            m = path_entry.pattern.match(request.path)
+            m = path_entry.pattern.match(request.path.decode())
             if m:
                 # We found a match!
                 return path_entry.callback, m.groupdict()
@@ -383,7 +391,7 @@ def __init__(self, path):
         self.url = path
 
     def render_GET(self, request):
-        return redirectTo(self.url, request)
+        return redirectTo(self.url.encode(), request)
 
     def getChild(self, name, request):
         if len(name) == 0:
@@ -404,12 +412,14 @@ def respond_with_json(request, code, json_object, send_cors=False,
         return
 
     if pretty_print:
-        json_bytes = encode_pretty_printed_json(json_object) + "\n"
+        json_bytes = (encode_pretty_printed_json(json_object) + "\n"
+                      ).encode("utf-8")
     else:
         if canonical_json or synapse.events.USE_FROZEN_DICTS:
+            # canonicaljson already encodes to bytes
             json_bytes = encode_canonical_json(json_object)
         else:
-            json_bytes = json.dumps(json_object)
+            json_bytes = json.dumps(json_object).encode("utf-8")
 
     return respond_with_json_bytes(
         request, code, json_bytes,

synapse/rest/client/v2_alpha/register.py

@@ -193,15 +193,15 @@ def __init__(self, hs):
     def on_POST(self, request):
         body = parse_json_object_from_request(request)
 
-        kind = "user"
-        if "kind" in request.args:
-            kind = request.args["kind"][0]
+        kind = b"user"
+        if b"kind" in request.args:
+            kind = request.args[b"kind"][0]
 
-        if kind == "guest":
+        if kind == b"guest":
             ret = yield self._do_guest_registration(body)
             defer.returnValue(ret)
             return
-        elif kind != "user":
+        elif kind != b"user":
             raise UnrecognizedRequestError(
                 "Do not understand membership kind: %s" % (kind,)
             )

synapse/rest/media/v1/media_storage.py

@@ -177,7 +177,7 @@ def ensure_media_is_in_local_cache(self, file_info):
             if res:
                 with res:
                     consumer = BackgroundFileConsumer(
-                        open(local_path, "w"), self.hs.get_reactor())
+                        open(local_path, "wb"), self.hs.get_reactor())
                     yield res.write_to_consumer(consumer)
                     yield consumer.wait()
                 defer.returnValue(local_path)

synapse/state.py

@@ -530,7 +530,7 @@ def resolve_state_groups(
 
 def _ordered_events(events):
     def key_func(e):
-        return -int(e.depth), hashlib.sha1(e.event_id.encode()).hexdigest()
+        return -int(e.depth), hashlib.sha1(e.event_id.encode('ascii')).hexdigest()
 
     return sorted(events, key=key_func)
 

synapse/storage/events.py

@@ -19,7 +19,7 @@
 from collections import OrderedDict, deque, namedtuple
 from functools import wraps
 
-from six import iteritems, itervalues
+from six import PY3, iteritems, itervalues
 from six.moves import range
 
 from canonicaljson import json
@@ -65,7 +65,10 @@
 
 
 def encode_json(json_object):
-    return frozendict_json_encoder.encode(json_object)
+    if PY3:
+        return frozendict_json_encoder.encode(json_object)
+    else:
+        return frozendict_json_encoder.encode(json_object).decode("utf-8")
 
 
 class _EventPeristenceQueue(object):
@@ -981,7 +984,7 @@ def _update_outliers_txn(self, txn, events_and_contexts):
 
                 metadata_json = encode_json(
                     event.internal_metadata.get_dict()
-                ).decode("UTF-8")
+                )
 
                 sql = (
                     "UPDATE event_json SET internal_metadata = ?"
@@ -1095,8 +1098,8 @@ def event_dict(event):
                     "room_id": event.room_id,
                     "internal_metadata": encode_json(
                         event.internal_metadata.get_dict()
-                    ).decode("UTF-8"),
-                    "json": encode_json(event_dict(event)).decode("UTF-8"),
+                    ),
+                    "json": encode_json(event_dict(event)),
                 }
                 for event, _ in events_and_contexts
             ],
@@ -1115,7 +1118,7 @@ def event_dict(event):
                     "type": event.type,
                     "processed": True,
                     "outlier": event.internal_metadata.is_outlier(),
-                    "content": encode_json(event.content).decode("UTF-8"),
+                    "content": encode_json(event.content),
                     "origin_server_ts": int(event.origin_server_ts),
                     "received_ts": self._clock.time_msec(),
                     "sender": event.sender,

synapse/storage/signatures.py

@@ -82,7 +82,16 @@ def _get_event_reference_hashes_txn(self, txn, event_id):
             " WHERE event_id = ?"
         )
         txn.execute(query, (event_id, ))
-        return {k: v for k, v in txn}
+        if six.PY2:
+            return {k: v for k, v in txn}
+        else:
+            done = {}
+            for k, v in txn:
+                if not isinstance(v, bytes):
+                    done[k] = v.encode('ascii')
+                else:
+                    done[k] = v
+            return done
 
 
 class SignatureStore(SignatureWorkerStore):

synapse/types.py

@@ -137,7 +137,7 @@ def __deepcopy__(self, memo):
     @classmethod
     def from_string(cls, s):
         """Parse the string given by 's' into a structure object."""
-        if len(s) < 1 or s[0] != cls.SIGIL:
+        if len(s) < 1 or s[0:1] != cls.SIGIL:
             raise SynapseError(400, "Expected %s string to start with '%s'" % (
                 cls.__name__, cls.SIGIL,
             ))

synapse/util/frozenutils.py

@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from six import string_types
+from six import binary_type, text_type
 
 from canonicaljson import json
 from frozendict import frozendict
@@ -26,7 +26,7 @@ def freeze(o):
     if isinstance(o, frozendict):
         return o
 
-    if isinstance(o, string_types):
+    if isinstance(o, (binary_type, text_type)):
         return o
 
     try:
@@ -41,7 +41,7 @@ def unfreeze(o):
     if isinstance(o, (dict, frozendict)):
         return dict({k: unfreeze(v) for k, v in o.items()})
 
-    if isinstance(o, string_types):
+    if isinstance(o, (binary_type, text_type)):
         return o
 
     try: