From 0eab06b9844fb8b41b292f5c3b5fe3f6b930c878 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 31 Aug 2018 00:22:42 +1000 Subject: [PATCH 01/21] port over the http changes --- synapse/appservice/api.py | 13 +++--- synapse/http/client.py | 61 +++++++++++++++---------- synapse/http/matrixfederationclient.py | 63 +++++++++++++++----------- synapse/http/site.py | 4 +- synapse/python_dependencies.py | 1 + 5 files changed, 82 insertions(+), 60 deletions(-) diff --git a/synapse/appservice/api.py b/synapse/appservice/api.py index 6980e5890ec4..9ccc5a80fc59 100644 --- a/synapse/appservice/api.py +++ b/synapse/appservice/api.py @@ -13,7 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. import logging -import urllib + +from six.moves import urllib from prometheus_client import Counter @@ -98,7 +99,7 @@ def __init__(self, hs): def query_user(self, service, user_id): if service.url is None: defer.returnValue(False) - uri = service.url + ("/users/%s" % urllib.quote(user_id)) + uri = service.url + ("/users/%s" % urllib.parse.quote(user_id)) response = None try: response = yield self.get_json(uri, { @@ -119,7 +120,7 @@ def query_user(self, service, user_id): def query_alias(self, service, alias): if service.url is None: defer.returnValue(False) - uri = service.url + ("/rooms/%s" % urllib.quote(alias)) + uri = service.url + ("/rooms/%s" % urllib.parse.quote(alias)) response = None try: response = yield self.get_json(uri, { @@ -153,7 +154,7 @@ def query_3pe(self, service, kind, protocol, fields): service.url, APP_SERVICE_PREFIX, kind, - urllib.quote(protocol) + urllib.parse.quote(protocol) ) try: response = yield self.get_json(uri, fields) @@ -188,7 +189,7 @@ def _get(): uri = "%s%s/thirdparty/protocol/%s" % ( service.url, APP_SERVICE_PREFIX, - urllib.quote(protocol) + urllib.parse.quote(protocol) ) try: info = yield self.get_json(uri, {}) @@ -228,7 +229,7 @@ def push_bulk(self, service, events, txn_id=None): txn_id = str(txn_id) uri = service.url + ("/transactions/%s" % - urllib.quote(txn_id)) + urllib.parse.quote(txn_id)) try: yield self.put_json( uri=uri, diff --git a/synapse/http/client.py b/synapse/http/client.py index ab4fbf59b28d..e60496c17865 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -13,11 +13,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + import logging -import urllib -from six import StringIO +from six import text_type +from six.moves import urllib +import treq from canonicaljson import encode_canonical_json, json from prometheus_client import Counter @@ -83,6 +85,8 @@ def __init__(self, hs): if hs.config.user_agent_suffix: self.user_agent = "%s %s" % (self.user_agent, hs.config.user_agent_suffix,) + self.user_agent = self.user_agent.encode('ascii') + @defer.inlineCallbacks def request(self, method, uri, *args, **kwargs): # A small wrapper around self.agent.request() so we can easily attach @@ -90,11 +94,18 @@ def request(self, method, uri, *args, **kwargs): outgoing_requests_counter.labels(method).inc() # log request but strip `access_token` (AS requests for example include this) - logger.info("Sending request %s %s", method, redact_uri(uri)) + logger.info("Sending request %s %s", method, redact_uri(uri.encode('ascii'))) try: - request_deferred = self.agent.request( - method, uri, *args, **kwargs + if "data" in kwargs: + data = kwargs.pop("data") + elif "bodyProducer" in kwargs: + data = kwargs.pop("bodyProducer") + else: + data = None + + request_deferred = treq.request( + method, uri, *args, agent=self.agent, data=data, **kwargs ) add_timeout_to_deferred( request_deferred, 60, self.hs.get_reactor(), @@ -105,14 +116,14 @@ def request(self, method, uri, *args, **kwargs): incoming_responses_counter.labels(method, response.code).inc() logger.info( "Received response to %s %s: %s", - method, redact_uri(uri), response.code + method, redact_uri(uri.encode('ascii')), response.code ) defer.returnValue(response) except Exception as e: incoming_responses_counter.labels(method, "ERR").inc() logger.info( "Error sending request to %s %s: %s %s", - method, redact_uri(uri), type(e).__name__, e.message + method, redact_uri(uri.encode('ascii')), type(e).__name__, e.args[0] ) raise @@ -137,7 +148,8 @@ def post_urlencoded_get_json(self, uri, args={}, headers=None): # TODO: Do we ever want to log message contents? logger.debug("post_urlencoded_get_json args: %s", args) - query_bytes = urllib.urlencode(encode_urlencode_args(args), True) + query_bytes = urllib.parse.urlencode( + encode_urlencode_args(args), True).encode("utf8") actual_headers = { b"Content-Type": [b"application/x-www-form-urlencoded"], @@ -148,15 +160,14 @@ def post_urlencoded_get_json(self, uri, args={}, headers=None): response = yield self.request( "POST", - uri.encode("ascii"), + uri, headers=Headers(actual_headers), - bodyProducer=FileBodyProducer(StringIO(query_bytes)) + bodyProducer=query_bytes ) - body = yield make_deferred_yieldable(readBody(response)) - if 200 <= response.code < 300: - defer.returnValue(json.loads(body)) + body = yield make_deferred_yieldable(treq.json_content(response)) + defer.returnValue(body) else: raise HttpResponseException(response.code, response.phrase, body) @@ -191,9 +202,9 @@ def post_json_get_json(self, uri, post_json, headers=None): response = yield self.request( "POST", - uri.encode("ascii"), + uri, headers=Headers(actual_headers), - bodyProducer=FileBodyProducer(StringIO(json_str)) + data=json_str ) body = yield make_deferred_yieldable(readBody(response)) @@ -248,7 +259,7 @@ def put_json(self, uri, json_body, args={}, headers=None): ValueError: if the response was not JSON """ if len(args): - query_bytes = urllib.urlencode(args, True) + query_bytes = urllib.parse.urlencode(args, True) uri = "%s?%s" % (uri, query_bytes) json_str = encode_canonical_json(json_body) @@ -262,9 +273,9 @@ def put_json(self, uri, json_body, args={}, headers=None): response = yield self.request( "PUT", - uri.encode("ascii"), + uri, headers=Headers(actual_headers), - bodyProducer=FileBodyProducer(StringIO(json_str)) + data=json_str ) body = yield make_deferred_yieldable(readBody(response)) @@ -293,7 +304,7 @@ def get_raw(self, uri, args={}, headers=None): HttpResponseException on a non-2xx HTTP response. """ if len(args): - query_bytes = urllib.urlencode(args, True) + query_bytes = urllib.parse.urlencode(args, True) uri = "%s?%s" % (uri, query_bytes) actual_headers = { @@ -304,7 +315,7 @@ def get_raw(self, uri, args={}, headers=None): response = yield self.request( "GET", - uri.encode("ascii"), + uri, headers=Headers(actual_headers), ) @@ -339,7 +350,7 @@ def get_file(self, url, output_stream, max_size=None, headers=None): response = yield self.request( "GET", - url.encode("ascii"), + url, headers=Headers(actual_headers), ) @@ -434,12 +445,12 @@ class CaptchaServerHttpClient(SimpleHttpClient): @defer.inlineCallbacks def post_urlencoded_get_raw(self, url, args={}): - query_bytes = urllib.urlencode(encode_urlencode_args(args), True) + query_bytes = urllib.parse.urlencode(encode_urlencode_args(args), True) response = yield self.request( "POST", - url.encode("ascii"), - bodyProducer=FileBodyProducer(StringIO(query_bytes)), + url, + data=query_bytes, headers=Headers({ b"Content-Type": [b"application/x-www-form-urlencoded"], b"User-Agent": [self.user_agent], @@ -510,7 +521,7 @@ def encode_urlencode_args(args): def encode_urlencode_arg(arg): - if isinstance(arg, unicode): + if isinstance(arg, text_type): return arg.encode('utf-8') elif isinstance(arg, list): return [encode_urlencode_arg(i) for i in arg] diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index b34bb8e31ab4..65302a859294 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -17,20 +17,22 @@ import logging import random import sys -import urllib -from six import string_types -from six.moves.urllib import parse as urlparse +from six import PY3, string_types +from six.moves import urllib +import treq from canonicaljson import encode_canonical_json, json from prometheus_client import Counter from signedjson.sign import sign_json +from zope.interface import implementer from twisted.internet import defer, protocol, reactor from twisted.internet.error import DNSLookupError from twisted.web._newclient import ResponseDone from twisted.web.client import Agent, HTTPConnectionPool, readBody from twisted.web.http_headers import Headers +from twisted.web.iweb import IBodyProducer import synapse.metrics import synapse.util.retryutils @@ -58,13 +60,18 @@ MAX_LONG_RETRIES = 10 MAX_SHORT_RETRIES = 3 +if PY3: + MAXINT = sys.maxsize +else: + MAXINT = sys.maxint + class MatrixFederationEndpointFactory(object): def __init__(self, hs): self.tls_client_options_factory = hs.tls_client_options_factory def endpointForURI(self, uri): - destination = uri.netloc + destination = uri.netloc.decode('ascii') return matrix_federation_endpoint( reactor, destination, timeout=10, @@ -93,12 +100,12 @@ def __init__(self, hs): ) self.clock = hs.get_clock() self._store = hs.get_datastore() - self.version_string = hs.version_string + self.version_string = hs.version_string.encode('ascii') self._next_id = 1 def _create_url(self, destination, path_bytes, param_bytes, query_bytes): - return urlparse.urlunparse( - ("matrix", destination, path_bytes, param_bytes, query_bytes, "") + return urllib.parse.urlunparse( + (b"matrix", destination, path_bytes, param_bytes, query_bytes, b"") ) @defer.inlineCallbacks @@ -152,16 +159,16 @@ def _request(self, destination, method, path, headers_dict[b"User-Agent"] = [self.version_string] headers_dict[b"Host"] = [destination] - url_bytes = self._create_url( + url = self._create_url( destination, path_bytes, param_bytes, query_bytes - ) + ).decode('ascii') txn_id = "%s-O-%s" % (method, self._next_id) - self._next_id = (self._next_id + 1) % (sys.maxint - 1) + self._next_id = (self._next_id + 1) % (MAXINT - 1) outbound_logger.info( "{%s} [%s] Sending request: %s %s", - txn_id, destination, method, url_bytes + txn_id, destination, method, url ) # XXX: Would be much nicer to retry only at the transaction-layer @@ -171,23 +178,24 @@ def _request(self, destination, method, path, else: retries_left = MAX_SHORT_RETRIES - http_url_bytes = urlparse.urlunparse( - ("", "", path_bytes, param_bytes, query_bytes, "") - ) + http_url = urllib.parse.urlunparse( + (b"", b"", path_bytes, param_bytes, query_bytes, b"") + ).decode('ascii') log_result = None try: while True: producer = None if body_callback: - producer = body_callback(method, http_url_bytes, headers_dict) + producer = body_callback(method, http_url, headers_dict) try: - request_deferred = self.agent.request( + request_deferred = treq.request( method, - url_bytes, - Headers(headers_dict), - producer + url, + headers=Headers(headers_dict), + data=producer, + agent=self.agent, ) add_timeout_to_deferred( request_deferred, @@ -218,7 +226,7 @@ def _request(self, destination, method, path, txn_id, destination, method, - url_bytes, + url, _flatten_response_never_received(e), ) @@ -297,11 +305,11 @@ def sign_request(self, destination, method, url_bytes, headers_dict, auth_headers = [] for key, sig in request["signatures"][self.server_name].items(): - auth_headers.append(bytes( + auth_headers.append(( "X-Matrix origin=%s,key=\"%s\",sig=\"%s\"" % ( self.server_name, key, sig, - ) - )) + )).encode('ascii') + ) headers_dict[b"Authorization"] = auth_headers @@ -576,7 +584,7 @@ def get_file(self, destination, path, output_stream, args={}, vs = [vs] encoded_args[k] = [v.encode("UTF-8") for v in vs] - query_bytes = urllib.urlencode(encoded_args, True) + query_bytes = urllib.parse.urlencode(encoded_args, True).encode('utf8') logger.debug("Query bytes: %s Retry DNS: %s", query_bytes, retry_on_dns_fail) def body_callback(method, url_bytes, headers_dict): @@ -639,6 +647,7 @@ def _readBodyToFile(response, stream, max_size): return d +@implementer(IBodyProducer) class _JsonProducer(object): """ Used by the twisted http client to create the HTTP body from json """ @@ -693,7 +702,7 @@ def check_content_type_is_json(headers): "No Content-Type header" ) - c_type = c_type[0] # only the first header + c_type = c_type[0].decode('ascii') # only the first header val, options = cgi.parse_header(c_type) if val != "application/json": raise RuntimeError( @@ -711,6 +720,6 @@ def encode_query_args(args): vs = [vs] encoded_args[k] = [v.encode("UTF-8") for v in vs] - query_bytes = urllib.urlencode(encoded_args, True) + query_bytes = urllib.parse.urlencode(encoded_args, True) - return query_bytes + return query_bytes.encode('utf8') diff --git a/synapse/http/site.py b/synapse/http/site.py index 88ed3714f9be..f0828c654243 100644 --- a/synapse/http/site.py +++ b/synapse/http/site.py @@ -204,14 +204,14 @@ def _started_processing(self, servlet_name): self.start_time = time.time() self.request_metrics = RequestMetrics() self.request_metrics.start( - self.start_time, name=servlet_name, method=self.method, + self.start_time, name=servlet_name, method=self.method.decode('ascii'), ) self.site.access_logger.info( "%s - %s - Received request: %s %s", self.getClientIP(), self.site.site_tag, - self.method, + self.method.decode('ascii'), self.get_redacted_uri() ) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index 942d7c721f0e..c8bc53842870 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -40,6 +40,7 @@ "pynacl>=1.2.1": ["nacl>=1.2.1", "nacl.bindings"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], "Twisted>=17.1.0": ["twisted>=17.1.0"], + "treq>=17.8.0": ["treq>=17.8.0"], # We use crypto.get_elliptic_curve which is only supported in >=0.15 "pyopenssl>=0.15": ["OpenSSL>=0.15"], From 31e8034e44943d9ea6d76c7c7e8a0d3e7566adb6 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Fri, 31 Aug 2018 00:24:42 +1000 Subject: [PATCH 02/21] changelog --- changelog.d/3771.misc | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog.d/3771.misc diff --git a/changelog.d/3771.misc b/changelog.d/3771.misc new file mode 100644 index 000000000000..47aa34bc04b6 --- /dev/null +++ b/changelog.d/3771.misc @@ -0,0 +1 @@ +http/ is now ported to Python 3. From 58d592e0528189deb1f0829d1b2f313b96635592 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 4 Sep 2018 21:35:02 +1000 Subject: [PATCH 03/21] simplify using treq --- synapse/http/client.py | 39 ++-------------- synapse/http/matrixfederationclient.py | 64 ++++---------------------- 2 files changed, 14 insertions(+), 89 deletions(-) diff --git a/synapse/http/client.py b/synapse/http/client.py index e60496c17865..9d7b7feb8ad3 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -25,14 +25,13 @@ from OpenSSL import SSL from OpenSSL.SSL import VERIFY_NONE -from twisted.internet import defer, protocol, reactor, ssl, task +from twisted.internet import defer, protocol, reactor, ssl from twisted.internet.endpoints import HostnameEndpoint, wrapClientTLS from twisted.web._newclient import ResponseDone from twisted.web.client import ( Agent, BrowserLikeRedirectAgent, ContentDecoderAgent, - FileBodyProducer as TwistedFileBodyProducer, GzipDecoder, HTTPConnectionPool, PartialDownloadError, @@ -88,7 +87,7 @@ def __init__(self, hs): self.user_agent = self.user_agent.encode('ascii') @defer.inlineCallbacks - def request(self, method, uri, *args, **kwargs): + def request(self, method, uri, data=b''): # A small wrapper around self.agent.request() so we can easily attach # counters to it outgoing_requests_counter.labels(method).inc() @@ -97,15 +96,8 @@ def request(self, method, uri, *args, **kwargs): logger.info("Sending request %s %s", method, redact_uri(uri.encode('ascii'))) try: - if "data" in kwargs: - data = kwargs.pop("data") - elif "bodyProducer" in kwargs: - data = kwargs.pop("bodyProducer") - else: - data = None - request_deferred = treq.request( - method, uri, *args, agent=self.agent, data=data, **kwargs + method, uri, agent=self.agent, data=data ) add_timeout_to_deferred( request_deferred, 60, self.hs.get_reactor(), @@ -162,7 +154,7 @@ def post_urlencoded_get_json(self, uri, args={}, headers=None): "POST", uri, headers=Headers(actual_headers), - bodyProducer=query_bytes + data=query_bytes ) if 200 <= response.code < 300: @@ -553,26 +545,3 @@ def getContext(self, hostname=None, port=None): def creatorForNetloc(self, hostname, port): return self - - -class FileBodyProducer(TwistedFileBodyProducer): - """Workaround for https://twistedmatrix.com/trac/ticket/8473 - - We override the pauseProducing and resumeProducing methods in twisted's - FileBodyProducer so that they do not raise exceptions if the task has - already completed. - """ - - def pauseProducing(self): - try: - super(FileBodyProducer, self).pauseProducing() - except task.TaskDone: - # task has already completed - pass - - def resumeProducing(self): - try: - super(FileBodyProducer, self).resumeProducing() - except task.NotPaused: - # task was not paused (probably because it had already completed) - pass diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 65302a859294..5d9aba253935 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -25,14 +25,12 @@ from canonicaljson import encode_canonical_json, json from prometheus_client import Counter from signedjson.sign import sign_json -from zope.interface import implementer from twisted.internet import defer, protocol, reactor from twisted.internet.error import DNSLookupError from twisted.web._newclient import ResponseDone from twisted.web.client import Agent, HTTPConnectionPool, readBody from twisted.web.http_headers import Headers -from twisted.web.iweb import IBodyProducer import synapse.metrics import synapse.util.retryutils @@ -110,7 +108,7 @@ def _create_url(self, destination, path_bytes, param_bytes, query_bytes): @defer.inlineCallbacks def _request(self, destination, method, path, - body_callback, headers_dict={}, param_bytes=b"", + data=None, headers_dict={}, param_bytes=b"", query_bytes=b"", retry_on_dns_fail=True, timeout=None, long_retries=False, ignore_backoff=False, @@ -185,16 +183,14 @@ def _request(self, destination, method, path, log_result = None try: while True: - producer = None - if body_callback: - producer = body_callback(method, http_url, headers_dict) + self.sign_request(destination, method, http_url, headers_dict) try: request_deferred = treq.request( method, url, headers=Headers(headers_dict), - data=producer, + data=data, agent=self.agent, ) add_timeout_to_deferred( @@ -355,22 +351,17 @@ def put_json(self, destination, path, args={}, data={}, """ if not json_data_callback: - def json_data_callback(): - return data - - def body_callback(method, url_bytes, headers_dict): + json_data = data + else: json_data = json_data_callback() - self.sign_request( - destination, method, url_bytes, headers_dict, json_data - ) - producer = _JsonProducer(json_data) - return producer + + encoded_data = encode_canonical_json(json_data) response = yield self._request( destination, "PUT", path, - body_callback=body_callback, + data=encoded_data, headers_dict={"Content-Type": ["application/json"]}, query_bytes=encode_query_args(args), long_retries=long_retries, @@ -419,18 +410,14 @@ def post_json(self, destination, path, data={}, long_retries=False, is not on our federation whitelist """ - def body_callback(method, url_bytes, headers_dict): - self.sign_request( - destination, method, url_bytes, headers_dict, data - ) - return _JsonProducer(data) + encoded_data = encode_canonical_json(data) response = yield self._request( destination, "POST", path, query_bytes=encode_query_args(args), - body_callback=body_callback, + data=encoded_data, headers_dict={"Content-Type": ["application/json"]}, long_retries=long_retries, timeout=timeout, @@ -479,16 +466,11 @@ def get_json(self, destination, path, args=None, retry_on_dns_fail=True, logger.debug("Query bytes: %s Retry DNS: %s", args, retry_on_dns_fail) - def body_callback(method, url_bytes, headers_dict): - self.sign_request(destination, method, url_bytes, headers_dict) - return None - response = yield self._request( destination, "GET", path, query_bytes=encode_query_args(args), - body_callback=body_callback, retry_on_dns_fail=retry_on_dns_fail, timeout=timeout, ignore_backoff=ignore_backoff, @@ -531,7 +513,6 @@ def delete_json(self, destination, path, long_retries=False, Fails with ``FederationDeniedError`` if this destination is not on our federation whitelist """ - response = yield self._request( destination, "DELETE", @@ -647,31 +628,6 @@ def _readBodyToFile(response, stream, max_size): return d -@implementer(IBodyProducer) -class _JsonProducer(object): - """ Used by the twisted http client to create the HTTP body from json - """ - def __init__(self, jsn): - self.reset(jsn) - - def reset(self, jsn): - self.body = encode_canonical_json(jsn) - self.length = len(self.body) - - def startProducing(self, consumer): - consumer.write(self.body) - return defer.succeed(None) - - def pauseProducing(self): - pass - - def stopProducing(self): - pass - - def resumeProducing(self): - pass - - def _flatten_response_never_received(e): if hasattr(e, "reasons"): reasons = ", ".join( From 4a8ce735c4577088dcb09902d8fbe899a7fae452 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 4 Sep 2018 21:35:14 +1000 Subject: [PATCH 04/21] 15.1 will probably do --- synapse/python_dependencies.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/python_dependencies.py b/synapse/python_dependencies.py index c8bc53842870..6dd517932078 100644 --- a/synapse/python_dependencies.py +++ b/synapse/python_dependencies.py @@ -40,7 +40,7 @@ "pynacl>=1.2.1": ["nacl>=1.2.1", "nacl.bindings"], "service_identity>=1.0.0": ["service_identity>=1.0.0"], "Twisted>=17.1.0": ["twisted>=17.1.0"], - "treq>=17.8.0": ["treq>=17.8.0"], + "treq>=15.1": ["treq>=15.1"], # We use crypto.get_elliptic_curve which is only supported in >=0.15 "pyopenssl>=0.15": ["OpenSSL>=0.15"], From ce38311eb2031c3da9ed7a7628c4164c9749b0dc Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 4 Sep 2018 21:46:11 +1000 Subject: [PATCH 05/21] fix headers --- synapse/http/client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/http/client.py b/synapse/http/client.py index 9d7b7feb8ad3..4ba54fed05ef 100644 --- a/synapse/http/client.py +++ b/synapse/http/client.py @@ -87,7 +87,7 @@ def __init__(self, hs): self.user_agent = self.user_agent.encode('ascii') @defer.inlineCallbacks - def request(self, method, uri, data=b''): + def request(self, method, uri, data=b'', headers=None): # A small wrapper around self.agent.request() so we can easily attach # counters to it outgoing_requests_counter.labels(method).inc() @@ -97,7 +97,7 @@ def request(self, method, uri, data=b''): try: request_deferred = treq.request( - method, uri, agent=self.agent, data=data + method, uri, agent=self.agent, data=data, headers=headers ) add_timeout_to_deferred( request_deferred, 60, self.hs.get_reactor(), From 8acf7ff93ba30008bc152e9a4d95f5584e5d8867 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 4 Sep 2018 22:05:04 +1000 Subject: [PATCH 06/21] fix mfc --- synapse/http/matrixfederationclient.py | 43 +++++++++++--------------- 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 5d9aba253935..fef20ed93f67 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -29,7 +29,7 @@ from twisted.internet import defer, protocol, reactor from twisted.internet.error import DNSLookupError from twisted.web._newclient import ResponseDone -from twisted.web.client import Agent, HTTPConnectionPool, readBody +from twisted.web.client import Agent, HTTPConnectionPool from twisted.web.http_headers import Headers import synapse.metrics @@ -108,7 +108,7 @@ def _create_url(self, destination, path_bytes, param_bytes, query_bytes): @defer.inlineCallbacks def _request(self, destination, method, path, - data=None, headers_dict={}, param_bytes=b"", + data=None, json=None, headers_dict={}, param_bytes=b"", query_bytes=b"", retry_on_dns_fail=True, timeout=None, long_retries=False, ignore_backoff=False, @@ -182,9 +182,12 @@ def _request(self, destination, method, path, log_result = None try: - while True: - self.sign_request(destination, method, http_url, headers_dict) + if json: + data = encode_canonical_json(json) + + self.sign_request(destination, method, http_url, headers_dict) + while True: try: request_deferred = treq.request( method, @@ -256,7 +259,7 @@ def _request(self, destination, method, path, # :'( # Update transactions table? with logcontext.PreserveLoggingContext(): - body = yield readBody(response) + body = yield treq.content(response) raise HttpResponseException( response.code, response.phrase, body ) @@ -355,13 +358,11 @@ def put_json(self, destination, path, args={}, data={}, else: json_data = json_data_callback() - encoded_data = encode_canonical_json(json_data) - response = yield self._request( destination, "PUT", path, - data=encoded_data, + json=json_data, headers_dict={"Content-Type": ["application/json"]}, query_bytes=encode_query_args(args), long_retries=long_retries, @@ -375,8 +376,8 @@ def put_json(self, destination, path, args={}, data={}, check_content_type_is_json(response.headers) with logcontext.PreserveLoggingContext(): - body = yield readBody(response) - defer.returnValue(json.loads(body)) + body = yield treq.json_content(response) + defer.returnValue(body) @defer.inlineCallbacks def post_json(self, destination, path, data={}, long_retries=False, @@ -409,15 +410,12 @@ def post_json(self, destination, path, data={}, long_retries=False, Fails with ``FederationDeniedError`` if this destination is not on our federation whitelist """ - - encoded_data = encode_canonical_json(data) - response = yield self._request( destination, "POST", path, query_bytes=encode_query_args(args), - data=encoded_data, + json=data, headers_dict={"Content-Type": ["application/json"]}, long_retries=long_retries, timeout=timeout, @@ -429,9 +427,9 @@ def post_json(self, destination, path, data={}, long_retries=False, check_content_type_is_json(response.headers) with logcontext.PreserveLoggingContext(): - body = yield readBody(response) + body = yield treq.json_content(response) - defer.returnValue(json.loads(body)) + defer.returnValue(body) @defer.inlineCallbacks def get_json(self, destination, path, args=None, retry_on_dns_fail=True, @@ -481,9 +479,9 @@ def get_json(self, destination, path, args=None, retry_on_dns_fail=True, check_content_type_is_json(response.headers) with logcontext.PreserveLoggingContext(): - body = yield readBody(response) + body = yield treq.json_content(response) - defer.returnValue(json.loads(body)) + defer.returnValue(body) @defer.inlineCallbacks def delete_json(self, destination, path, long_retries=False, @@ -529,9 +527,9 @@ def delete_json(self, destination, path, long_retries=False, check_content_type_is_json(response.headers) with logcontext.PreserveLoggingContext(): - body = yield readBody(response) + body = yield treq.json_content(response) - defer.returnValue(json.loads(body)) + defer.returnValue(body) @defer.inlineCallbacks def get_file(self, destination, path, output_stream, args={}, @@ -568,16 +566,11 @@ def get_file(self, destination, path, output_stream, args={}, query_bytes = urllib.parse.urlencode(encoded_args, True).encode('utf8') logger.debug("Query bytes: %s Retry DNS: %s", query_bytes, retry_on_dns_fail) - def body_callback(method, url_bytes, headers_dict): - self.sign_request(destination, method, url_bytes, headers_dict) - return None - response = yield self._request( destination, "GET", path, query_bytes=query_bytes, - body_callback=body_callback, retry_on_dns_fail=retry_on_dns_fail, ignore_backoff=ignore_backoff, ) From f556a4abebd0a5a9b9678b92ead2c18ec3dc45b0 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 4 Sep 2018 22:06:31 +1000 Subject: [PATCH 07/21] another fix --- synapse/http/matrixfederationclient.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index fef20ed93f67..9be26ece6dc0 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -151,14 +151,14 @@ def _request(self, destination, method, path, ignore_backoff=ignore_backoff, ) - destination = destination.encode("ascii") + destination_bytes = destination.encode("ascii") path_bytes = path.encode("ascii") with limiter: headers_dict[b"User-Agent"] = [self.version_string] - headers_dict[b"Host"] = [destination] + headers_dict[b"Host"] = [destination_bytes] url = self._create_url( - destination, path_bytes, param_bytes, query_bytes + destination_bytes, path_bytes, param_bytes, query_bytes ).decode('ascii') txn_id = "%s-O-%s" % (method, self._next_id) From 14d125bb12a4aa35345950dd25a9ea37a20e2b28 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Tue, 4 Sep 2018 22:11:19 +1000 Subject: [PATCH 08/21] try this --- synapse/http/matrixfederationclient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 9be26ece6dc0..499e1fb4bed0 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -185,7 +185,7 @@ def _request(self, destination, method, path, if json: data = encode_canonical_json(json) - self.sign_request(destination, method, http_url, headers_dict) + self.sign_request(destination, method, http_url_bytes, headers_dict) while True: try: From 29604335afa093557f26cf28138f9d94ded68554 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 5 Sep 2018 00:50:23 +1000 Subject: [PATCH 09/21] fixes --- synapse/http/matrixfederationclient.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 499e1fb4bed0..d91dc184aa38 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -176,16 +176,18 @@ def _request(self, destination, method, path, else: retries_left = MAX_SHORT_RETRIES - http_url = urllib.parse.urlunparse( + http_url_bytes = urllib.parse.urlunparse( (b"", b"", path_bytes, param_bytes, query_bytes, b"") - ).decode('ascii') + ) + http_url = http_url_bytes.decode('ascii') log_result = None try: if json: data = encode_canonical_json(json) - - self.sign_request(destination, method, http_url_bytes, headers_dict) + self.sign_request(destination, method, http_url_bytes, headers_dict, json) + else: + self.sign_request(destination, method, http_url_bytes, headers_dict) while True: try: From 38b3f63e79f65176faa715255d85a60ba599316b Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 5 Sep 2018 01:00:10 +1000 Subject: [PATCH 10/21] cleanup --- synapse/http/matrixfederationclient.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index d91dc184aa38..a6a41ab26de6 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -22,7 +22,7 @@ from six.moves import urllib import treq -from canonicaljson import encode_canonical_json, json +from canonicaljson import encode_canonical_json from prometheus_client import Counter from signedjson.sign import sign_json @@ -176,18 +176,17 @@ def _request(self, destination, method, path, else: retries_left = MAX_SHORT_RETRIES - http_url_bytes = urllib.parse.urlunparse( + http_url = urllib.parse.urlunparse( (b"", b"", path_bytes, param_bytes, query_bytes, b"") - ) - http_url = http_url_bytes.decode('ascii') + ).decode('ascii') log_result = None try: if json: data = encode_canonical_json(json) - self.sign_request(destination, method, http_url_bytes, headers_dict, json) + self.sign_request(destination, method, http_url, headers_dict, json) else: - self.sign_request(destination, method, http_url_bytes, headers_dict) + self.sign_request(destination, method, http_url, headers_dict) while True: try: From 092c3b7406dbeb9bee0ae3de7c40173400a07ac4 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 5 Sep 2018 02:51:32 +1000 Subject: [PATCH 11/21] some cleanups --- synapse/http/matrixfederationclient.py | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index a6a41ab26de6..e7663712dff9 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -108,7 +108,7 @@ def _create_url(self, destination, path_bytes, param_bytes, query_bytes): @defer.inlineCallbacks def _request(self, destination, method, path, - data=None, json=None, headers_dict={}, param_bytes=b"", + data=None, json=None, json_data_callback=None, headers_dict={}, param_bytes=b"", query_bytes=b"", retry_on_dns_fail=True, timeout=None, long_retries=False, ignore_backoff=False, @@ -182,14 +182,17 @@ def _request(self, destination, method, path, log_result = None try: - if json: - data = encode_canonical_json(json) - self.sign_request(destination, method, http_url, headers_dict, json) - else: - self.sign_request(destination, method, http_url, headers_dict) - while True: try: + if json_callback: + data = encode_canonical_json(json_data_callback()) + self.sign_request(destination, method, http_url, headers_dict, json) + elif json: + data = encode_canonical_json(json) + self.sign_request(destination, method, http_url, headers_dict, json) + else: + self.sign_request(destination, method, http_url, headers_dict) + request_deferred = treq.request( method, url, @@ -355,15 +358,13 @@ def put_json(self, destination, path, args={}, data={}, """ if not json_data_callback: - json_data = data - else: - json_data = json_data_callback() + json_data_callback = lambda: data response = yield self._request( destination, "PUT", path, - json=json_data, + json_callback=json_data_callback, headers_dict={"Content-Type": ["application/json"]}, query_bytes=encode_query_args(args), long_retries=long_retries, From 3f5ac4fe606792da4d4558e7b51d3dadac728d10 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 5 Sep 2018 02:53:56 +1000 Subject: [PATCH 12/21] and fix --- synapse/http/matrixfederationclient.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index e7663712dff9..74852594ed07 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -108,7 +108,8 @@ def _create_url(self, destination, path_bytes, param_bytes, query_bytes): @defer.inlineCallbacks def _request(self, destination, method, path, - data=None, json=None, json_data_callback=None, headers_dict={}, param_bytes=b"", + data=None, json=None, json_callback=None, headers_dict={}, + param_bytes=b"", query_bytes=b"", retry_on_dns_fail=True, timeout=None, long_retries=False, ignore_backoff=False, @@ -185,7 +186,7 @@ def _request(self, destination, method, path, while True: try: if json_callback: - data = encode_canonical_json(json_data_callback()) + data = encode_canonical_json(json_callback()) self.sign_request(destination, method, http_url, headers_dict, json) elif json: data = encode_canonical_json(json) From 5bfcd748faacc38112ed4fcda9d9ee2ee8cfb7ed Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 5 Sep 2018 02:57:20 +1000 Subject: [PATCH 13/21] remove unused var --- synapse/http/matrixfederationclient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 74852594ed07..6bd7e6591b69 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -108,7 +108,7 @@ def _create_url(self, destination, path_bytes, param_bytes, query_bytes): @defer.inlineCallbacks def _request(self, destination, method, path, - data=None, json=None, json_callback=None, headers_dict={}, + json=None, json_callback=None, headers_dict={}, param_bytes=b"", query_bytes=b"", retry_on_dns_fail=True, timeout=None, long_retries=False, From d5041dde578babc98bb8ba281c489401752664e7 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 5 Sep 2018 03:41:37 +1000 Subject: [PATCH 14/21] fix --- setup.cfg | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index c2620be6c52c..7c6b43b01917 100644 --- a/setup.cfg +++ b/setup.cfg @@ -17,8 +17,8 @@ ignore = [pep8] max-line-length = 90 # W503 requires that binary operators be at the end, not start, of lines. Erik -# doesn't like it. E203 is contrary to PEP8. -ignore = W503,E203 +# doesn't like it. E203 is contrary to PEP8. E731 is silly. +ignore = W503,E203,E731 [flake8] # note that flake8 inherits the "ignore" settings from "pep8" (because it uses From 3058050e7a38ac042c4aa5ec350dc14f499b2f38 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 5 Sep 2018 03:43:11 +1000 Subject: [PATCH 15/21] fix --- synapse/http/matrixfederationclient.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 6bd7e6591b69..42b688efbbe6 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -186,11 +186,13 @@ def _request(self, destination, method, path, while True: try: if json_callback: - data = encode_canonical_json(json_callback()) - self.sign_request(destination, method, http_url, headers_dict, json) - elif json: + json = json_callback() + + if json: data = encode_canonical_json(json) - self.sign_request(destination, method, http_url, headers_dict, json) + self.sign_request( + destination, method, http_url, headers_dict, json + ) else: self.sign_request(destination, method, http_url, headers_dict) From f371ac40dfc59b6b6b7e1360dcfadc0014fb2ffd Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 5 Sep 2018 04:01:04 +1000 Subject: [PATCH 16/21] fix --- setup.cfg | 1 + synapse/http/matrixfederationclient.py | 1 + 2 files changed, 2 insertions(+) diff --git a/setup.cfg b/setup.cfg index 7c6b43b01917..52feaa9cc7b5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -24,6 +24,7 @@ ignore = W503,E203,E731 # note that flake8 inherits the "ignore" settings from "pep8" (because it uses # pep8 to do those checks), but not the "max-line-length" setting max-line-length = 90 +ignore=W503,E203,E731 [isort] line_length = 89 diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 42b688efbbe6..8828b0dfb20e 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -194,6 +194,7 @@ def _request(self, destination, method, path, destination, method, http_url, headers_dict, json ) else: + data = None self.sign_request(destination, method, http_url, headers_dict) request_deferred = treq.request( From 0d0602baac04c53f1c124218f1216ec94e7cd665 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 5 Sep 2018 04:11:42 +1000 Subject: [PATCH 17/21] fix weird pep8 thing --- synapse/federation/federation_server.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/synapse/federation/federation_server.py b/synapse/federation/federation_server.py index 3e0cd294a1b2..6e52c4b6b558 100644 --- a/synapse/federation/federation_server.py +++ b/synapse/federation/federation_server.py @@ -838,9 +838,9 @@ def on_edu(self, edu_type, origin, content): ) return self._send_edu( - edu_type=edu_type, - origin=origin, - content=content, + edu_type=edu_type, + origin=origin, + content=content, ) def on_query(self, query_type, args): @@ -851,6 +851,6 @@ def on_query(self, query_type, args): return handler(args) return self._get_query_client( - query_type=query_type, - args=args, + query_type=query_type, + args=args, ) From 69cc6303e72e934040d9ae12a77e1a8495c6ff5a Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 5 Sep 2018 22:25:19 +1000 Subject: [PATCH 18/21] cleanup some docstrings --- synapse/http/matrixfederationclient.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 8828b0dfb20e..2cd88014cf08 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -108,17 +108,21 @@ def _create_url(self, destination, path_bytes, param_bytes, query_bytes): @defer.inlineCallbacks def _request(self, destination, method, path, - json=None, json_callback=None, headers_dict={}, + json=None, json_callback=None, param_bytes=b"", query_bytes=b"", retry_on_dns_fail=True, timeout=None, long_retries=False, ignore_backoff=False, backoff_on_404=False): - """ Creates and sends a request to the given server + """ + Creates and sends a request to the given server. + Args: destination (str): The remote server to send the HTTP request to. method (str): HTTP method path (str): The HTTP path + json (dict or None): JSON to send in the body. + json_callback (func or None): A callback to generate the JSON. ignore_backoff (bool): true to ignore the historical backoff data and try the request anyway. backoff_on_404 (bool): Back off if we get a 404 @@ -152,14 +156,17 @@ def _request(self, destination, method, path, ignore_backoff=ignore_backoff, ) - destination_bytes = destination.encode("ascii") + headers_dict = {} path_bytes = path.encode("ascii") - with limiter: - headers_dict[b"User-Agent"] = [self.version_string] - headers_dict[b"Host"] = [destination_bytes] + headers_dict = { + "User-Agent": self.version_string, + "Host": destination, + } + + with limiter: url = self._create_url( - destination_bytes, path_bytes, param_bytes, query_bytes + destination.encode("ascii"), path_bytes, param_bytes, query_bytes ).decode('ascii') txn_id = "%s-O-%s" % (method, self._next_id) @@ -190,6 +197,7 @@ def _request(self, destination, method, path, if json: data = encode_canonical_json(json) + headers_dict["Content-Type"] = ["application/json"] self.sign_request( destination, method, http_url, headers_dict, json ) @@ -369,7 +377,6 @@ def put_json(self, destination, path, args={}, data={}, "PUT", path, json_callback=json_data_callback, - headers_dict={"Content-Type": ["application/json"]}, query_bytes=encode_query_args(args), long_retries=long_retries, timeout=timeout, @@ -422,7 +429,6 @@ def post_json(self, destination, path, data={}, long_retries=False, path, query_bytes=encode_query_args(args), json=data, - headers_dict={"Content-Type": ["application/json"]}, long_retries=long_retries, timeout=timeout, ignore_backoff=ignore_backoff, @@ -522,7 +528,6 @@ def delete_json(self, destination, path, long_retries=False, "DELETE", path, query_bytes=encode_query_args(args), - headers_dict={"Content-Type": ["application/json"]}, long_retries=long_retries, timeout=timeout, ignore_backoff=ignore_backoff, From a86b8640c0e07249b3e119a1acc5940d8a66ed09 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 5 Sep 2018 22:31:51 +1000 Subject: [PATCH 19/21] see if I can clean this up --- synapse/http/matrixfederationclient.py | 27 +++++++++++--------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 2cd88014cf08..859ce362ed3d 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -110,7 +110,7 @@ def _create_url(self, destination, path_bytes, param_bytes, query_bytes): def _request(self, destination, method, path, json=None, json_callback=None, param_bytes=b"", - query_bytes=b"", retry_on_dns_fail=True, + query=None, retry_on_dns_fail=True, timeout=None, long_retries=False, ignore_backoff=False, backoff_on_404=False): @@ -123,6 +123,7 @@ def _request(self, destination, method, path, path (str): The HTTP path json (dict or None): JSON to send in the body. json_callback (func or None): A callback to generate the JSON. + query (dict or None): Query arguments. ignore_backoff (bool): true to ignore the historical backoff data and try the request anyway. backoff_on_404 (bool): Back off if we get a 404 @@ -158,6 +159,10 @@ def _request(self, destination, method, path, headers_dict = {} path_bytes = path.encode("ascii") + if query: + query_bytes = encode_query_args(query) + else: + query_bytes = b"" headers_dict = { "User-Agent": self.version_string, @@ -377,7 +382,7 @@ def put_json(self, destination, path, args={}, data={}, "PUT", path, json_callback=json_data_callback, - query_bytes=encode_query_args(args), + query=args, long_retries=long_retries, timeout=timeout, ignore_backoff=ignore_backoff, @@ -427,7 +432,7 @@ def post_json(self, destination, path, data={}, long_retries=False, destination, "POST", path, - query_bytes=encode_query_args(args), + query=args, json=data, long_retries=long_retries, timeout=timeout, @@ -480,7 +485,7 @@ def get_json(self, destination, path, args=None, retry_on_dns_fail=True, destination, "GET", path, - query_bytes=encode_query_args(args), + query=args, retry_on_dns_fail=retry_on_dns_fail, timeout=timeout, ignore_backoff=ignore_backoff, @@ -527,7 +532,7 @@ def delete_json(self, destination, path, long_retries=False, destination, "DELETE", path, - query_bytes=encode_query_args(args), + query=args, long_retries=long_retries, timeout=timeout, ignore_backoff=ignore_backoff, @@ -567,21 +572,11 @@ def get_file(self, destination, path, output_stream, args={}, Fails with ``FederationDeniedError`` if this destination is not on our federation whitelist """ - - encoded_args = {} - for k, vs in args.items(): - if isinstance(vs, string_types): - vs = [vs] - encoded_args[k] = [v.encode("UTF-8") for v in vs] - - query_bytes = urllib.parse.urlencode(encoded_args, True).encode('utf8') - logger.debug("Query bytes: %s Retry DNS: %s", query_bytes, retry_on_dns_fail) - response = yield self._request( destination, "GET", path, - query_bytes=query_bytes, + query=args, retry_on_dns_fail=retry_on_dns_fail, ignore_backoff=ignore_backoff, ) From f7552bcad9ac2462aed55b0756f79c58c7abd3de Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 5 Sep 2018 22:39:00 +1000 Subject: [PATCH 20/21] fix --- synapse/http/matrixfederationclient.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 859ce362ed3d..3b517f73a444 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -165,8 +165,8 @@ def _request(self, destination, method, path, query_bytes = b"" headers_dict = { - "User-Agent": self.version_string, - "Host": destination, + "User-Agent": [self.version_string], + "Host": [destination], } with limiter: From dea7fb9270443b6cfc3fc16d9b68a73a387cd572 Mon Sep 17 00:00:00 2001 From: Amber Brown Date: Wed, 5 Sep 2018 22:55:11 +1000 Subject: [PATCH 21/21] fix --- synapse/http/matrixfederationclient.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synapse/http/matrixfederationclient.py b/synapse/http/matrixfederationclient.py index 3b517f73a444..6a1fc8ca553f 100644 --- a/synapse/http/matrixfederationclient.py +++ b/synapse/http/matrixfederationclient.py @@ -116,7 +116,7 @@ def _request(self, destination, method, path, backoff_on_404=False): """ Creates and sends a request to the given server. - + Args: destination (str): The remote server to send the HTTP request to. method (str): HTTP method