From f804d24193f720e73b634bcc94827d654bfd2e13 Mon Sep 17 00:00:00 2001 From: Dmitry Erlikh Date: Sat, 19 Dec 2020 17:46:53 +0100 Subject: [PATCH 01/26] Add llhttp submodule --- .gitmodules | 3 +++ vendor/llhttp | 1 + 2 files changed, 4 insertions(+) create mode 160000 vendor/llhttp diff --git a/.gitmodules b/.gitmodules index c58945aa9fc..71f9d2c85b8 100644 --- a/.gitmodules +++ b/.gitmodules @@ -2,3 +2,6 @@ path = vendor/http-parser url = git://github.com/nodejs/http-parser.git branch = 54f55a2 +[submodule "vendor/llhttp"] + path = vendor/llhttp + url = https://github.com/nodejs/llhttp.git diff --git a/vendor/llhttp b/vendor/llhttp new file mode 160000 index 00000000000..c6a35cccf5c --- /dev/null +++ b/vendor/llhttp @@ -0,0 +1 @@ +Subproject commit c6a35cccf5c8b36f82036c23cf9c50a7dc2dbd0a From 86e016c51657e6daf54bb7199f3703320626ef0b Mon Sep 17 00:00:00 2001 From: Dmitry Erlikh Date: Sun, 20 Dec 2020 10:17:33 +0100 Subject: [PATCH 02/26] cython w/o url parsing --- aiohttp/_cparser.pxd | 284 ++++++++++++++++++++++----------------- aiohttp/_http_parser.pyx | 93 ++++++------- setup.py | 1 + 3 files changed, 210 insertions(+), 168 deletions(-) diff --git a/aiohttp/_cparser.pxd b/aiohttp/_cparser.pxd index 0f9fc009236..8b4216c128d 100644 --- a/aiohttp/_cparser.pxd +++ b/aiohttp/_cparser.pxd @@ -1,140 +1,180 @@ -from libc.stdint cimport uint16_t, uint32_t, uint64_t - - -cdef extern from "../vendor/http-parser/http_parser.h": - ctypedef int (*http_data_cb) (http_parser*, - const char *at, - size_t length) except -1 - - ctypedef int (*http_cb) (http_parser*) except -1 - - struct http_parser: - unsigned int type - unsigned int flags - unsigned int state - unsigned int header_state - unsigned int index - - uint32_t nread +from libc.stdint cimport ( + int8_t, + int16_t, + int32_t, + int64_t, + uint8_t, + uint16_t, + uint32_t, + uint64_t, +) + +cdef extern from "../vendor/llhttp/build/llhttp.h": + + ctypedef int (*llhttp_data_cb)(llhttp_t*, const char *at, size_t length) except -1 + ctypedef int (*llhttp_cb)(llhttp_t*) except -1 + + struct llhttp_t: + int32_t _index + void* _span_pos0 + void* _span_cb0 + int32_t error + const char* reason + const char* error_pos + void* data + void* _current uint64_t content_length - - unsigned short http_major - unsigned short http_minor - unsigned int status_code - unsigned int method - unsigned int http_errno - - unsigned int upgrade - - void *data - - struct http_parser_settings: - http_cb on_message_begin - http_data_cb on_url - http_data_cb on_status - http_data_cb on_header_field - http_data_cb on_header_value - http_cb on_headers_complete - http_data_cb on_body - http_cb on_message_complete - http_cb on_chunk_header - http_cb on_chunk_complete - - enum http_parser_type: - HTTP_REQUEST, - HTTP_RESPONSE, - HTTP_BOTH - - enum http_errno: + uint8_t type + uint8_t method + uint8_t http_major + uint8_t http_minor + uint8_t header_state + uint8_t lenient_flags + uint8_t upgrade + uint8_t finish + uint16_t flags + uint16_t status_code + void* settings + + struct llhttp_settings_t: + llhttp_cb on_message_begin + llhttp_data_cb on_url + llhttp_data_cb on_status + llhttp_data_cb on_header_field + llhttp_data_cb on_header_value + llhttp_cb on_headers_complete + llhttp_data_cb on_body + llhttp_cb on_message_complete + llhttp_cb on_chunk_header + llhttp_cb on_chunk_complete + + llhttp_cb on_url_complete + llhttp_cb on_status_complete + llhttp_cb on_header_field_complete + llhttp_cb on_header_value_complete + + enum llhttp_errno_t: HPE_OK, - HPE_CB_message_begin, - HPE_CB_url, - HPE_CB_header_field, - HPE_CB_header_value, - HPE_CB_headers_complete, - HPE_CB_body, - HPE_CB_message_complete, - HPE_CB_status, - HPE_CB_chunk_header, - HPE_CB_chunk_complete, - HPE_INVALID_EOF_STATE, - HPE_HEADER_OVERFLOW, + HPE_INTERNAL, + HPE_STRICT, + HPE_LF_EXPECTED, + HPE_UNEXPECTED_CONTENT_LENGTH, HPE_CLOSED_CONNECTION, - HPE_INVALID_VERSION, - HPE_INVALID_STATUS, HPE_INVALID_METHOD, HPE_INVALID_URL, - HPE_INVALID_HOST, - HPE_INVALID_PORT, - HPE_INVALID_PATH, - HPE_INVALID_QUERY_STRING, - HPE_INVALID_FRAGMENT, - HPE_LF_EXPECTED, + HPE_INVALID_CONSTANT, + HPE_INVALID_VERSION, HPE_INVALID_HEADER_TOKEN, HPE_INVALID_CONTENT_LENGTH, HPE_INVALID_CHUNK_SIZE, - HPE_INVALID_CONSTANT, - HPE_INVALID_INTERNAL_STATE, - HPE_STRICT, + HPE_INVALID_STATUS, + HPE_INVALID_EOF_STATE, + HPE_INVALID_TRANSFER_ENCODING, + HPE_CB_MESSAGE_BEGIN, + HPE_CB_HEADERS_COMPLETE, + HPE_CB_MESSAGE_COMPLETE, + HPE_CB_CHUNK_HEADER, + HPE_CB_CHUNK_COMPLETE, HPE_PAUSED, - HPE_UNKNOWN + HPE_PAUSED_UPGRADE, + HPE_USER - enum flags: - F_CHUNKED, + enum llhttp_flags: F_CONNECTION_KEEP_ALIVE, F_CONNECTION_CLOSE, F_CONNECTION_UPGRADE, - F_TRAILING, + F_CHUNKED, F_UPGRADE, + F_CONTENT_LENGTH, F_SKIPBODY, - F_CONTENTLENGTH - - enum http_method: - DELETE, GET, HEAD, POST, PUT, CONNECT, OPTIONS, TRACE, COPY, - LOCK, MKCOL, MOVE, PROPFIND, PROPPATCH, SEARCH, UNLOCK, BIND, - REBIND, UNBIND, ACL, REPORT, MKACTIVITY, CHECKOUT, MERGE, - MSEARCH, NOTIFY, SUBSCRIBE, UNSUBSCRIBE, PATCH, PURGE, MKCALENDAR, - LINK, UNLINK - - void http_parser_init(http_parser *parser, http_parser_type type) - - size_t http_parser_execute(http_parser *parser, - const http_parser_settings *settings, - const char *data, - size_t len) - - int http_should_keep_alive(const http_parser *parser) - - void http_parser_settings_init(http_parser_settings *settings) - - const char *http_errno_name(http_errno err) - const char *http_errno_description(http_errno err) - const char *http_method_str(http_method m) - - # URL Parser - - enum http_parser_url_fields: - UF_SCHEMA = 0, - UF_HOST = 1, - UF_PORT = 2, - UF_PATH = 3, - UF_QUERY = 4, - UF_FRAGMENT = 5, - UF_USERINFO = 6, - UF_MAX = 7 - - struct http_parser_url_field_data: - uint16_t off - uint16_t len + F_TRAILING, + F_TRANSFER_ENCODING - struct http_parser_url: - uint16_t field_set - uint16_t port - http_parser_url_field_data[UF_MAX] field_data + enum llhttp_lenient_flags: + LENIENT_HEADERS, + LENIENT_CHUNKED_LENGTH - void http_parser_url_init(http_parser_url *u) + enum llhttp_type: + HTTP_REQUEST, + HTTP_RESPONSE, + HTTP_BOTH - int http_parser_parse_url(const char *buf, - size_t buflen, - int is_connect, - http_parser_url *u) + enum llhttp_finish_t: + HTTP_FINISH_SAFE, + HTTP_FINISH_SAFE_WITH_CB, + HTTP_FINISH_UNSAFE + + enum llhttp_method_t: + HTTP_DELETE, + HTTP_GET, + HTTP_HEAD, + HTTP_POST, + HTTP_PUT, + HTTP_CONNECT, + HTTP_OPTIONS, + HTTP_TRACE, + HTTP_COPY, + HTTP_LOCK, + HTTP_MKCOL, + HTTP_MOVE, + HTTP_PROPFIND, + HTTP_PROPPATCH, + HTTP_SEARCH, + HTTP_UNLOCK, + HTTP_BIND, + HTTP_REBIND, + HTTP_UNBIND, + HTTP_ACL, + HTTP_REPORT, + HTTP_MKACTIVITY, + HTTP_CHECKOUT, + HTTP_MERGE, + HTTP_MSEARCH, + HTTP_NOTIFY, + HTTP_SUBSCRIBE, + HTTP_UNSUBSCRIBE, + HTTP_PATCH, + HTTP_PURGE, + HTTP_MKCALENDAR, + HTTP_LINK, + HTTP_UNLINK, + HTTP_SOURCE, + HTTP_PRI, + HTTP_DESCRIBE, + HTTP_ANNOUNCE, + HTTP_SETUP, + HTTP_PLAY, + HTTP_PAUSE, + HTTP_TEARDOWN, + HTTP_GET_PARAMETER, + HTTP_SET_PARAMETER, + HTTP_REDIRECT, + HTTP_RECORD, + HTTP_FLUSH + + void llhttp_settings_init(llhttp_settings_t* settings) + void llhttp_init(llhttp_t* parser, llhttp_type type, + const llhttp_settings_t* settings) + + llhttp_errno_t llhttp_execute(llhttp_t* parser, const char* data, size_t len) + llhttp_errno_t llhttp_finish(llhttp_t* parser) + + int llhttp_message_needs_eof(const llhttp_t* parser) + + int llhttp_should_keep_alive(const llhttp_t* parser) + + void llhttp_pause(llhttp_t* parser) + void llhttp_resume(llhttp_t* parser) + + void llhttp_resume_after_upgrade(llhttp_t* parser) + + llhttp_errno_t llhttp_get_errno(const llhttp_t* parser) + const char* llhttp_get_error_reason(const llhttp_t* parser) + void llhttp_set_error_reason(llhttp_t* parser, const char* reason) + const char* llhttp_get_error_pos(const llhttp_t* parser) + const char* llhttp_errno_name(llhttp_errno_t err) + + const char* llhttp_method_name(llhttp_method_t method) + + void llhttp_set_lenient_headers(llhttp_t* parser, int enabled) + void llhttp_set_lenient_chunked_length(llhttp_t* parser, int enabled) diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index c24e31057a8..15596d9139c 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -80,13 +80,13 @@ cdef inline object extend(object buf, const char* at, size_t length): memcpy(ptr + s, at, length) -DEF METHODS_COUNT = 34; +DEF METHODS_COUNT = 46; cdef list _http_method = [] for i in range(METHODS_COUNT): _http_method.append( - cparser.http_method_str( i).decode('ascii')) + cparser.llhttp_method_name( i).decode('ascii')) cdef inline str http_method_str(int i): @@ -272,8 +272,8 @@ cdef _new_response_message(object version, cdef class HttpParser: cdef: - cparser.http_parser* _cparser - cparser.http_parser_settings* _csettings + cparser.llhttp_t* _cparser + cparser.llhttp_settings_t* _csettings bytearray _raw_name bytearray _raw_value @@ -310,13 +310,13 @@ cdef class HttpParser: Py_buffer py_buf def __cinit__(self): - self._cparser = \ - PyMem_Malloc(sizeof(cparser.http_parser)) + self._cparser = \ + PyMem_Malloc(sizeof(cparser.llhttp_t)) if self._cparser is NULL: raise MemoryError() - self._csettings = \ - PyMem_Malloc(sizeof(cparser.http_parser_settings)) + self._csettings = \ + PyMem_Malloc(sizeof(cparser.llhttp_settings_t)) if self._csettings is NULL: raise MemoryError() @@ -324,19 +324,18 @@ cdef class HttpParser: PyMem_Free(self._cparser) PyMem_Free(self._csettings) - cdef _init(self, cparser.http_parser_type mode, + cdef _init(self, cparser.llhttp_type mode, object protocol, object loop, int limit, object timer=None, size_t max_line_size=8190, size_t max_headers=32768, size_t max_field_size=8190, payload_exception=None, bint response_with_body=True, bint read_until_eof=False, bint auto_decompress=True): - cparser.http_parser_init(self._cparser, mode) + cparser.llhttp_settings_init(self._csettings) + cparser.llhttp_init(self._cparser, mode, self._csettings) self._cparser.data = self self._cparser.content_length = 0 - cparser.http_parser_settings_init(self._csettings) - self._protocol = protocol self._loop = loop self._timer = timer @@ -417,7 +416,7 @@ cdef class HttpParser: self._process_header() method = http_method_str(self._cparser.method) - should_close = not cparser.http_should_keep_alive(self._cparser) + should_close = not cparser.llhttp_should_keep_alive(self._cparser) upgrade = self._cparser.upgrade chunked = self._cparser.flags & cparser.F_CHUNKED @@ -485,7 +484,7 @@ cdef class HttpParser: pass cdef inline http_version(self): - cdef cparser.http_parser* parser = self._cparser + cdef cparser.llhttp_t* parser = self._cparser if parser.http_major == 1: if parser.http_minor == 0: @@ -504,12 +503,11 @@ cdef class HttpParser: if self._cparser.flags & cparser.F_CHUNKED: raise TransferEncodingError( "Not enough data for satisfy transfer length header.") - elif self._cparser.flags & cparser.F_CONTENTLENGTH: + elif self._cparser.flags & cparser.F_CONTENT_LENGTH: raise ContentLengthError( "Not enough data for satisfy content length header.") - elif self._cparser.http_errno != cparser.HPE_OK: - desc = cparser.http_errno_description( - self._cparser.http_errno) + elif cparser.llhttp_get_errno(self._cparser) != cparser.HPE_OK: + desc = cparser.llhttp_get_error_reason(self._cparser) raise PayloadEncodingError(desc.decode('latin-1')) else: self._payload.feed_eof() @@ -526,22 +524,20 @@ cdef class HttpParser: PyObject_GetBuffer(data, &self.py_buf, PyBUF_SIMPLE) data_len = self.py_buf.len - nb = cparser.http_parser_execute( + nb = cparser.llhttp_execute( self._cparser, - self._csettings, self.py_buf.buf, data_len) PyBuffer_Release(&self.py_buf) - if (self._cparser.http_errno != cparser.HPE_OK): + if (cparser.llhttp_get_errno(self._cparser) != cparser.HPE_OK): if self._payload_error == 0: if self._last_error is not None: ex = self._last_error self._last_error = None else: - ex = parser_error_from_errno( - self._cparser.http_errno) + ex = parser_error_from_errno(self._cparser) self._payload = None raise ex @@ -608,7 +604,7 @@ cdef class HttpResponseParser(HttpParser): else: self._reason = self._reason or '' -cdef int cb_on_message_begin(cparser.http_parser* parser) except -1: +cdef int cb_on_message_begin(cparser.llhttp_t* parser) except -1: cdef HttpParser pyparser = parser.data pyparser._started = True @@ -620,7 +616,7 @@ cdef int cb_on_message_begin(cparser.http_parser* parser) except -1: return 0 -cdef int cb_on_url(cparser.http_parser* parser, +cdef int cb_on_url(cparser.llhttp_t* parser, const char *at, size_t length) except -1: cdef HttpParser pyparser = parser.data try: @@ -635,7 +631,7 @@ cdef int cb_on_url(cparser.http_parser* parser, return 0 -cdef int cb_on_status(cparser.http_parser* parser, +cdef int cb_on_status(cparser.llhttp_t* parser, const char *at, size_t length) except -1: cdef HttpParser pyparser = parser.data cdef str reason @@ -651,7 +647,7 @@ cdef int cb_on_status(cparser.http_parser* parser, return 0 -cdef int cb_on_header_field(cparser.http_parser* parser, +cdef int cb_on_header_field(cparser.llhttp_t* parser, const char *at, size_t length) except -1: cdef HttpParser pyparser = parser.data cdef Py_ssize_t size @@ -669,7 +665,7 @@ cdef int cb_on_header_field(cparser.http_parser* parser, return 0 -cdef int cb_on_header_value(cparser.http_parser* parser, +cdef int cb_on_header_value(cparser.llhttp_t* parser, const char *at, size_t length) except -1: cdef HttpParser pyparser = parser.data cdef Py_ssize_t size @@ -686,7 +682,7 @@ cdef int cb_on_header_value(cparser.http_parser* parser, return 0 -cdef int cb_on_headers_complete(cparser.http_parser* parser) except -1: +cdef int cb_on_headers_complete(cparser.llhttp_t* parser) except -1: cdef HttpParser pyparser = parser.data try: pyparser._on_status_complete() @@ -701,7 +697,7 @@ cdef int cb_on_headers_complete(cparser.http_parser* parser) except -1: return 0 -cdef int cb_on_body(cparser.http_parser* parser, +cdef int cb_on_body(cparser.llhttp_t* parser, const char *at, size_t length) except -1: cdef HttpParser pyparser = parser.data cdef bytes body = at[:length] @@ -718,7 +714,7 @@ cdef int cb_on_body(cparser.http_parser* parser, return 0 -cdef int cb_on_message_complete(cparser.http_parser* parser) except -1: +cdef int cb_on_message_complete(cparser.llhttp_t* parser) except -1: cdef HttpParser pyparser = parser.data try: pyparser._started = False @@ -730,7 +726,7 @@ cdef int cb_on_message_complete(cparser.http_parser* parser) except -1: return 0 -cdef int cb_on_chunk_header(cparser.http_parser* parser) except -1: +cdef int cb_on_chunk_header(cparser.llhttp_t* parser) except -1: cdef HttpParser pyparser = parser.data try: pyparser._on_chunk_header() @@ -741,7 +737,7 @@ cdef int cb_on_chunk_header(cparser.http_parser* parser) except -1: return 0 -cdef int cb_on_chunk_complete(cparser.http_parser* parser) except -1: +cdef int cb_on_chunk_complete(cparser.llhttp_t* parser) except -1: cdef HttpParser pyparser = parser.data try: pyparser._on_chunk_complete() @@ -752,19 +748,21 @@ cdef int cb_on_chunk_complete(cparser.http_parser* parser) except -1: return 0 -cdef parser_error_from_errno(cparser.http_errno errno): - cdef bytes desc = cparser.http_errno_description(errno) - - if errno in (cparser.HPE_CB_message_begin, - cparser.HPE_CB_url, - cparser.HPE_CB_header_field, - cparser.HPE_CB_header_value, - cparser.HPE_CB_headers_complete, - cparser.HPE_CB_body, - cparser.HPE_CB_message_complete, - cparser.HPE_CB_status, - cparser.HPE_CB_chunk_header, - cparser.HPE_CB_chunk_complete): +cdef parser_error_from_errno(cparser.llhttp_t* parser): + cdef cparser.llhttp_errno_t errno = cparser.llhttp_get_errno(parser) + cdef bytes desc = cparser.llhttp_get_error_reason(parser) + + if errno in (cparser.HPE_CB_MESSAGE_BEGIN, + cparser.HPE_CB_HEADERS_COMPLETE, + cparser.HPE_CB_MESSAGE_COMPLETE, + cparser.HPE_CB_CHUNK_HEADER, + cparser.HPE_CB_CHUNK_COMPLETE, + cparser.HPE_INVALID_CONSTANT, + cparser.HPE_INVALID_HEADER_TOKEN, + cparser.HPE_INVALID_CONTENT_LENGTH, + cparser.HPE_INVALID_CHUNK_SIZE, + cparser.HPE_INVALID_EOF_STATE, + cparser.HPE_INVALID_TRANSFER_ENCODING): cls = BadHttpMessage elif errno == cparser.HPE_INVALID_STATUS: @@ -773,6 +771,9 @@ cdef parser_error_from_errno(cparser.http_errno errno): elif errno == cparser.HPE_INVALID_METHOD: cls = BadStatusLine + elif errno == cparser.HPE_INVALID_VERSION: + cls = BadStatusLine + elif errno == cparser.HPE_INVALID_URL: cls = InvalidURLError diff --git a/setup.py b/setup.py index 54b548c7b44..57c6940f4b7 100644 --- a/setup.py +++ b/setup.py @@ -34,6 +34,7 @@ [ "aiohttp/_http_parser.c", "vendor/http-parser/http_parser.c", + "vendor/llhttp/build/c/llhttp.c", "aiohttp/_find_header.c", ], define_macros=[("HTTP_PARSER_STRICT", 0)], From 964d85ffae5c5bd8546ba8e7475d8f351aec1c42 Mon Sep 17 00:00:00 2001 From: Dmitry Erlikh Date: Sun, 20 Dec 2020 10:18:01 +0100 Subject: [PATCH 03/26] url_parse from old http parser --- aiohttp/_cparser.pxd | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/aiohttp/_cparser.pxd b/aiohttp/_cparser.pxd index 8b4216c128d..e26b0d1bfb4 100644 --- a/aiohttp/_cparser.pxd +++ b/aiohttp/_cparser.pxd @@ -178,3 +178,31 @@ cdef extern from "../vendor/llhttp/build/llhttp.h": void llhttp_set_lenient_headers(llhttp_t* parser, int enabled) void llhttp_set_lenient_chunked_length(llhttp_t* parser, int enabled) + +cdef extern from "../vendor/http-parser/http_parser.h": + # URL Parser + enum http_parser_url_fields: + UF_SCHEMA = 0, + UF_HOST = 1, + UF_PORT = 2, + UF_PATH = 3, + UF_QUERY = 4, + UF_FRAGMENT = 5, + UF_USERINFO = 6, + UF_MAX = 7 + + struct http_parser_url_field_data: + uint16_t off + uint16_t len + + struct http_parser_url: + uint16_t field_set + uint16_t port + http_parser_url_field_data[UF_MAX] field_data + + void http_parser_url_init(http_parser_url *u) + + int http_parser_parse_url(const char *buf, + size_t buflen, + int is_connect, + http_parser_url *u) From 50a3107da5e29b9bf2ba6e950edfac3f485e4e2a Mon Sep 17 00:00:00 2001 From: Dmitry Erlikh Date: Tue, 22 Dec 2020 10:53:33 +0100 Subject: [PATCH 04/26] Fix C parser build --- aiohttp/_cparser.pxd | 51 ++++++++++++++------------------------------ setup.py | 8 ++++--- 2 files changed, 21 insertions(+), 38 deletions(-) diff --git a/aiohttp/_cparser.pxd b/aiohttp/_cparser.pxd index e26b0d1bfb4..41f2f56e493 100644 --- a/aiohttp/_cparser.pxd +++ b/aiohttp/_cparser.pxd @@ -11,10 +11,7 @@ from libc.stdint cimport ( cdef extern from "../vendor/llhttp/build/llhttp.h": - ctypedef int (*llhttp_data_cb)(llhttp_t*, const char *at, size_t length) except -1 - ctypedef int (*llhttp_cb)(llhttp_t*) except -1 - - struct llhttp_t: + struct llhttp__internal_s: int32_t _index void* _span_pos0 void* _span_cb0 @@ -36,7 +33,13 @@ cdef extern from "../vendor/llhttp/build/llhttp.h": uint16_t status_code void* settings - struct llhttp_settings_t: + ctypedef llhttp__internal_s llhttp__internal_t + ctypedef llhttp__internal_t llhttp_t + + ctypedef int (*llhttp_data_cb)(llhttp_t*, const char *at, size_t length) except -1 + ctypedef int (*llhttp_cb)(llhttp_t*) except -1 + + struct llhttp_settings_s: llhttp_cb on_message_begin llhttp_data_cb on_url llhttp_data_cb on_status @@ -53,7 +56,9 @@ cdef extern from "../vendor/llhttp/build/llhttp.h": llhttp_cb on_header_field_complete llhttp_cb on_header_value_complete - enum llhttp_errno_t: + ctypedef llhttp_settings_s llhttp_settings_t + + enum llhttp_errno: HPE_OK, HPE_INTERNAL, HPE_STRICT, @@ -79,6 +84,8 @@ cdef extern from "../vendor/llhttp/build/llhttp.h": HPE_PAUSED_UPGRADE, HPE_USER + ctypedef llhttp_errno llhttp_errno_t + enum llhttp_flags: F_CONNECTION_KEEP_ALIVE, F_CONNECTION_CLOSE, @@ -104,7 +111,7 @@ cdef extern from "../vendor/llhttp/build/llhttp.h": HTTP_FINISH_SAFE_WITH_CB, HTTP_FINISH_UNSAFE - enum llhttp_method_t: + enum llhttp_method: HTTP_DELETE, HTTP_GET, HTTP_HEAD, @@ -152,6 +159,8 @@ cdef extern from "../vendor/llhttp/build/llhttp.h": HTTP_RECORD, HTTP_FLUSH + ctypedef llhttp_method llhttp_method_t; + void llhttp_settings_init(llhttp_settings_t* settings) void llhttp_init(llhttp_t* parser, llhttp_type type, const llhttp_settings_t* settings) @@ -178,31 +187,3 @@ cdef extern from "../vendor/llhttp/build/llhttp.h": void llhttp_set_lenient_headers(llhttp_t* parser, int enabled) void llhttp_set_lenient_chunked_length(llhttp_t* parser, int enabled) - -cdef extern from "../vendor/http-parser/http_parser.h": - # URL Parser - enum http_parser_url_fields: - UF_SCHEMA = 0, - UF_HOST = 1, - UF_PORT = 2, - UF_PATH = 3, - UF_QUERY = 4, - UF_FRAGMENT = 5, - UF_USERINFO = 6, - UF_MAX = 7 - - struct http_parser_url_field_data: - uint16_t off - uint16_t len - - struct http_parser_url: - uint16_t field_set - uint16_t port - http_parser_url_field_data[UF_MAX] field_data - - void http_parser_url_init(http_parser_url *u) - - int http_parser_parse_url(const char *buf, - size_t buflen, - int is_connect, - http_parser_url *u) diff --git a/setup.py b/setup.py index 57c6940f4b7..fce1ae3c3da 100644 --- a/setup.py +++ b/setup.py @@ -33,11 +33,13 @@ "aiohttp._http_parser", [ "aiohttp/_http_parser.c", - "vendor/http-parser/http_parser.c", - "vendor/llhttp/build/c/llhttp.c", "aiohttp/_find_header.c", + "vendor/llhttp/build/c/llhttp.c", + "vendor/llhttp/src/native/api.c", + "vendor/llhttp/src/native/http.c", ], - define_macros=[("HTTP_PARSER_STRICT", 0)], + define_macros=[("LLHTTP_STRICT_MODE", 0)], + include_dirs=["vendor/llhttp/build"], ), Extension("aiohttp._helpers", ["aiohttp/_helpers.c"]), Extension("aiohttp._http_writer", ["aiohttp/_http_writer.c"]), From f6530869dab57950cc7299825766304f1df0cebc Mon Sep 17 00:00:00 2001 From: Dmitry Erlikh Date: Tue, 22 Dec 2020 10:54:10 +0100 Subject: [PATCH 05/26] Add c parser load test --- tests/test_http_parser.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_http_parser.py b/tests/test_http_parser.py index 77c62addf6b..6d80b4306a9 100644 --- a/tests/test_http_parser.py +++ b/tests/test_http_parser.py @@ -13,6 +13,7 @@ import aiohttp from aiohttp import http_exceptions, streams from aiohttp.http_parser import ( + NO_EXTENSIONS, DeflateBuffer, HttpPayloadParser, HttpRequestParserPy, @@ -85,6 +86,14 @@ def stream(): return mock.Mock() +@pytest.mark.skipif(NO_EXTENSIONS, reason="Extentions available but not imported") +def test_c_parser_loaded(): + assert "HttpRequestParserC" in dir(aiohttp.http_parser) + assert "HttpResponseParserC" in dir(aiohttp.http_parser) + assert "RawRequestMessageC" in dir(aiohttp.http_parser) + assert "RawResponseMessageC" in dir(aiohttp.http_parser) + + def test_parse_headers(parser: Any) -> None: text = b"""GET /test HTTP/1.1\r test: line\r From d509d63ad766ec143d7ac19c1b29e79730fd02de Mon Sep 17 00:00:00 2001 From: Dmitry Erlikh Date: Tue, 22 Dec 2020 11:49:10 +0100 Subject: [PATCH 06/26] Check for upgrade paused error code --- aiohttp/_http_parser.pyx | 109 ++------------------------------------- 1 file changed, 5 insertions(+), 104 deletions(-) diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index 15596d9139c..b6a512c8dfa 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -531,7 +531,8 @@ cdef class HttpParser: PyBuffer_Release(&self.py_buf) - if (cparser.llhttp_get_errno(self._cparser) != cparser.HPE_OK): + cdef cparser.llhttp_errno_t errno = cparser.llhttp_get_errno(self._cparser) + if errno not in (cparser.HPE_OK, cparser.HPE_PAUSED_UPGRADE): if self._payload_error == 0: if self._last_error is not None: ex = self._last_error @@ -568,20 +569,13 @@ cdef class HttpRequestParser(HttpParser): payload_exception, response_with_body, read_until_eof) cdef object _on_status_complete(self): - cdef Py_buffer py_buf if not self._buf: return self._path = self._buf.decode('utf-8', 'surrogateescape') - if self._cparser.method == 5: # CONNECT + try: self._url = URL(self._path) - else: - PyObject_GetBuffer(self._buf, &py_buf, PyBUF_SIMPLE) - try: - self._url = _parse_url(py_buf.buf, - py_buf.len) - finally: - PyBuffer_Release(&py_buf) - PyByteArray_Resize(self._buf, 0) + finally: + PyByteArray_Resize(self._buf, 0) cdef class HttpResponseParser(HttpParser): @@ -781,96 +775,3 @@ cdef parser_error_from_errno(cparser.llhttp_t* parser): cls = BadHttpMessage return cls(desc.decode('latin-1')) - - -def parse_url(url): - cdef: - Py_buffer py_buf - char* buf_data - - PyObject_GetBuffer(url, &py_buf, PyBUF_SIMPLE) - try: - buf_data = py_buf.buf - return _parse_url(buf_data, py_buf.len) - finally: - PyBuffer_Release(&py_buf) - - -cdef _parse_url(char* buf_data, size_t length): - cdef: - cparser.http_parser_url* parsed - int res - str schema = None - str host = None - object port = None - str path = None - str query = None - str fragment = None - str user = None - str password = None - str userinfo = None - object result = None - int off - int ln - - parsed = \ - PyMem_Malloc(sizeof(cparser.http_parser_url)) - if parsed is NULL: - raise MemoryError() - cparser.http_parser_url_init(parsed) - try: - res = cparser.http_parser_parse_url(buf_data, length, 0, parsed) - - if res == 0: - if parsed.field_set & (1 << cparser.UF_SCHEMA): - off = parsed.field_data[cparser.UF_SCHEMA].off - ln = parsed.field_data[cparser.UF_SCHEMA].len - schema = buf_data[off:off+ln].decode('utf-8', 'surrogateescape') - else: - schema = '' - - if parsed.field_set & (1 << cparser.UF_HOST): - off = parsed.field_data[cparser.UF_HOST].off - ln = parsed.field_data[cparser.UF_HOST].len - host = buf_data[off:off+ln].decode('utf-8', 'surrogateescape') - else: - host = '' - - if parsed.field_set & (1 << cparser.UF_PORT): - port = parsed.port - - if parsed.field_set & (1 << cparser.UF_PATH): - off = parsed.field_data[cparser.UF_PATH].off - ln = parsed.field_data[cparser.UF_PATH].len - path = buf_data[off:off+ln].decode('utf-8', 'surrogateescape') - else: - path = '' - - if parsed.field_set & (1 << cparser.UF_QUERY): - off = parsed.field_data[cparser.UF_QUERY].off - ln = parsed.field_data[cparser.UF_QUERY].len - query = buf_data[off:off+ln].decode('utf-8', 'surrogateescape') - else: - query = '' - - if parsed.field_set & (1 << cparser.UF_FRAGMENT): - off = parsed.field_data[cparser.UF_FRAGMENT].off - ln = parsed.field_data[cparser.UF_FRAGMENT].len - fragment = buf_data[off:off+ln].decode('utf-8', 'surrogateescape') - else: - fragment = '' - - if parsed.field_set & (1 << cparser.UF_USERINFO): - off = parsed.field_data[cparser.UF_USERINFO].off - ln = parsed.field_data[cparser.UF_USERINFO].len - userinfo = buf_data[off:off+ln].decode('utf-8', 'surrogateescape') - - user, sep, password = userinfo.partition(':') - - return URL_build(scheme=schema, - user=user, password=password, host=host, port=port, - path=path, query_string=query, fragment=fragment, encoded=True) - else: - raise InvalidURLError("invalid url {!r}".format(buf_data)) - finally: - PyMem_Free(parsed) From 94aa71175ff0329022df3986549beaac9e41d198 Mon Sep 17 00:00:00 2001 From: Dmitry Erlikh Date: Tue, 22 Dec 2020 12:13:56 +0100 Subject: [PATCH 07/26] Fix connection: upgrade --- aiohttp/_http_parser.pyx | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index b6a512c8dfa..5fb12521605 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -520,18 +520,23 @@ cdef class HttpParser: cdef: size_t data_len size_t nb + cdef cparser.llhttp_errno_t errno PyObject_GetBuffer(data, &self.py_buf, PyBUF_SIMPLE) data_len = self.py_buf.len - nb = cparser.llhttp_execute( + errno = cparser.llhttp_execute( self._cparser, self.py_buf.buf, data_len) + if errno is cparser.HPE_PAUSED_UPGRADE: + cparser.llhttp_resume_after_upgrade(self._cparser) + + nb = cparser.llhttp_get_error_pos(self._cparser) - self.py_buf.buf + PyBuffer_Release(&self.py_buf) - cdef cparser.llhttp_errno_t errno = cparser.llhttp_get_errno(self._cparser) if errno not in (cparser.HPE_OK, cparser.HPE_PAUSED_UPGRADE): if self._payload_error == 0: if self._last_error is not None: From d15e220f20575ad9cf51f248e793da87761466c2 Mon Sep 17 00:00:00 2001 From: Dmitry Erlikh Date: Tue, 22 Dec 2020 13:07:07 +0100 Subject: [PATCH 08/26] Fix no content-length --- aiohttp/_http_parser.pyx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aiohttp/_http_parser.pyx b/aiohttp/_http_parser.pyx index 5fb12521605..25800a62974 100644 --- a/aiohttp/_http_parser.pyx +++ b/aiohttp/_http_parser.pyx @@ -452,7 +452,7 @@ cdef class HttpParser: if (ULLONG_MAX > self._cparser.content_length > 0 or chunked or self._cparser.method == 5 or # CONNECT: 5 (self._cparser.status_code >= 199 and - self._cparser.content_length == ULLONG_MAX and + self._cparser.content_length == 0 and self._read_until_eof) ): payload = StreamReader( From 512ed788fe80e938ad06c0e82d8f8b8992a9c27d Mon Sep 17 00:00:00 2001 From: Dmitry Erlikh Date: Tue, 22 Dec 2020 13:13:25 +0100 Subject: [PATCH 09/26] Remove http-parser --- .gitmodules | 4 ---- vendor/http-parser | 1 - 2 files changed, 5 deletions(-) delete mode 160000 vendor/http-parser diff --git a/.gitmodules b/.gitmodules index 71f9d2c85b8..d0a98c490b2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,3 @@ -[submodule "vendor/http-parser"] - path = vendor/http-parser - url = git://github.com/nodejs/http-parser.git - branch = 54f55a2 [submodule "vendor/llhttp"] path = vendor/llhttp url = https://github.com/nodejs/llhttp.git diff --git a/vendor/http-parser b/vendor/http-parser deleted file mode 160000 index 2343fd6b521..00000000000 --- a/vendor/http-parser +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 2343fd6b5214b2ded2cdcf76de2bf60903bb90cd From 20609bf26f486b70402f6dadf771e0ff0a6e20b0 Mon Sep 17 00:00:00 2001 From: Dmitry Erlikh Date: Tue, 22 Dec 2020 13:15:56 +0100 Subject: [PATCH 10/26] Add changes file --- CHANGES/3561.feature | 1 + 1 file changed, 1 insertion(+) create mode 100644 CHANGES/3561.feature diff --git a/CHANGES/3561.feature b/CHANGES/3561.feature new file mode 100644 index 00000000000..db42f95f08e --- /dev/null +++ b/CHANGES/3561.feature @@ -0,0 +1 @@ +Switch from http-parser to llhttp From ad8f6d92a8d20e59de9c4dca9fc0eb76c5fad96e Mon Sep 17 00:00:00 2001 From: Dmitry Erlikh Date: Tue, 22 Dec 2020 13:24:07 +0100 Subject: [PATCH 11/26] Fix submodules check in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index fce1ae3c3da..93ba0784c58 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ NO_EXTENSIONS = True -if IS_GIT_REPO and not (HERE / "vendor/http-parser/README.md").exists(): +if IS_GIT_REPO and not (HERE / "vendor/llhttp/README.md").exists(): print("Install submodules when building from git clone", file=sys.stderr) print("Hint:", file=sys.stderr) print(" git submodule update --init", file=sys.stderr) From a24cef46a6cadc10b86416731b7019a13138ad85 Mon Sep 17 00:00:00 2001 From: Dmitry Erlikh Date: Tue, 22 Dec 2020 13:54:21 +0100 Subject: [PATCH 12/26] ci --- .github/workflows/ci.yml | 32 +++++++++++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index be50d905be7..3cb614a3347 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -71,9 +71,39 @@ jobs: run: | LC_ALL=C sort -c CONTRIBUTORS.txt + gen_llhttp: + name: Generate llhttp sources + needs: lint + runs-on: ubuntu-latest + timeout-minutes: 5 + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + submodules: true + - name: Cache llhttp generated files + uses: actions/cache@v2 + id: cache + with: + key: llhttp-${{ hashFiles('vendor/llhttp/package.json', 'vendor/llhttp/src/**/*') }} + path: | + vendor/llhttp/build/llhttp.h + vendor/llhttp/build/c/llhttp.c + - name: Setup NodeJS + uses: actions/setup-node@v2 + if: steps.cache.outputs.cache-hit != 'true' + with: + node-version: '14' + - name: Generate sources + if: steps.cache.outputs.cache-hit != 'true' + run: | + cd vendor/llhttp + npm install + make generate + test: name: Test - needs: lint + needs: gen_llhttp strategy: matrix: pyver: [3.7, 3.8, 3.9] From c1afb1c99ce8c494b770836384690a0ad367934d Mon Sep 17 00:00:00 2001 From: Dmitry Erlikh Date: Tue, 22 Dec 2020 14:12:29 +0100 Subject: [PATCH 13/26] restore llhttp files --- .github/workflows/ci.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3cb614a3347..72ffb67314c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -90,8 +90,8 @@ jobs: vendor/llhttp/build/llhttp.h vendor/llhttp/build/c/llhttp.c - name: Setup NodeJS - uses: actions/setup-node@v2 if: steps.cache.outputs.cache-hit != 'true' + uses: actions/setup-node@v2 with: node-version: '14' - name: Generate sources @@ -150,6 +150,15 @@ jobs: if: ${{ matrix.no-extensions == '' }} run: | make cythonize + - name: Restore llhttp generated files + if: ${{ matrix.no-extensions == '' }} + uses: actions/cache@v2 + with: + key: llhttp-${{ hashFiles('vendor/llhttp/package.json', 'vendor/llhttp/src/**/*') }} + path: | + vendor/llhttp/build/llhttp.h + vendor/llhttp/build/c/llhttp.c + restore-keys: llhttp- - name: Run unittests env: COLOR: 'yes' From 08d9d2a41a72702f650bc9ba6790c54b45560c97 Mon Sep 17 00:00:00 2001 From: Dmitry Erlikh Date: Tue, 22 Dec 2020 14:38:09 +0100 Subject: [PATCH 14/26] fail if cache not restored --- .github/workflows/ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 72ffb67314c..86d35eed7cd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -152,6 +152,7 @@ jobs: make cythonize - name: Restore llhttp generated files if: ${{ matrix.no-extensions == '' }} + id: llhttp-restore-cache uses: actions/cache@v2 with: key: llhttp-${{ hashFiles('vendor/llhttp/package.json', 'vendor/llhttp/src/**/*') }} @@ -159,6 +160,10 @@ jobs: vendor/llhttp/build/llhttp.h vendor/llhttp/build/c/llhttp.c restore-keys: llhttp- + - name: Check llhttp cache + if: steps.llhttp-restore-cache.outputs.cache-hit != 'true' && ${{ matrix.no-extensions == '' }} + run: | + exit 1 - name: Run unittests env: COLOR: 'yes' From f8cf7d6b11fb272fbaa558670ab19e5d9d0a28b6 Mon Sep 17 00:00:00 2001 From: Dmitry Erlikh Date: Tue, 22 Dec 2020 15:14:26 +0100 Subject: [PATCH 15/26] Upload artifact --- .github/workflows/ci.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 86d35eed7cd..0c3b26fc20a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -96,10 +96,17 @@ jobs: node-version: '14' - name: Generate sources if: steps.cache.outputs.cache-hit != 'true' + working-directory: vendor/llhttp run: | - cd vendor/llhttp npm install make generate + - name: Upload llhttp generated files + uses: actions/upload-artifact@v2 + with: + name: llhttp-${{ hashFiles('vendor/llhttp/package.json', 'vendor/llhttp/src/**/*') }} + path: | + vendor/llhttp/build/llhttp.h + vendor/llhttp/build/c/llhttp.c test: name: Test From 551e5e665308dc1c56f8af3c16fec10d5bc2b80a Mon Sep 17 00:00:00 2001 From: Dmitry Erlikh Date: Tue, 22 Dec 2020 15:23:05 +0100 Subject: [PATCH 16/26] Download artifact --- .github/workflows/ci.yml | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0c3b26fc20a..ae38c06fc30 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -159,18 +159,10 @@ jobs: make cythonize - name: Restore llhttp generated files if: ${{ matrix.no-extensions == '' }} - id: llhttp-restore-cache - uses: actions/cache@v2 + uses: actions/download-artifact@v2 with: - key: llhttp-${{ hashFiles('vendor/llhttp/package.json', 'vendor/llhttp/src/**/*') }} - path: | - vendor/llhttp/build/llhttp.h - vendor/llhttp/build/c/llhttp.c - restore-keys: llhttp- - - name: Check llhttp cache - if: steps.llhttp-restore-cache.outputs.cache-hit != 'true' && ${{ matrix.no-extensions == '' }} - run: | - exit 1 + name: llhttp-${{ hashFiles('vendor/llhttp/package.json', 'vendor/llhttp/src/**/*') }} + path: vendor/llhttp/build/ - name: Run unittests env: COLOR: 'yes' From 051a6c8922ff570a3afc9ec9bbd8632f276224e4 Mon Sep 17 00:00:00 2001 From: Dmitry Erlikh Date: Tue, 22 Dec 2020 15:48:07 +0100 Subject: [PATCH 17/26] update Makefile --- .gitignore | 1 + Makefile | 14 +++++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 8556509c6f7..7d38dd91998 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,7 @@ .idea .install-cython .install-deps +.llhttp-gen .installed.cfg .mypy_cache .noseids diff --git a/Makefile b/Makefile index 9a1e8311495..43aa4f0712d 100644 --- a/Makefile +++ b/Makefile @@ -57,9 +57,18 @@ aiohttp/_find_header.c: $(call to-hash,aiohttp/hdrs.py ./tools/gen.py) aiohttp/%.c: aiohttp/%.pyx $(call to-hash,$(CYS)) aiohttp/_find_header.c cython -3 -o $@ $< -I aiohttp +vendor/llhttp/node_modules: vendor/llhttp/package.json + cd vendor/llhttp; npm install + +.llhttp-gen: vendor/llhttp/node_modules + $(MAKE) -C vendor/llhttp generate + @touch .llhttp-gen + +.PHONY: generate-llhttp +generate-llhttp: .llhttp-gen .PHONY: cythonize -cythonize: .install-cython $(PYXS:.pyx=.c) +cythonize: .install-cython $(PYXS:.pyx=.c) generate-llhttp .install-deps: .install-cython $(PYXS:.pyx=.c) $(call to-hash,$(CYS) $(REQS)) pip install -r requirements/dev.txt @@ -125,6 +134,9 @@ clean: @rm -rf aiohttp.egg-info @rm -f .install-deps @rm -f .install-cython + @rm -rf vendor/llhttp/node_modules + @rm -f .llhttp-gen + @$(MAKE) -C vendor/llhttp clean .PHONY: doc doc: From a72d8a5928a7730ed77b0d34781b7e843701c2b2 Mon Sep 17 00:00:00 2001 From: Dmitry Erlikh Date: Tue, 22 Dec 2020 15:54:51 +0100 Subject: [PATCH 18/26] Use make to generate llhttp sources --- .github/workflows/ci.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ae38c06fc30..c8227f2252f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -94,12 +94,10 @@ jobs: uses: actions/setup-node@v2 with: node-version: '14' - - name: Generate sources + - name: Generate llhttp sources if: steps.cache.outputs.cache-hit != 'true' - working-directory: vendor/llhttp run: | - npm install - make generate + make generate-llhttp - name: Upload llhttp generated files uses: actions/upload-artifact@v2 with: From ee0d83b6e094df5de667b94bf00f3da31eb02b68 Mon Sep 17 00:00:00 2001 From: Dmitry Erlikh Date: Tue, 22 Dec 2020 15:55:14 +0100 Subject: [PATCH 19/26] use dir for cache and upload artifact actions --- .github/workflows/ci.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c8227f2252f..44378f328bb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -86,9 +86,7 @@ jobs: id: cache with: key: llhttp-${{ hashFiles('vendor/llhttp/package.json', 'vendor/llhttp/src/**/*') }} - path: | - vendor/llhttp/build/llhttp.h - vendor/llhttp/build/c/llhttp.c + path: vendor/llhttp/build - name: Setup NodeJS if: steps.cache.outputs.cache-hit != 'true' uses: actions/setup-node@v2 @@ -102,9 +100,7 @@ jobs: uses: actions/upload-artifact@v2 with: name: llhttp-${{ hashFiles('vendor/llhttp/package.json', 'vendor/llhttp/src/**/*') }} - path: | - vendor/llhttp/build/llhttp.h - vendor/llhttp/build/c/llhttp.c + path: vendor/llhttp/build test: name: Test From 101a993259f6b7d2e25e496d51fb4d9c0b190720 Mon Sep 17 00:00:00 2001 From: Dmitry Erlikh Date: Tue, 22 Dec 2020 16:15:27 +0100 Subject: [PATCH 20/26] remove artifact suffix --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 44378f328bb..da0968ba2ca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -99,7 +99,7 @@ jobs: - name: Upload llhttp generated files uses: actions/upload-artifact@v2 with: - name: llhttp-${{ hashFiles('vendor/llhttp/package.json', 'vendor/llhttp/src/**/*') }} + name: llhttp path: vendor/llhttp/build test: @@ -155,7 +155,7 @@ jobs: if: ${{ matrix.no-extensions == '' }} uses: actions/download-artifact@v2 with: - name: llhttp-${{ hashFiles('vendor/llhttp/package.json', 'vendor/llhttp/src/**/*') }} + name: llhttp path: vendor/llhttp/build/ - name: Run unittests env: From 9393c1ce1d9aa77840280e7a2255a8f5a5682864 Mon Sep 17 00:00:00 2001 From: Dmitry Erlikh Date: Tue, 22 Dec 2020 16:32:23 +0100 Subject: [PATCH 21/26] Add SHA1 suffiix to artifact name --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index da0968ba2ca..472cc0bf777 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -99,7 +99,7 @@ jobs: - name: Upload llhttp generated files uses: actions/upload-artifact@v2 with: - name: llhttp + name: llhttp-${{ env.GITHUB_SHA }} path: vendor/llhttp/build test: @@ -155,7 +155,7 @@ jobs: if: ${{ matrix.no-extensions == '' }} uses: actions/download-artifact@v2 with: - name: llhttp + name: llhttp-${{ env.GITHUB_SHA }} path: vendor/llhttp/build/ - name: Run unittests env: From 4761a0d10fad079dda73e1bd6df5fe634d9122b8 Mon Sep 17 00:00:00 2001 From: Dmitry Erlikh Date: Tue, 22 Dec 2020 16:35:43 +0100 Subject: [PATCH 22/26] Set llhttp branch --- .gitmodules | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitmodules b/.gitmodules index d0a98c490b2..5f9f397b8e2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,4 @@ [submodule "vendor/llhttp"] path = vendor/llhttp url = https://github.com/nodejs/llhttp.git + branch = v3.0.0 From ac8e889d396d28f36319d2ddff3c376dedc96bac Mon Sep 17 00:00:00 2001 From: Dmitry Erlikh Date: Tue, 22 Dec 2020 17:12:34 +0100 Subject: [PATCH 23/26] SHA1 -> RUN_ID for artifact name --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 472cc0bf777..c77fd24d25b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -99,7 +99,7 @@ jobs: - name: Upload llhttp generated files uses: actions/upload-artifact@v2 with: - name: llhttp-${{ env.GITHUB_SHA }} + name: llhttp-${{ env.GITHUB_RUN_ID }} path: vendor/llhttp/build test: @@ -155,7 +155,7 @@ jobs: if: ${{ matrix.no-extensions == '' }} uses: actions/download-artifact@v2 with: - name: llhttp-${{ env.GITHUB_SHA }} + name: llhttp-${{ env.GITHUB_RUN_ID }} path: vendor/llhttp/build/ - name: Run unittests env: From 6367b13c9874256c9352c5680eaebf57e9f2824d Mon Sep 17 00:00:00 2001 From: Dmitry Erlikh Date: Tue, 22 Dec 2020 17:21:18 +0100 Subject: [PATCH 24/26] Remove suffix from artifact name --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c77fd24d25b..da0968ba2ca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -99,7 +99,7 @@ jobs: - name: Upload llhttp generated files uses: actions/upload-artifact@v2 with: - name: llhttp-${{ env.GITHUB_RUN_ID }} + name: llhttp path: vendor/llhttp/build test: @@ -155,7 +155,7 @@ jobs: if: ${{ matrix.no-extensions == '' }} uses: actions/download-artifact@v2 with: - name: llhttp-${{ env.GITHUB_RUN_ID }} + name: llhttp path: vendor/llhttp/build/ - name: Run unittests env: From 9933ff7532220f50ba22de6f603a502446a9c743 Mon Sep 17 00:00:00 2001 From: Dmitry Erlikh Date: Tue, 22 Dec 2020 17:23:05 +0100 Subject: [PATCH 25/26] fail if no files in artifact --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index da0968ba2ca..df486fa6bfa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -101,6 +101,7 @@ jobs: with: name: llhttp path: vendor/llhttp/build + if-no-files-found: error test: name: Test From e12379538bd2e6c2015f2ea50bb490861f4fa7c2 Mon Sep 17 00:00:00 2001 From: Dmitry Erlikh Date: Tue, 22 Dec 2020 17:50:38 +0100 Subject: [PATCH 26/26] fix make all build --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 43aa4f0712d..82c66eb809c 100644 --- a/Makefile +++ b/Makefile @@ -68,7 +68,7 @@ vendor/llhttp/node_modules: vendor/llhttp/package.json generate-llhttp: .llhttp-gen .PHONY: cythonize -cythonize: .install-cython $(PYXS:.pyx=.c) generate-llhttp +cythonize: .install-cython $(PYXS:.pyx=.c) .install-deps: .install-cython $(PYXS:.pyx=.c) $(call to-hash,$(CYS) $(REQS)) pip install -r requirements/dev.txt @@ -85,7 +85,7 @@ fmt format: mypy: mypy aiohttp tests -.develop: .install-deps $(call to-hash,$(PYS) $(CYS) $(CS)) +.develop: .install-deps generate-llhttp $(call to-hash,$(PYS) $(CYS) $(CS)) pip install -e . @touch .develop