diff --git a/.github/issue_template.md b/.github/issue_template.md index 7c313535128c..18b87fca8e78 100644 --- a/.github/issue_template.md +++ b/.github/issue_template.md @@ -1,7 +1,7 @@ Note: This is just a template, so feel free to use/remove the unnecessary things ### Description -- Type: Bug | Enhancement\Feature Request | Question +- Type: Bug | Enhancement\Feature Request - Priority: Blocker | Major | Minor --------------------------------------------------------------- @@ -38,4 +38,4 @@ Version: ## Question -**Please first check for answers in the [Mbed TLS knowledge Base](https://tls.mbed.org/kb), and preferably file an issue in the [Mbed TLS support forum](https://forums.mbed.com/c/mbed-tls)** +**Please first check for answers in the [Mbed TLS knowledge Base](https://tls.mbed.org/kb). If you can't find the answer you're looking for then please use the [Mbed TLS mailing list](https://lists.trustedfirmware.org/mailman/listinfo/mbed-tls)** diff --git a/ChangeLog b/ChangeLog index bcceebb7d5d9..5d49004fb073 100644 --- a/ChangeLog +++ b/ChangeLog @@ -6,9 +6,19 @@ New deprecations * Deprecate MBEDTLS_SSL_HW_RECORD_ACCEL that enables function hooks in the SSL module for hardware acceleration of individual records. +Security + * Fix issue in DTLS handling of new associations with the same parameters + (RFC 6347 section 4.2.8): an attacker able to send forged UDP packets to + the server could cause it to drop established associations with + legitimate clients, resulting in a Denial of Service. This could only + happen when MBEDTLS_SSL_DTLS_CLIENT_PORT_REUSE was enabled in config.h + (which it is by default). + Bugfix * Fix compilation failure when both MBEDTLS_SSL_PROTO_DTLS and MBEDTLS_SSL_HW_RECORD_ACCEL are enabled. + * Remove a spurious check in ssl_parse_client_psk_identity that triggered + a warning with some compilers. Fix contributed by irwir in #2856. Changes * Mbed Crypto is no longer a Git submodule. The crypto part of the library diff --git a/ChangeLog.d/00README.md b/ChangeLog.d/00README.md new file mode 100644 index 000000000000..b559e2336cbe --- /dev/null +++ b/ChangeLog.d/00README.md @@ -0,0 +1,67 @@ +# Pending changelog entry directory + +This directory contains changelog entries that have not yet been merged +to the changelog file ([`../ChangeLog`](../ChangeLog)). + +## Changelog entry file format + +A changelog entry file must have the extension `*.txt` and must have the +following format: + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Security + * Change description. + * Another change description. + +Features + * Yet another change description. This is a long change description that + spans multiple lines. + * Yet again another change description. + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +The permitted changelog entry categories are as follows: + + + API changes + Default behavior changes + Requirement changes + New deprecations + Removals + Features + Security + Bugfix + Changes + +Use “Changes” for anything that doesn't fit in the other categories, such as +performance, documentation and test improvements. + +## How to write a changelog entry + +Each entry starts with three spaces, an asterisk and a space. Continuation +lines start with 5 spaces. Lines wrap at 79 characters. + +Write full English sentences with proper capitalization and punctuation. Use +the present tense. Use the imperative where applicable. For example: “Fix a +bug in mbedtls_xxx() ….” + +Include GitHub issue numbers where relevant. Use the format “#1234” for an +Mbed TLS issue. Add other external references such as CVE numbers where +applicable. + +Credit the author of the contribution if the contribution is not a member of +the Mbed TLS development team. Also credit bug reporters where applicable. + +**Explain why, not how**. Remember that the audience is the users of the +library, not its developers. In particular, for a bug fix, explain the +consequences of the bug, not how the bug was fixed. For a new feature, explain +why one might be interested in the feature. For an API change or a deprecation, +explain how to update existing applications. + +See [existing entries](../ChangeLog) for examples. + +## How `ChangeLog` is updated + +Run [`../scripts/assemble_changelog.py`](../scripts/assemble_changelog.py) +from a Git working copy +to move the entries from files in `ChangeLog.d` to the main `ChangeLog` file. diff --git a/ChangeLog.d/README b/ChangeLog.d/README deleted file mode 100644 index 2f9f049f94c4..000000000000 --- a/ChangeLog.d/README +++ /dev/null @@ -1,21 +0,0 @@ -This directory contains changelog entries that have not yet been merged -to the changelog file (../ChangeLog.md). - -A changelog entry file must have the extension *.md and must have the -following format: - -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -### Section title - -* Change descritpion. -* Another change description. - -### Another section title - -* Yet another change description. -* Yet again another change description. - -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -See STANDARD_SECTIONS in ../scripts/assemble_changelog.py for -recognized section titles. diff --git a/include/mbedtls/check_config.h b/include/mbedtls/check_config.h index d904d5a7a150..fa3caa7c41a2 100644 --- a/include/mbedtls/check_config.h +++ b/include/mbedtls/check_config.h @@ -619,6 +619,23 @@ #error "MBEDTLS_SSL_PROTO_TLS1_2 defined, but not all prerequisites" #endif +#if (defined(MBEDTLS_SSL_PROTO_SSL3) || defined(MBEDTLS_SSL_PROTO_TLS1) || \ + defined(MBEDTLS_SSL_PROTO_TLS1_1) || defined(MBEDTLS_SSL_PROTO_TLS1_2)) && \ + !(defined(MBEDTLS_KEY_EXCHANGE_RSA_ENABLED) || \ + defined(MBEDTLS_KEY_EXCHANGE_DHE_RSA_ENABLED) || \ + defined(MBEDTLS_KEY_EXCHANGE_ECDHE_RSA_ENABLED) || \ + defined(MBEDTLS_KEY_EXCHANGE_ECDHE_ECDSA_ENABLED) || \ + defined(MBEDTLS_KEY_EXCHANGE_ECDH_RSA_ENABLED) || \ + defined(MBEDTLS_KEY_EXCHANGE_ECDH_ECDSA_ENABLED) || \ + defined(MBEDTLS_KEY_EXCHANGE_PSK_ENABLED) || \ + defined(MBEDTLS_KEY_EXCHANGE_DHE_PSK_ENABLED) || \ + defined(MBEDTLS_KEY_EXCHANGE_RSA_PSK_ENABLED) || \ + defined(MBEDTLS_KEY_EXCHANGE_ECDHE_PSK_ENABLED) || \ + defined(MBEDTLS_KEY_EXCHANGE_ECJPAKE_ENABLED) ) +#error "One or more versions of the TLS protocol are enabled " \ + "but no key exchange methods defined with MBEDTLS_KEY_EXCHANGE_xxxx" +#endif + #if defined(MBEDTLS_SSL_PROTO_DTLS) && \ !defined(MBEDTLS_SSL_PROTO_TLS1_1) && \ !defined(MBEDTLS_SSL_PROTO_TLS1_2) @@ -763,6 +780,10 @@ #error "MBEDTLS_X509_CREATE_C defined, but not all prerequisites" #endif +#if defined(MBEDTLS_CERTS_C) && !defined(MBEDTLS_X509_USE_C) +#error "MBEDTLS_CERTS_C defined, but not all prerequisites" +#endif + #if defined(MBEDTLS_X509_CRT_PARSE_C) && ( !defined(MBEDTLS_X509_USE_C) ) #error "MBEDTLS_X509_CRT_PARSE_C defined, but not all prerequisites" #endif diff --git a/include/mbedtls/config.h b/include/mbedtls/config.h index d5502a9473be..901e26d897c4 100644 --- a/include/mbedtls/config.h +++ b/include/mbedtls/config.h @@ -1520,8 +1520,8 @@ /** \def MBEDTLS_SSL_EXTENDED_MASTER_SECRET * - * Enable support for Extended Master Secret, aka Session Hash - * (draft-ietf-tls-session-hash-02). + * Enable support for RFC 7627: Session Hash and Extended Master Secret + * Extension. * * This was introduced as "the proper fix" to the Triple Handshake familiy of * attacks, but it is recommended to always use it (even if you disable @@ -1539,7 +1539,8 @@ /** * \def MBEDTLS_SSL_FALLBACK_SCSV * - * Enable support for FALLBACK_SCSV (draft-ietf-tls-downgrade-scsv-00). + * Enable support for RFC 7507: Fallback Signaling Cipher Suite Value (SCSV) + * for Preventing Protocol Downgrade Attacks. * * For servers, it is recommended to always enable this, unless you support * only one version of TLS, or know for sure that none of your clients diff --git a/library/hkdf.c b/library/hkdf.c index 379035ddbb5a..82df597a4c69 100644 --- a/library/hkdf.c +++ b/library/hkdf.c @@ -115,7 +115,7 @@ int mbedtls_hkdf_expand( const mbedtls_md_info_t *md, const unsigned char *prk, n = okm_len / hash_len; - if( (okm_len % hash_len) != 0 ) + if( okm_len % hash_len != 0 ) { n++; } @@ -131,11 +131,13 @@ int mbedtls_hkdf_expand( const mbedtls_md_info_t *md, const unsigned char *prk, mbedtls_md_init( &ctx ); - if( (ret = mbedtls_md_setup( &ctx, md, 1) ) != 0 ) + if( ( ret = mbedtls_md_setup( &ctx, md, 1 ) ) != 0 ) { goto exit; } + memset( t, 0, hash_len ); + /* * Compute T = T(1) | T(2) | T(3) | ... | T(N) * Where T(N) is defined in RFC 5869 Section 2.3 diff --git a/library/ssl_cli.c b/library/ssl_cli.c index c0b440a2da65..ff6b7b69e357 100644 --- a/library/ssl_cli.c +++ b/library/ssl_cli.c @@ -2344,7 +2344,7 @@ static int ssl_parse_server_psk_hint( mbedtls_ssl_context *ssl, unsigned char *end ) { int ret = MBEDTLS_ERR_SSL_FEATURE_UNAVAILABLE; - size_t len; + uint16_t len; ((void) ssl); /* @@ -2361,7 +2361,7 @@ static int ssl_parse_server_psk_hint( mbedtls_ssl_context *ssl, len = (*p)[0] << 8 | (*p)[1]; *p += 2; - if( end - (*p) < (int) len ) + if( end - (*p) < len ) { MBEDTLS_SSL_DEBUG_MSG( 1, ( "bad server key exchange message " "(psk_identity_hint length)" ) ); diff --git a/library/ssl_msg.c b/library/ssl_msg.c index 18fa55574dea..428ace5bacec 100644 --- a/library/ssl_msg.c +++ b/library/ssl_msg.c @@ -3197,16 +3197,17 @@ static int ssl_check_dtls_clihlo_cookie( * that looks like a ClientHello. * * - if the input looks like a ClientHello without cookies, - * send back HelloVerifyRequest, then - * return MBEDTLS_ERR_SSL_HELLO_VERIFY_REQUIRED + * send back HelloVerifyRequest, then return 0 * - if the input looks like a ClientHello with a valid cookie, * reset the session of the current context, and * return MBEDTLS_ERR_SSL_CLIENT_RECONNECT * - if anything goes wrong, return a specific error code * - * mbedtls_ssl_read_record() will ignore the record if anything else than - * MBEDTLS_ERR_SSL_CLIENT_RECONNECT or 0 is returned, although this function - * cannot not return 0. + * This function is called (through ssl_check_client_reconnect()) when an + * unexpected record is found in ssl_get_next_record(), which will discard the + * record if we return 0, and bubble up the return value otherwise (this + * includes the case of MBEDTLS_ERR_SSL_CLIENT_RECONNECT and of unexpected + * errors, and is the right thing to do in both cases). */ static int ssl_handle_possible_reconnect( mbedtls_ssl_context *ssl ) { @@ -3218,6 +3219,8 @@ static int ssl_handle_possible_reconnect( mbedtls_ssl_context *ssl ) { /* If we can't use cookies to verify reachability of the peer, * drop the record. */ + MBEDTLS_SSL_DEBUG_MSG( 1, ( "no cookie callbacks, " + "can't check reconnect validity" ) ); return( 0 ); } @@ -3233,16 +3236,23 @@ static int ssl_handle_possible_reconnect( mbedtls_ssl_context *ssl ) if( ret == MBEDTLS_ERR_SSL_HELLO_VERIFY_REQUIRED ) { + int send_ret; + MBEDTLS_SSL_DEBUG_MSG( 1, ( "sending HelloVerifyRequest" ) ); + MBEDTLS_SSL_DEBUG_BUF( 4, "output record sent to network", + ssl->out_buf, len ); /* Don't check write errors as we can't do anything here. * If the error is permanent we'll catch it later, * if it's not, then hopefully it'll work next time. */ - (void) ssl->f_send( ssl->p_bio, ssl->out_buf, len ); - ret = 0; + send_ret = ssl->f_send( ssl->p_bio, ssl->out_buf, len ); + MBEDTLS_SSL_DEBUG_RET( 2, "ssl->f_send", send_ret ); + (void) send_ret; + + return( 0 ); } if( ret == 0 ) { - /* Got a valid cookie, partially reset context */ + MBEDTLS_SSL_DEBUG_MSG( 1, ( "cookie is valid, resetting context" ) ); if( ( ret = mbedtls_ssl_session_reset_int( ssl, 1 ) ) != 0 ) { MBEDTLS_SSL_DEBUG_RET( 1, "reset", ret ); @@ -4415,6 +4425,7 @@ static int ssl_get_next_record( mbedtls_ssl_context *ssl ) ssl->in_msglen = rec.data_len; ret = ssl_check_client_reconnect( ssl ); + MBEDTLS_SSL_DEBUG_RET( 2, "ssl_check_client_reconnect", ret ); if( ret != 0 ) return( ret ); #endif diff --git a/library/ssl_srv.c b/library/ssl_srv.c index 469c67eec932..006bc69c300a 100644 --- a/library/ssl_srv.c +++ b/library/ssl_srv.c @@ -3812,7 +3812,7 @@ static int ssl_parse_client_psk_identity( mbedtls_ssl_context *ssl, unsigned cha const unsigned char *end ) { int ret = 0; - size_t n; + uint16_t n; if( ssl_conf_has_psk_or_cb( ssl->conf ) == 0 ) { @@ -3832,7 +3832,7 @@ static int ssl_parse_client_psk_identity( mbedtls_ssl_context *ssl, unsigned cha n = ( (*p)[0] << 8 ) | (*p)[1]; *p += 2; - if( n < 1 || n > 65535 || n > (size_t) ( end - *p ) ) + if( n == 0 || n > end - *p ) { MBEDTLS_SSL_DEBUG_MSG( 1, ( "bad client key exchange message" ) ); return( MBEDTLS_ERR_SSL_BAD_HS_CLIENT_KEY_EXCHANGE ); diff --git a/library/x509.c b/library/x509.c index 7f8181be2733..c451332c2873 100644 --- a/library/x509.c +++ b/library/x509.c @@ -1064,7 +1064,7 @@ int mbedtls_x509_self_test( int verbose ) mbedtls_x509_crt_free( &clicert ); #else ((void) verbose); -#endif /* MBEDTLS_CERTS_C && MBEDTLS_SHA1_C */ +#endif /* MBEDTLS_CERTS_C && MBEDTLS_SHA256_C */ return( ret ); } diff --git a/programs/test/udp_proxy.c b/programs/test/udp_proxy.c index 979910e6bc8d..7447571f4fb3 100644 --- a/programs/test/udp_proxy.c +++ b/programs/test/udp_proxy.c @@ -133,6 +133,7 @@ int main( void ) " modifying CID in first instance of the packet.\n" \ " protect_hvr=0/1 default: 0 (don't protect HelloVerifyRequest)\n" \ " protect_len=%%d default: (don't protect packets of this size)\n" \ + " inject_clihlo=0/1 default: 0 (don't inject fake ClientHello)\n" \ "\n" \ " seed=%%d default: (use current time)\n" \ USAGE_PACK \ @@ -166,6 +167,7 @@ static struct options unsigned bad_cid; /* inject corrupted CID record */ int protect_hvr; /* never drop or delay HelloVerifyRequest */ int protect_len; /* never drop/delay packet of the given size*/ + int inject_clihlo; /* inject fake ClientHello after handshake */ unsigned pack; /* merge packets into single datagram for * at most \c merge milliseconds if > 0 */ unsigned int seed; /* seed for "random" events */ @@ -314,6 +316,12 @@ static void get_options( int argc, char *argv[] ) if( opt.protect_len < 0 ) exit_usage( p, q ); } + else if( strcmp( p, "inject_clihlo" ) == 0 ) + { + opt.inject_clihlo = atoi( q ); + if( opt.inject_clihlo < 0 || opt.inject_clihlo > 1 ) + exit_usage( p, q ); + } else if( strcmp( p, "seed" ) == 0 ) { opt.seed = atoi( q ); @@ -523,11 +531,41 @@ void print_packet( const packet *p, const char *why ) fflush( stdout ); } +/* + * In order to test the server's behaviour when receiving a ClientHello after + * the connection is established (this could be a hard reset from the client, + * but the server must not drop the existing connection before establishing + * client reachability, see RFC 6347 Section 4.2.8), we memorize the first + * ClientHello we see (which can't have a cookie), then replay it after the + * first ApplicationData record - then we're done. + * + * This is controlled by the inject_clihlo option. + * + * We want an explicit state and a place to store the packet. + */ +typedef enum { + ICH_INIT, /* haven't seen the first ClientHello yet */ + ICH_CACHED, /* cached the initial ClientHello */ + ICH_INJECTED, /* ClientHello already injected, done */ +} inject_clihlo_state_t; + +static inject_clihlo_state_t inject_clihlo_state; +static packet initial_clihlo; + int send_packet( const packet *p, const char *why ) { int ret; mbedtls_net_context *dst = p->dst; + /* save initial ClientHello? */ + if( opt.inject_clihlo != 0 && + inject_clihlo_state == ICH_INIT && + strcmp( p->type, "ClientHello" ) == 0 ) + { + memcpy( &initial_clihlo, p, sizeof( packet ) ); + inject_clihlo_state = ICH_CACHED; + } + /* insert corrupted CID record? */ if( opt.bad_cid != 0 && strcmp( p->type, "CID" ) == 0 && @@ -592,6 +630,23 @@ int send_packet( const packet *p, const char *why ) } } + /* Inject ClientHello after first ApplicationData */ + if( opt.inject_clihlo != 0 && + inject_clihlo_state == ICH_CACHED && + strcmp( p->type, "ApplicationData" ) == 0 ) + { + print_packet( &initial_clihlo, "injected" ); + + if( ( ret = dispatch_data( dst, initial_clihlo.buf, + initial_clihlo.len ) ) <= 0 ) + { + mbedtls_printf( " ! dispatch returned %d\n", ret ); + return( ret ); + } + + inject_clihlo_state = ICH_INJECTED; + } + return( 0 ); } diff --git a/scripts/assemble_changelog.py b/scripts/assemble_changelog.py index c868a6c7e322..ffa3f161b8ec 100755 --- a/scripts/assemble_changelog.py +++ b/scripts/assemble_changelog.py @@ -36,7 +36,7 @@ # This file is part of Mbed TLS (https://tls.mbed.org) import argparse -from collections import OrderedDict +from collections import OrderedDict, namedtuple import datetime import functools import glob @@ -51,51 +51,149 @@ def __init__(self, filename, line_number, message, *args, **kwargs): message.format(*args, **kwargs)) super().__init__(message) +class CategoryParseError(Exception): + def __init__(self, line_offset, error_message): + self.line_offset = line_offset + self.error_message = error_message + super().__init__('{}: {}'.format(line_offset, error_message)) + class LostContent(Exception): def __init__(self, filename, line): message = ('Lost content from {}: "{}"'.format(filename, line)) super().__init__(message) -STANDARD_SECTIONS = ( - b'Interface changes', +# The category names we use in the changelog. +# If you edit this, update ChangeLog.d/README.md. +STANDARD_CATEGORIES = ( + b'API changes', b'Default behavior changes', b'Requirement changes', b'New deprecations', b'Removals', - b'New features', + b'Features', b'Security', - b'Bug fixes', - b'Performance improvements', - b'Other changes', + b'Bugfix', + b'Changes', ) -class ChangeLog: - """An Mbed TLS changelog. +CategoryContent = namedtuple('CategoryContent', [ + 'name', 'title_line', # Title text and line number of the title + 'body', 'body_line', # Body text and starting line number of the body +]) - A changelog is a file in Markdown format. Each level 2 section title - starts a version, and versions are sorted in reverse chronological - order. Lines with a level 2 section title must start with '##'. +class ChangelogFormat: + """Virtual class documenting how to write a changelog format class.""" - Within a version, there are multiple sections, each devoted to a kind - of change: bug fix, feature request, etc. Section titles should match - entries in STANDARD_SECTIONS exactly. + @classmethod + def extract_top_version(cls, changelog_file_content): + """Split out the top version section. - Within each section, each separate change should be on a line starting - with a '*' bullet. There may be blank lines surrounding titles, but - there should not be any blank line inside a section. - """ + If the top version is already released, create a new top + version section for an unreleased version. + + Return ``(header, top_version_title, top_version_body, trailer)`` + where the "top version" is the existing top version section if it's + for unreleased changes, and a newly created section otherwise. + To assemble the changelog after modifying top_version_body, + concatenate the four pieces. + """ + raise NotImplementedError + + @classmethod + def version_title_text(cls, version_title): + """Return the text of a formatted version section title.""" + raise NotImplementedError - _title_re = re.compile(br'#*') - def title_level(self, line): - """Determine whether the line is a title. + @classmethod + def split_categories(cls, version_body): + """Split a changelog version section body into categories. - Return (level, content) where level is the Markdown section level - (1 for '#', 2 for '##', etc.) and content is the section title - without leading or trailing whitespace. For a non-title line, - the level is 0. + Return a list of `CategoryContent` the name is category title + without any formatting. """ - level = re.match(self._title_re, line).end() - return level, line[level:].strip() + raise NotImplementedError + + @classmethod + def format_category(cls, title, body): + """Construct the text of a category section from its title and body.""" + raise NotImplementedError + +class TextChangelogFormat(ChangelogFormat): + """The traditional Mbed TLS changelog format.""" + + _unreleased_version_text = b'= mbed TLS x.x.x branch released xxxx-xx-xx' + @classmethod + def is_released_version(cls, title): + # Look for an incomplete release date + return not re.search(br'[0-9x]{4}-[0-9x]{2}-[0-9x]?x', title) + + _top_version_re = re.compile(br'(?:\A|\n)(=[^\n]*\n+)(.*?\n)(?:=|$)', + re.DOTALL) + @classmethod + def extract_top_version(cls, changelog_file_content): + """A version section starts with a line starting with '='.""" + m = re.search(cls._top_version_re, changelog_file_content) + top_version_start = m.start(1) + top_version_end = m.end(2) + top_version_title = m.group(1) + top_version_body = m.group(2) + if cls.is_released_version(top_version_title): + top_version_end = top_version_start + top_version_title = cls._unreleased_version_text + b'\n\n' + top_version_body = b'' + return (changelog_file_content[:top_version_start], + top_version_title, top_version_body, + changelog_file_content[top_version_end:]) + + @classmethod + def version_title_text(cls, version_title): + return re.sub(br'\n.*', version_title, re.DOTALL) + + _category_title_re = re.compile(br'(^\w.*)\n+', re.MULTILINE) + @classmethod + def split_categories(cls, version_body): + """A category title is a line with the title in column 0.""" + if not version_body: + return [] + title_matches = list(re.finditer(cls._category_title_re, version_body)) + if not title_matches or title_matches[0].start() != 0: + # There is junk before the first category. + raise CategoryParseError(0, 'Junk found where category expected') + title_starts = [m.start(1) for m in title_matches] + body_starts = [m.end(0) for m in title_matches] + body_ends = title_starts[1:] + [len(version_body)] + bodies = [version_body[body_start:body_end].rstrip(b'\n') + b'\n' + for (body_start, body_end) in zip(body_starts, body_ends)] + title_lines = [version_body[:pos].count(b'\n') for pos in title_starts] + body_lines = [version_body[:pos].count(b'\n') for pos in body_starts] + return [CategoryContent(title_match.group(1), title_line, + body, body_line) + for title_match, title_line, body, body_line + in zip(title_matches, title_lines, bodies, body_lines)] + + @classmethod + def format_category(cls, title, body): + # `split_categories` ensures that each body ends with a newline. + # Make sure that there is additionally a blank line between categories. + if not body.endswith(b'\n\n'): + body += b'\n' + return title + b'\n' + body + +class ChangeLog: + """An Mbed TLS changelog. + + A changelog file consists of some header text followed by one or + more version sections. The version sections are in reverse + chronological order. Each version section consists of a title and a body. + + The body of a version section consists of zero or more category + subsections. Each category subsection consists of a title and a body. + + A changelog entry file has the same format as the body of a version section. + + A `ChangelogFormat` object defines the concrete syntax of the changelog. + Entry files must have the same format as the changelog file. + """ # Only accept dotted version numbers (e.g. "3.1", not "3"). # Refuse ".x" in a version number where x is a letter: this indicates @@ -103,135 +201,59 @@ def title_level(self, line): _version_number_re = re.compile(br'[0-9]+\.[0-9A-Za-z.]+') _incomplete_version_number_re = re.compile(br'.*\.[A-Za-z]') - def section_is_released_version(self, title): - """Whether this section is for a released version. - - True if the given level-2 section title indicates that this section - contains released changes, otherwise False. - """ - # Assume that a released version has a numerical version number - # that follows a particular pattern. These criteria may be revised - # as needed in future versions of this script. - version_number = re.search(self._version_number_re, title) - if version_number: - return not re.search(self._incomplete_version_number_re, - version_number.group(0)) - else: - return False - - def unreleased_version_title(self): - """The title to use if creating a new section for an unreleased version.""" - # pylint: disable=no-self-use; this method may be overridden - return b'Unreleased changes' - - def __init__(self, input_stream): + def add_categories_from_text(self, filename, line_offset, + text, allow_unknown_category): + """Parse a version section or entry file.""" + try: + categories = self.format.split_categories(text) + except CategoryParseError as e: + raise InputFormatError(filename, line_offset + e.line_offset, + e.error_message) + for category in categories: + if not allow_unknown_category and \ + category.name not in self.categories: + raise InputFormatError(filename, + line_offset + category.title_line, + 'Unknown category: "{}"', + category.name.decode('utf8')) + self.categories[category.name] += category.body + + def __init__(self, input_stream, changelog_format): """Create a changelog object. Populate the changelog object from the content of the file - input_stream. This is typically a file opened for reading, but - can be any generator returning the lines to read. + input_stream. """ - # Content before the level-2 section where the new entries are to be - # added. - self.header = [] - # Content of the level-3 sections of where the new entries are to - # be added. - self.section_content = OrderedDict() - for section in STANDARD_SECTIONS: - self.section_content[section] = [] - # Content of level-2 sections for already-released versions. - self.trailer = [] - self.read_main_file(input_stream) - - def read_main_file(self, input_stream): - """Populate the changelog object from the content of the file. - - This method is only intended to be called as part of the constructor - of the class and may not act sensibly on an object that is already - partially populated. - """ - # Parse the first level-2 section, containing changelog entries - # for unreleased changes. - # If we'll be expanding this section, everything before the first - # level-3 section title ("###...") following the first level-2 - # section title ("##...") is passed through as the header - # and everything after the second level-2 section title is passed - # through as the trailer. Inside the first level-2 section, - # split out the level-3 sections. - # If we'll be creating a new version, the header is everything - # before the point where we want to add the level-2 section - # for this version, and the trailer is what follows. - level_2_seen = 0 - current_section = None - for line in input_stream: - level, content = self.title_level(line) - if level == 2: - level_2_seen += 1 - if level_2_seen == 1: - if self.section_is_released_version(content): - self.header.append(b'## ' + - self.unreleased_version_title() + - b'\n\n') - level_2_seen = 2 - elif level == 3 and level_2_seen == 1: - current_section = content - self.section_content.setdefault(content, []) - if level_2_seen == 1 and current_section is not None: - if level != 3 and line.strip(): - self.section_content[current_section].append(line) - elif level_2_seen <= 1: - self.header.append(line) - else: - self.trailer.append(line) + self.format = changelog_format + whole_file = input_stream.read() + (self.header, + self.top_version_title, top_version_body, + self.trailer) = self.format.extract_top_version(whole_file) + # Split the top version section into categories. + self.categories = OrderedDict() + for category in STANDARD_CATEGORIES: + self.categories[category] = b'' + offset = (self.header + self.top_version_title).count(b'\n') + 1 + self.add_categories_from_text(input_stream.name, offset, + top_version_body, True) def add_file(self, input_stream): """Add changelog entries from a file. - - Read lines from input_stream, which is typically a file opened - for reading. These lines must contain a series of level 3 - Markdown sections with recognized titles. The corresponding - content is injected into the respective sections in the changelog. - The section titles must be either one of the hard-coded values - in STANDARD_SECTIONS in assemble_changelog.py or already present - in ChangeLog.md. Section titles must match byte-for-byte except that - leading or trailing whitespace is ignored. """ - filename = input_stream.name - current_section = None - for line_number, line in enumerate(input_stream, 1): - if not line.strip(): - continue - level, content = self.title_level(line) - if level == 3: - current_section = content - if current_section not in self.section_content: - raise InputFormatError(filename, line_number, - 'Section {} is not recognized', - str(current_section)[1:]) - elif level == 0: - if current_section is None: - raise InputFormatError(filename, line_number, - 'Missing section title at the beginning of the file') - self.section_content[current_section].append(line) - else: - raise InputFormatError(filename, line_number, - 'Only level 3 headers (###) are permitted') + self.add_categories_from_text(input_stream.name, 1, + input_stream.read(), False) def write(self, filename): """Write the changelog to the specified file. """ with open(filename, 'wb') as out: - for line in self.header: - out.write(line) - for section, lines in self.section_content.items(): - if not lines: + out.write(self.header) + out.write(self.top_version_title) + for title, body in self.categories.items(): + if not body: continue - out.write(b'### ' + section + b'\n\n') - for line in lines: - out.write(line) - out.write(b'\n') - for line in self.trailer: - out.write(line) + out.write(self.format.format_category(title, body)) + out.write(self.trailer) @functools.total_ordering @@ -403,7 +425,7 @@ def list_files_to_merge(options): "Oldest" is defined by `EntryFileSortKey`. """ - files_to_merge = glob.glob(os.path.join(options.dir, '*.md')) + files_to_merge = glob.glob(os.path.join(options.dir, '*.txt')) files_to_merge.sort(key=EntryFileSortKey) return files_to_merge @@ -416,7 +438,7 @@ def merge_entries(options): Remove the merged entries if options.keep_entries is false. """ with open(options.input, 'rb') as input_file: - changelog = ChangeLog(input_file) + changelog = ChangeLog(input_file, TextChangelogFormat) files_to_merge = list_files_to_merge(options) if not files_to_merge: sys.stderr.write('There are no pending changelog entries.\n') @@ -454,9 +476,9 @@ def main(): help='Directory to read entries from' ' (default: ChangeLog.d)') parser.add_argument('--input', '-i', metavar='FILE', - default='ChangeLog.md', + default='ChangeLog', help='Existing changelog file to read from and augment' - ' (default: ChangeLog.md)') + ' (default: ChangeLog)') parser.add_argument('--keep-entries', action='store_true', dest='keep_entries', default=None, help='Keep the files containing entries' @@ -470,7 +492,7 @@ def main(): ' (default: overwrite the input)') parser.add_argument('--list-files-only', action='store_true', - help=('Only list the files that would be processed' + help=('Only list the files that would be processed ' '(with some debugging information)')) options = parser.parse_args() set_defaults(options) diff --git a/tests/ssl-opt.sh b/tests/ssl-opt.sh index 35f742f67784..9b6eee1529bb 100755 --- a/tests/ssl-opt.sh +++ b/tests/ssl-opt.sh @@ -7279,8 +7279,8 @@ run_test "DTLS cookie: enabled, nbio" \ not_with_valgrind # spurious resend run_test "DTLS client reconnect from same port: reference" \ - "$P_SRV dtls=1 exchanges=2 read_timeout=1000" \ - "$P_CLI dtls=1 exchanges=2 debug_level=2 hs_timeout=500-1000" \ + "$P_SRV dtls=1 exchanges=2 read_timeout=20000 hs_timeout=10000-20000" \ + "$P_CLI dtls=1 exchanges=2 debug_level=2 hs_timeout=10000-20000" \ 0 \ -C "resend" \ -S "The operation timed out" \ @@ -7288,8 +7288,8 @@ run_test "DTLS client reconnect from same port: reference" \ not_with_valgrind # spurious resend run_test "DTLS client reconnect from same port: reconnect" \ - "$P_SRV dtls=1 exchanges=2 read_timeout=1000" \ - "$P_CLI dtls=1 exchanges=2 debug_level=2 hs_timeout=500-1000 reconnect_hard=1" \ + "$P_SRV dtls=1 exchanges=2 read_timeout=20000 hs_timeout=10000-20000" \ + "$P_CLI dtls=1 exchanges=2 debug_level=2 hs_timeout=10000-20000 reconnect_hard=1" \ 0 \ -C "resend" \ -S "The operation timed out" \ @@ -7318,6 +7318,14 @@ run_test "DTLS client reconnect from same port: no cookies" \ -s "The operation timed out" \ -S "Client initiated reconnection from same port" +run_test "DTLS client reconnect from same port: attacker-injected" \ + -p "$P_PXY inject_clihlo=1" \ + "$P_SRV dtls=1 exchanges=2 debug_level=1" \ + "$P_CLI dtls=1 exchanges=2" \ + 0 \ + -s "possible client reconnect from the same port" \ + -S "Client initiated reconnection from same port" + # Tests for various cases of client authentication with DTLS # (focused on handshake flows and message parsing) @@ -8387,8 +8395,8 @@ run_test "DTLS fragmenting: 3d, openssl client, DTLS 1.0" \ not_with_valgrind # spurious resend due to timeout run_test "DTLS proxy: reference" \ -p "$P_PXY" \ - "$P_SRV dtls=1 debug_level=2" \ - "$P_CLI dtls=1 debug_level=2" \ + "$P_SRV dtls=1 debug_level=2 hs_timeout=10000-20000" \ + "$P_CLI dtls=1 debug_level=2 hs_timeout=10000-20000" \ 0 \ -C "replayed record" \ -S "replayed record" \ @@ -8405,8 +8413,8 @@ run_test "DTLS proxy: reference" \ not_with_valgrind # spurious resend due to timeout run_test "DTLS proxy: duplicate every packet" \ -p "$P_PXY duplicate=1" \ - "$P_SRV dtls=1 dgram_packing=0 debug_level=2" \ - "$P_CLI dtls=1 dgram_packing=0 debug_level=2" \ + "$P_SRV dtls=1 dgram_packing=0 debug_level=2 hs_timeout=10000-20000" \ + "$P_CLI dtls=1 dgram_packing=0 debug_level=2 hs_timeout=10000-20000" \ 0 \ -c "replayed record" \ -s "replayed record" \