Skip to content

Commit

Permalink
[!] bugfix on multipath, add scheduler according to path quality and …
Browse files Browse the repository at this point in the history 
…path status (#370)

* [!] fix compile error on macos

* [!] downgrade the required version of libevent

* [!] fix compability with elder versions

* [=] delete duplicated macro

* [=] code format

* [!] fix sending duplicated ACK when no pkts received on Standby paths.
[+] reinject frames which will affect stream delivery.
[~] optimize mpquic scheduler according to path qulity and path status.
[!] fix sending RESET_STREAM  and STOP_SENDING frames in Initial and Handshake packets.
[!] fix delivery priority of HANDSHAKE_DONE frame blocked by server's STREAM frames responding to 0RTT requests.
[!] fix CID rotation when multipath is enabled.
  • Loading branch information
@Kulsk
Kulsk authored Nov 16, 2023
1 parent b610a83 commit b354ac7
Show file tree
Hide file tree
Showing 28 changed files with 594 additions and 248 deletions.
2 changes: 1 addition & 1 deletion demo/demo_client.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1926,7 +1926,7 @@ xqc_demo_cli_parse_args(int argc, char *argv[], xqc_demo_cli_client_args_t *args
printf("option multipath on\n");
args->net_cfg.multipath = 1;
break;

case 'o':
printf("set interop mode\n");
args->quic_cfg.is_interop_mode = 1;
Expand Down
10 changes: 5 additions & 5 deletions include/xquic/xqc_configure.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@
#define xquic_VERSION_MAJOR 0
#define xquic_VERSION_MINOR 1

#define XQC_ENABLE_BBR2
#define XQC_ENABLE_RENO
#define XQC_ENABLE_COPA
#define XQC_ENABLE_UNLIMITED
#define XQC_ENABLE_MP_INTEROP
/* #undef XQC_ENABLE_BBR2 */
/* #undef XQC_ENABLE_RENO */
/* #undef XQC_ENABLE_COPA */
/* #undef XQC_ENABLE_UNLIMITED */
/* #undef XQC_ENABLE_MP_INTEROP */
/* #undef XQC_NO_PID_PACKET_PROCESS */
23 changes: 9 additions & 14 deletions include/xquic/xquic.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -838,20 +838,12 @@ typedef struct xqc_cc_params_s {
} xqc_cc_params_t;

typedef struct xqc_scheduler_params_u {
uint8_t customize_on;
union {
struct {
double lambda_max;
double lambda_min;
double lambda_init;
double lambda_inc;
double lambda_dec;
} blest_params;

struct {
double beta;
} ecf_params;
};
uint64_t rtt_us_thr_high;
uint64_t rtt_us_thr_low;
uint64_t bw_Bps_thr;
double loss_percent_thr_high;
double loss_percent_thr_low;
uint32_t pto_cnt_thr;
} xqc_scheduler_params_t;

typedef struct xqc_congestion_control_callback_s {
Expand Down Expand Up @@ -1330,6 +1322,9 @@ typedef struct xqc_conn_stats_s {
*/
int mp_state;

int total_rebind_count;
int total_rebind_valid;

xqc_path_metrics_t paths_info[XQC_MAX_PATHS_COUNT];
char conn_info[XQC_CONN_INFO_LEN];

Expand Down
7 changes: 2 additions & 5 deletions scripts/case_test.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3460,15 +3460,12 @@ clear_log
echo -e "h3_ext_finish_bytestream_during_transmission...\c"
./test_client -l d -T 2 -s 102400 -U 1 -Q 65535 -E -x 304 > stdlog
cli_res1=`grep ">>>>>>>> pass:1" stdlog | wc -l`
cli_res2=`grep "\[dgram\]|echo_check|same_content:yes|" stdlog`
cli_res3=(`grep "\[h3-dgram\]|recv_dgram_bytes:" stdlog | egrep -o ':[0-9]+' | egrep -o '[0-9]+'`)
cli_res4=(`grep "\[bytestream\]|bytes_sent:" stdlog | egrep -o ':[0-9]+' | egrep -o '[0-9]+'`)
cli_res5=`grep "\[bytestream\]|same_content:yes|" stdlog | wc -l`
cli_res6=`grep "send pure fin" clog`
errlog=`grep_err_log | grep -v "send data after fin sent"`
if [ "$cli_res1" == "1" ] && [ -n "$cli_res2" ] \
&& [ ${cli_res3[0]} -ge 102400 ] && [ ${cli_res3[1]} -ge 102400 ] \
&& [ ${cli_res4[0]} -ge 102400 ] && [ ${cli_res4[1]} -ge 102400 ] \
if [ "$cli_res1" == "1" ] \
&& [ ${cli_res4[0]} -eq 102400 ] && [ ${cli_res4[1]} -eq 102400 ] \
&& [ "$cli_res5" == "1" ] && [ -n "$cli_res6" ] && [ -z "$errlog" ]; then
echo ">>>>>>>> pass:1"
case_print_result "h3_ext_finish_bytestream_during_transmission" "pass"
Expand Down
4 changes: 2 additions & 2 deletions scripts/xquic.lds
View file
Original file line number Diff line number Diff line change
Expand Up @@ -116,8 +116,8 @@ XQUIC_VERS_1.0 {
xqc_conn_get_conn_settings_template;
xqc_conn_get_lastest_rtt;
xqc_log_enable;
xqc_h3_request_update_settings;
xqc_stream_update_settings;
xqc_h3_request_update_settings;
xqc_stream_update_settings;
local:
*;
};
8 changes: 8 additions & 0 deletions src/common/utils/ringarray/xqc_ring_array.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -239,3 +239,11 @@ xqc_rarray_resize(xqc_rarray_t *ra, uint64_t cap)

return XQC_OK;
}

void
xqc_rarray_reinit(xqc_rarray_t *ra)
{
xqc_memzero(ra->buf, ra->cap * ra->esize);
ra->count = 0;
ra->offset = 0;
}
2 changes: 2 additions & 0 deletions src/common/utils/ringarray/xqc_ring_array.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,4 +84,6 @@ xqc_int_t xqc_rarray_pop_from(xqc_rarray_t *ra, uint64_t idx);
*/
xqc_int_t xqc_rarray_resize(xqc_rarray_t *ra, uint64_t cap);

void xqc_rarray_reinit(xqc_rarray_t *ra);

#endif
31 changes: 5 additions & 26 deletions src/http3/xqc_h3_request.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -177,7 +177,7 @@ xqc_stream_info_print(xqc_h3_stream_t *h3_stream, xqc_request_stats_t *stats)
size_t cursor = 0, ret = 0;
int i;
int flag = 0;
uint32_t mp_settings = 0;
char mp_settings[XQC_MP_SETTINGS_STR_LEN] = {0};

if (h3c->conn->handshake_complete_time > 0) {
flag = 1;
Expand All @@ -187,32 +187,11 @@ xqc_stream_info_print(xqc_h3_stream_t *h3_stream, xqc_request_stats_t *stats)
flag |= 1 << 1;
}

if (h3c->conn->enable_multipath) {
mp_settings |= 1;
}

if (h3c->conn->local_settings.enable_multipath) {
mp_settings |= (1 << 1);
}

if (h3c->conn->remote_settings.enable_multipath) {
mp_settings |= (1 << 2);
}

if (h3c->conn->conn_settings.multipath_version == XQC_MULTIPATH_05) {
mp_settings |= (1 << 3);
}

if (h3c->conn->local_settings.multipath_version == XQC_MULTIPATH_05) {
mp_settings |= (1 << 4);
}

if (h3c->conn->remote_settings.multipath_version == XQC_MULTIPATH_05) {
mp_settings |= (1 << 5);
}
xqc_conn_encode_mp_settings(h3c->conn, mp_settings, XQC_MP_SETTINGS_STR_LEN);

ret = snprintf(buff, buff_size, "(%d,%"PRIu64",%d)#",
flag, h3_stream->recv_rate_limit, mp_settings);
ret = snprintf(buff, buff_size, "(%d,%"PRIu64",%s,%"PRIu64",%"PRIu64")#",
flag, h3_stream->recv_rate_limit, mp_settings,
h3_stream->send_offset, h3_stream->recv_offset);

cursor += ret;

Expand Down
2 changes: 2 additions & 0 deletions src/http3/xqc_h3_stream.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1908,6 +1908,8 @@ xqc_h3_stream_close_notify(xqc_stream_t *stream, void *user_data)
h3s->h3r->stream_fin_send_time = h3s->stream->stream_stats.local_fin_snd_time;
h3s->h3r->stream_fin_ack_time = h3s->stream->stream_stats.first_fin_ack_time;
h3s->h3r->stream_close_msg = h3s->stream->stream_close_msg;
h3s->send_offset = h3s->stream->stream_send_offset;
h3s->recv_offset = h3s->stream->stream_data_in.merged_offset_end;
}

if (h3s->h3_ext_bs && h3s->type == XQC_H3_STREAM_TYPE_BYTESTEAM) {
Expand Down
2 changes: 2 additions & 0 deletions src/http3/xqc_h3_stream.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,8 @@ typedef struct xqc_h3_stream_s {
uint32_t ref_cnt;

uint64_t recv_rate_limit;
uint64_t send_offset;
uint64_t recv_offset;

} xqc_h3_stream_t;

Expand Down
73 changes: 64 additions & 9 deletions src/transport/reinjection_control/xqc_reinj_deadline.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,10 +42,37 @@ xqc_deadline_reinj_ctl_init(void *reinj_ctl, xqc_connection_t *conn)
rctl->conn = conn;
}

static inline xqc_bool_t
xqc_deadline_reinj_check_packet(xqc_packet_out_t *po)
{
if (((po->po_frame_types & XQC_FRAME_BIT_STREAM)
|| (po->po_frame_types & XQC_FRAME_BIT_MAX_STREAM_DATA)
|| (po->po_frame_types & XQC_FRAME_BIT_RESET_STREAM)
|| (po->po_frame_types & XQC_FRAME_BIT_STOP_SENDING)
|| (po->po_frame_types & XQC_FRAME_BIT_MAX_STREAMS)
|| (po->po_frame_types & XQC_FRAME_BIT_MAX_DATA)
|| (po->po_frame_types & XQC_FRAME_BIT_DATA_BLOCKED)
|| (po->po_frame_types & XQC_FRAME_BIT_STREAM_DATA_BLOCKED)
|| (po->po_frame_types & XQC_FRAME_BIT_STREAMS_BLOCKED)
|| (po->po_frame_types & XQC_FRAME_BIT_CONNECTION_CLOSE))
&& !(po->po_flag & XQC_POF_NOT_REINJECT)
&& !(XQC_MP_PKT_REINJECTED(po))
&& (po->po_flag & XQC_POF_IN_FLIGHT))
{
return XQC_TRUE;
}

return XQC_FALSE;
}

static xqc_bool_t
xqc_deadline_reinj_can_reinject_before_sched(xqc_deadline_reinj_ctl_t *rctl,
xqc_packet_out_t *po)
{
if (!xqc_deadline_reinj_check_packet(po)) {
return XQC_FALSE;
}

xqc_connection_t *conn = rctl->conn;
xqc_usec_t now = xqc_monotonic_timestamp();
xqc_usec_t min_srtt = xqc_conn_get_min_srtt(conn, 0);
Expand All @@ -56,22 +83,47 @@ xqc_deadline_reinj_can_reinject_before_sched(xqc_deadline_reinj_ctl_t *rctl,
uint64_t lower_bound = conn->conn_settings.reinj_deadline_lower_bound;
double deadline = xqc_max(xqc_min(flexible, (double)hard), (double)lower_bound);

xqc_log(conn->log, XQC_LOG_DEBUG, "|deadline:%f|factor:%.4f|min_srtt:%ui|flexible:%f|hard:%ui|lower_bound:%ui|now:%ui|sent_time:%ui|frame:%s|",
deadline, factor, min_srtt, flexible, hard, lower_bound, now, po->po_sent_time, xqc_frame_type_2_str(po->po_frame_types));
xqc_log(conn->log, XQC_LOG_DEBUG,
"|deadline:%f|factor:%.4f|min_srtt:%ui|flexible:%f|hard:%ui|"
"lower_bound:%ui|now:%ui|sent_time:%ui|frame:%s|",
deadline, factor, min_srtt, flexible, hard,
lower_bound, now, po->po_sent_time,
xqc_frame_type_2_str(po->po_frame_types));

if (((po->po_frame_types & XQC_FRAME_BIT_STREAM)
|| (po->po_frame_types & XQC_FRAME_BIT_MAX_STREAM_DATA))
&& !(po->po_flag & XQC_POF_NOT_REINJECT)
&& !(XQC_MP_PKT_REINJECTED(po))
&& (po->po_flag & XQC_POF_IN_FLIGHT)
&& ((double)(now - po->po_sent_time) >= deadline))
{
if ((double)(now - po->po_sent_time) >= deadline) {
return XQC_TRUE;
}

return XQC_FALSE;
}


static xqc_bool_t
xqc_deadline_reinj_can_reinject_after_send(xqc_deadline_reinj_ctl_t *rctl,
xqc_packet_out_t *po)
{
if (!xqc_deadline_reinj_check_packet(po)) {
return XQC_FALSE;
}

xqc_connection_t *conn = rctl->conn;
xqc_path_ctx_t *path = NULL;
xqc_path_perf_class_t path_class;

path = xqc_conn_find_path_by_path_id(conn, po->po_path_id);

if (path) {
path_class = xqc_path_get_perf_class(path);
if (path_class == XQC_PATH_CLASS_STANDBY_LOW
|| path_class == XQC_PATH_CLASS_AVAILABLE_LOW)
{
return XQC_TRUE;
}
}

return XQC_FALSE;
}

static xqc_bool_t
xqc_deadline_reinj_can_reinject(void *ctl,
xqc_packet_out_t *po, xqc_reinjection_mode_t mode)
Expand All @@ -83,6 +135,9 @@ xqc_deadline_reinj_can_reinject(void *ctl,
case XQC_REINJ_UNACK_BEFORE_SCHED:
can_reinject = xqc_deadline_reinj_can_reinject_before_sched(rctl, po);
break;
case XQC_REINJ_UNACK_AFTER_SEND:
can_reinject = xqc_deadline_reinj_can_reinject_after_send(rctl, po);
break;
default:
can_reinject = XQC_FALSE;
break;
Expand Down
10 changes: 9 additions & 1 deletion src/transport/reinjection_control/xqc_reinj_default.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,15 @@ xqc_default_reinj_can_reinject_after_sched(xqc_default_reinj_ctl_t *rctl,

if (xqc_list_empty(&conn->conn_send_queue->sndq_send_packets)
&& ((po->po_frame_types & XQC_FRAME_BIT_STREAM)
|| (po->po_frame_types & XQC_FRAME_BIT_MAX_STREAM_DATA))
|| (po->po_frame_types & XQC_FRAME_BIT_MAX_STREAM_DATA)
|| (po->po_frame_types & XQC_FRAME_BIT_RESET_STREAM)
|| (po->po_frame_types & XQC_FRAME_BIT_STOP_SENDING)
|| (po->po_frame_types & XQC_FRAME_BIT_MAX_STREAMS)
|| (po->po_frame_types & XQC_FRAME_BIT_MAX_DATA)
|| (po->po_frame_types & XQC_FRAME_BIT_DATA_BLOCKED)
|| (po->po_frame_types & XQC_FRAME_BIT_STREAM_DATA_BLOCKED)
|| (po->po_frame_types & XQC_FRAME_BIT_STREAMS_BLOCKED)
|| (po->po_frame_types & XQC_FRAME_BIT_CONNECTION_CLOSE))
&& !(po->po_flag & XQC_POF_NOT_REINJECT)
&& !(XQC_MP_PKT_REINJECTED(po))
&& (po->po_flag & XQC_POF_IN_FLIGHT))
Expand Down
2 changes: 1 addition & 1 deletion src/transport/reinjection_control/xqc_reinj_dgram.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ xqc_dgram_reinj_can_reinject_after_send(xqc_dgram_reinj_ctl_t *rctl,
{
xqc_connection_t *conn = rctl->conn;

if ((po->po_frame_types & XQC_FRAME_BIT_DATAGRAM)
if ((po->po_frame_types & (XQC_FRAME_BIT_DATAGRAM | XQC_FRAME_BIT_CONNECTION_CLOSE))
&& !(po->po_flag & XQC_POF_NOT_REINJECT)
&& !(XQC_MP_PKT_REINJECTED(po))
&& (po->po_flag & XQC_POF_IN_FLIGHT))
Expand Down
Loading

0 comments on commit b354ac7

Please sign in to comment.