From 13d53bb1ff19c403956b1f1b9a2b680a4a7accd8 Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Mon, 27 Jul 2020 17:10:14 +0900 Subject: [PATCH 01/10] [refactor] build all the cleartext header bits before building the frames --- lib/quicly.c | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/lib/quicly.c b/lib/quicly.c index f04665c55..1e9a32781 100644 --- a/lib/quicly.c +++ b/lib/quicly.c @@ -2848,22 +2848,13 @@ static int commit_send_packet(quicly_conn_t *conn, quicly_send_context_t *s, enu s->dst = s->payload_buf.datagram + max_size; } - /* encode packet size, packet number, key-phase */ + /* encode packet size */ if (QUICLY_PACKET_IS_LONG_HEADER(*s->target.first_byte_at)) { uint16_t length = s->dst - s->dst_payload_from + s->target.cipher->aead->algo->tag_size + QUICLY_SEND_PN_SIZE; /* length is always 2 bytes, see _do_prepare_packet */ length |= 0x4000; quicly_encode16(s->dst_payload_from - QUICLY_SEND_PN_SIZE - 2, length); - } else { - if (conn->egress.packet_number >= conn->application->cipher.egress.key_update_pn.next) { - int ret; - if ((ret = update_1rtt_egress_key(conn)) != 0) - return ret; - } - if ((conn->application->cipher.egress.key_phase & 1) != 0) - *s->target.first_byte_at |= QUICLY_KEY_PHASE_BIT; } - quicly_encode16(s->dst_payload_from - QUICLY_SEND_PN_SIZE, (uint16_t)conn->egress.packet_number); /* encrypt the packet */ s->dst += s->target.cipher->aead->algo->tag_size; @@ -3013,8 +3004,16 @@ static int _do_allocate_frame(quicly_conn_t *conn, quicly_send_context_t *s, siz *s->dst++ = 0; } else { s->dst = emit_cid(s->dst, &conn->super.remote.cid_set.cids[0].cid); + /* set the key-phase bit, as well as updating 1rtt key if necessary */ + if (conn->egress.packet_number >= conn->application->cipher.egress.key_update_pn.next) { + int ret; + if ((ret = update_1rtt_egress_key(conn)) != 0) + return ret; + } + if ((conn->application->cipher.egress.key_phase & 1) != 0) + *s->target.first_byte_at |= QUICLY_KEY_PHASE_BIT; } - s->dst += QUICLY_SEND_PN_SIZE; /* space for PN bits, filled in at commit time */ + s->dst = quicly_encode16(s->dst, (uint16_t)conn->egress.packet_number); /* packet number */ s->dst_payload_from = s->dst; assert(s->target.cipher->aead != NULL); s->dst_end -= s->target.cipher->aead->algo->tag_size; From cf0fe6484a85642946df09c8912c35704cadb998 Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Mon, 27 Jul 2020 20:33:36 +0900 Subject: [PATCH 02/10] add support for DSR --- include/quicly.h | 15 ++++++++ lib/quicly.c | 94 +++++++++++++++++++++++++++++++++++++----------- 2 files changed, 88 insertions(+), 21 deletions(-) diff --git a/include/quicly.h b/include/quicly.h index 3e906d8cc..8b31ac73f 100644 --- a/include/quicly.h +++ b/include/quicly.h @@ -707,6 +707,16 @@ struct st_quicly_address_token_plaintext_t { } appdata; }; +typedef struct st_quicly_detached_send_packet_t { + ptls_cipher_suite_t *cipher; + const void *header_protection_secret; + const void *aead_secret; + const void *datagram; + size_t first_byte_at; + size_t payload_from; + uint64_t packet_number; +} quicly_detached_send_packet_t; + /** * returns a boolean indicating if given protocol version is supported */ @@ -821,6 +831,11 @@ int quicly_is_flow_capped(quicly_conn_t *conn); * @return a boolean indicating if quicly_send_stream can be called immediately */ int quicly_can_send_stream_data(quicly_conn_t *conn, quicly_send_context_t *s); +/** + * Detaches the packet being built, so that the stream data can be incorporate by a different process. This function can only be + * called from the quicly_stream_callbacks_t::on_send_emit callback. + */ +int quicly_stream_on_send_emit_detach_packet(quicly_detached_send_packet_t *detached); /** * Sends data of given stream. Called by stream scheduler. Only streams that can send some data or EOS should be specified. It is * the responsibilty of the stream scheduler to maintain a list of such streams. diff --git a/lib/quicly.c b/lib/quicly.c index 1e9a32781..95565d358 100644 --- a/lib/quicly.c +++ b/lib/quicly.c @@ -164,7 +164,7 @@ struct st_quicly_application_space_t { } ingress; struct { struct st_quicly_cipher_context_t key; - uint8_t secret[PTLS_MAX_DIGEST_SIZE]; + uint8_t aead_secret[PTLS_MAX_DIGEST_SIZE], header_protection_secret[PTLS_MAX_DIGEST_SIZE]; uint64_t key_phase; struct { /** @@ -1329,7 +1329,9 @@ static void free_application_space(struct st_quicly_application_space_t **space) #undef DISPOSE_INGRESS if ((*space)->cipher.egress.key.aead != NULL) dispose_cipher(&(*space)->cipher.egress.key); - ptls_clear_memory((*space)->cipher.egress.secret, sizeof((*space)->cipher.egress.secret)); + ptls_clear_memory((*space)->cipher.egress.aead_secret, sizeof((*space)->cipher.egress.aead_secret)); + ptls_clear_memory((*space)->cipher.egress.header_protection_secret, + sizeof((*space)->cipher.egress.header_protection_secret)); do_free_pn_space(&(*space)->super); *space = NULL; } @@ -1411,7 +1413,7 @@ static int update_1rtt_egress_key(quicly_conn_t *conn) int ret; /* generate next AEAD key, and increment key phase if it succeeds */ - if ((ret = update_1rtt_key(conn, cipher, 1, &space->cipher.egress.key.aead, space->cipher.egress.secret)) != 0) + if ((ret = update_1rtt_key(conn, cipher, 1, &space->cipher.egress.key.aead, space->cipher.egress.aead_secret)) != 0) return ret; ++space->cipher.egress.key_phase; @@ -1420,7 +1422,7 @@ static int update_1rtt_egress_key(quicly_conn_t *conn) space->cipher.egress.key_update_pn.next = UINT64_MAX; QUICLY_PROBE(CRYPTO_SEND_KEY_UPDATE, conn, conn->stash.now, space->cipher.egress.key_phase, - QUICLY_PROBE_HEXDUMP(space->cipher.egress.secret, cipher->hash->digest_size)); + QUICLY_PROBE_HEXDUMP(space->cipher.egress.aead_secret, cipher->hash->digest_size)); return 0; } @@ -2826,6 +2828,7 @@ enum en_quicly_send_packet_mode_t { QUICLY_COMMIT_SEND_PACKET_MODE_FULL_SIZE, QUICLY_COMMIT_SEND_PACKET_MODE_SMALL, QUICLY_COMMIT_SEND_PACKET_MODE_COALESCED, + QUICLY_COMMIT_SEND_PACKET_MODE_DETACHED }; static int commit_send_packet(quicly_conn_t *conn, quicly_send_context_t *s, enum en_quicly_send_packet_mode_t mode) @@ -2861,11 +2864,14 @@ static int commit_send_packet(quicly_conn_t *conn, quicly_send_context_t *s, enu datagram_size = s->dst - s->payload_buf.datagram; assert(datagram_size <= conn->egress.max_udp_payload_size); - conn->super.ctx->crypto_engine->encrypt_packet(conn->super.ctx->crypto_engine, conn, s->target.cipher->header_protection, - s->target.cipher->aead, ptls_iovec_init(s->payload_buf.datagram, datagram_size), - s->target.first_byte_at - s->payload_buf.datagram, - s->dst_payload_from - s->payload_buf.datagram, conn->egress.packet_number, - mode == QUICLY_COMMIT_SEND_PACKET_MODE_COALESCED); + if (mode != QUICLY_COMMIT_SEND_PACKET_MODE_DETACHED) { + conn->super.ctx->crypto_engine->encrypt_packet(conn->super.ctx->crypto_engine, conn, s->target.cipher->header_protection, + s->target.cipher->aead, + ptls_iovec_init(s->payload_buf.datagram, datagram_size), + s->target.first_byte_at - s->payload_buf.datagram, + s->dst_payload_from - s->payload_buf.datagram, conn->egress.packet_number, + mode == QUICLY_COMMIT_SEND_PACKET_MODE_COALESCED); + } /* update CC, commit sentmap */ if (s->target.ack_eliciting) { @@ -2887,8 +2893,12 @@ static int commit_send_packet(quicly_conn_t *conn, quicly_send_context_t *s, enu if (mode != QUICLY_COMMIT_SEND_PACKET_MODE_COALESCED) { conn->super.stats.num_bytes.sent += datagram_size; - s->datagrams[s->num_datagrams++] = (struct iovec){.iov_base = s->payload_buf.datagram, .iov_len = datagram_size}; - s->payload_buf.datagram += datagram_size; + if (mode == QUICLY_COMMIT_SEND_PACKET_MODE_DETACHED) { + --s->max_datagrams; + } else { + s->datagrams[s->num_datagrams++] = (struct iovec){.iov_base = s->payload_buf.datagram, .iov_len = datagram_size}; + s->payload_buf.datagram += datagram_size; + } s->target.cipher = NULL; s->target.first_byte_at = NULL; } @@ -3197,13 +3207,38 @@ int quicly_can_send_stream_data(quicly_conn_t *conn, quicly_send_context_t *s) return s->num_datagrams < s->max_datagrams; } +static __thread struct st_quicly_send_stream_detach_ctx_t { + quicly_conn_t *conn; + quicly_send_context_t *send_ctx; +} *send_stream_detach_ctx; /* becomes NULL when detached */ + +int quicly_stream_on_send_emit_detach_packet(quicly_detached_send_packet_t *detached) +{ + assert(send_stream_detach_ctx != NULL); + + quicly_conn_t *conn = send_stream_detach_ctx->conn; + quicly_send_context_t *s = send_stream_detach_ctx->send_ctx; + + detached->cipher = ptls_get_cipher(conn->crypto.tls); + detached->header_protection_secret = conn->application->cipher.egress.header_protection_secret; + detached->aead_secret = conn->application->cipher.egress.aead_secret; + detached->datagram = s->payload_buf.datagram; + detached->first_byte_at = s->target.first_byte_at - s->payload_buf.datagram; + detached->payload_from = s->dst_payload_from - s->payload_buf.datagram; + detached->packet_number = conn->egress.packet_number; + + send_stream_detach_ctx = NULL; + + return 0; +} + int quicly_send_stream(quicly_stream_t *stream, quicly_send_context_t *s) { uint64_t off = stream->sendstate.pending.ranges[0].start, end_off; quicly_sent_t *sent; uint8_t *frame_type_at; - size_t capacity, len; - int ret, wrote_all, is_fin; + size_t capacity; + int ret, wrote_all, is_fin, detached = 0; /* write frame type, stream_id and offset, calculate capacity */ if (stream->stream_id < 0) { @@ -3271,10 +3306,18 @@ int quicly_send_stream(quicly_stream_t *stream, quicly_send_context_t *s) /* write payload */ assert(capacity != 0); - len = capacity; - size_t emit_off = (size_t)(off - stream->sendstate.acked.ranges[0].end); + size_t emit_off = (size_t)(off - stream->sendstate.acked.ranges[0].end), len = capacity; QUICLY_PROBE(STREAM_ON_SEND_EMIT, stream->conn, stream->conn->stash.now, stream, emit_off, len); - stream->callbacks->on_send_emit(stream, emit_off, s->dst, &len, &wrote_all); + { + struct st_quicly_send_stream_detach_ctx_t detach_ctx = {stream->conn, s}; + send_stream_detach_ctx = &detach_ctx; + stream->callbacks->on_send_emit(stream, emit_off, s->dst, &len, &wrote_all); + if (send_stream_detach_ctx == NULL) { + detached = 1; + } else { + send_stream_detach_ctx = NULL; + } + } if (stream->conn->super.state >= QUICLY_STATE_CLOSING) { return QUICLY_ERROR_IS_CLOSING; } else if (stream->_send_aux.reset_stream.sender_state != QUICLY_SENDER_STATE_NONE) { @@ -3284,7 +3327,11 @@ int quicly_send_stream(quicly_stream_t *stream, quicly_send_context_t *s) assert(len != 0); /* update s->dst, insert length if necessary */ - if (frame_type_at == NULL || len < s->dst_end - s->dst) { + if (detached) { + /* detached, current frame is going to be the last frame */ + assert(frame_type_at != NULL); + } else if (frame_type_at == NULL || len < s->dst_end - s->dst) { + /* insert the length field, so that more frames can be added */ if (frame_type_at != NULL) *frame_type_at |= QUICLY_FRAME_TYPE_STREAM_BIT_LEN; size_t len_of_len = quicly_encodev_capacity(len); @@ -3329,6 +3376,12 @@ int quicly_send_stream(quicly_stream_t *stream, quicly_send_context_t *s) sent->data.stream.args.start = off; sent->data.stream.args.end = end_off + is_fin; + /* commit the packet immediately, if it has been detached */ + if (detached) { + if ((ret = commit_send_packet(stream->conn, s, QUICLY_COMMIT_SEND_PACKET_MODE_DETACHED)) != 0) + return ret; + } + return 0; } @@ -3843,18 +3896,17 @@ static int update_traffic_key_cb(ptls_update_traffic_key_t *self, ptls_t *tls, i return ret; if (conn->application == NULL && (ret = setup_application_space(conn)) != 0) return ret; - uint8_t *secret_store; if (is_enc) { if (conn->application->cipher.egress.key.aead != NULL) dispose_cipher(&conn->application->cipher.egress.key); SELECT_CIPHER_CONTEXT(&conn->application->cipher.egress.key); - secret_store = conn->application->cipher.egress.secret; + memcpy(conn->application->cipher.egress.aead_secret, secret, cipher->hash->digest_size); + memcpy(conn->application->cipher.egress.header_protection_secret, secret, cipher->hash->digest_size); } else { hp_slot = &conn->application->cipher.ingress.header_protection.one_rtt; aead_slot = &conn->application->cipher.ingress.aead[0]; - secret_store = conn->application->cipher.ingress.secret; + memcpy(conn->application->cipher.ingress.secret, secret, cipher->hash->digest_size); } - memcpy(secret_store, secret, cipher->hash->digest_size); } break; default: assert(!"logic flaw"); From f00fa6dc2f7823cd74df489178e60ad0365f2369 Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Sun, 2 Aug 2020 09:50:47 +0900 Subject: [PATCH 03/10] send fin separately when using detached mode, as it is impossible to set the bit once the packet image is being detached --- lib/quicly.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/quicly.c b/lib/quicly.c index 95565d358..1c02fd879 100644 --- a/lib/quicly.c +++ b/lib/quicly.c @@ -3349,8 +3349,13 @@ int quicly_send_stream(quicly_stream_t *stream, quicly_send_context_t *s) if (!quicly_sendstate_is_open(&stream->sendstate) && end_off == stream->sendstate.final_size) { assert(end_off + 1 == stream->sendstate.pending.ranges[stream->sendstate.pending.num_ranges - 1].end); assert(frame_type_at != NULL); - is_fin = 1; - *frame_type_at |= QUICLY_FRAME_TYPE_STREAM_BIT_FIN; + /* in case the frame is detached, we need to use a new packet for just carrying the fin bit */ + if (detached) { + is_fin = 0; + } else { + is_fin = 1; + *frame_type_at |= QUICLY_FRAME_TYPE_STREAM_BIT_FIN; + } } else { is_fin = 0; } From 63aabc0f4d418594bcc19476bf2a1dc8427920da Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Sun, 2 Aug 2020 09:51:12 +0900 Subject: [PATCH 04/10] guaranteed to suceed --- include/quicly.h | 2 +- lib/quicly.c | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/include/quicly.h b/include/quicly.h index 8b31ac73f..74dc0ddaf 100644 --- a/include/quicly.h +++ b/include/quicly.h @@ -835,7 +835,7 @@ int quicly_can_send_stream_data(quicly_conn_t *conn, quicly_send_context_t *s); * Detaches the packet being built, so that the stream data can be incorporate by a different process. This function can only be * called from the quicly_stream_callbacks_t::on_send_emit callback. */ -int quicly_stream_on_send_emit_detach_packet(quicly_detached_send_packet_t *detached); +void quicly_stream_on_send_emit_detach_packet(quicly_detached_send_packet_t *detached); /** * Sends data of given stream. Called by stream scheduler. Only streams that can send some data or EOS should be specified. It is * the responsibilty of the stream scheduler to maintain a list of such streams. diff --git a/lib/quicly.c b/lib/quicly.c index 1c02fd879..200475c0c 100644 --- a/lib/quicly.c +++ b/lib/quicly.c @@ -3212,12 +3212,12 @@ static __thread struct st_quicly_send_stream_detach_ctx_t { quicly_send_context_t *send_ctx; } *send_stream_detach_ctx; /* becomes NULL when detached */ -int quicly_stream_on_send_emit_detach_packet(quicly_detached_send_packet_t *detached) +void quicly_stream_on_send_emit_detach_packet(quicly_detached_send_packet_t *detached) { assert(send_stream_detach_ctx != NULL); - quicly_conn_t *conn = send_stream_detach_ctx->conn; quicly_send_context_t *s = send_stream_detach_ctx->send_ctx; + send_stream_detach_ctx = NULL; detached->cipher = ptls_get_cipher(conn->crypto.tls); detached->header_protection_secret = conn->application->cipher.egress.header_protection_secret; @@ -3226,10 +3226,6 @@ int quicly_stream_on_send_emit_detach_packet(quicly_detached_send_packet_t *deta detached->first_byte_at = s->target.first_byte_at - s->payload_buf.datagram; detached->payload_from = s->dst_payload_from - s->payload_buf.datagram; detached->packet_number = conn->egress.packet_number; - - send_stream_detach_ctx = NULL; - - return 0; } int quicly_send_stream(quicly_stream_t *stream, quicly_send_context_t *s) From 3378079404ed26a4b6a901cbc78f7d8e6187b9a4 Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Sun, 2 Aug 2020 15:34:19 +0900 Subject: [PATCH 05/10] use only one generation of 1-RTT traffic keys in one call to `quicly_send` --- lib/quicly.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/lib/quicly.c b/lib/quicly.c index 200475c0c..7f7f115ab 100644 --- a/lib/quicly.c +++ b/lib/quicly.c @@ -3014,12 +3014,7 @@ static int _do_allocate_frame(quicly_conn_t *conn, quicly_send_context_t *s, siz *s->dst++ = 0; } else { s->dst = emit_cid(s->dst, &conn->super.remote.cid_set.cids[0].cid); - /* set the key-phase bit, as well as updating 1rtt key if necessary */ - if (conn->egress.packet_number >= conn->application->cipher.egress.key_update_pn.next) { - int ret; - if ((ret = update_1rtt_egress_key(conn)) != 0) - return ret; - } + /* set the key-phase bit */ if ((conn->application->cipher.egress.key_phase & 1) != 0) *s->target.first_byte_at |= QUICLY_KEY_PHASE_BIT; } @@ -3994,7 +3989,16 @@ static int do_send(quicly_conn_t *conn, quicly_send_context_t *s) /* send encrypted frames */ if (conn->application != NULL && (s->current.cipher = &conn->application->cipher.egress.key)->header_protection != NULL) { + /* setup context */ s->current.first_byte = conn->application->one_rtt_writable ? QUICLY_QUIC_BIT : QUICLY_PACKET_TYPE_0RTT; + /* Update 1-RTT traffic keys if necessary. Doing it here guarantees that all 1-RTT packets being generated use the same + * set of keys (and reduces the amount of checks that the detached mode handlers need to employ for handling potential key + * updates). */ + if (conn->egress.packet_number >= conn->application->cipher.egress.key_update_pn.next) { + assert(conn->application->one_rtt_writable); + if ((ret = update_1rtt_egress_key(conn)) != 0) + goto Exit; + } /* acks */ if (conn->application->one_rtt_writable && conn->egress.send_ack_at <= conn->stash.now && conn->application->super.unacked_count != 0) { From 21a03942435c6f2871234c0388aa2249a19b1fb1 Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Sun, 2 Aug 2020 15:41:22 +0900 Subject: [PATCH 06/10] clang-format --- lib/quicly.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/lib/quicly.c b/lib/quicly.c index 7f7f115ab..44d3cba00 100644 --- a/lib/quicly.c +++ b/lib/quicly.c @@ -2865,12 +2865,11 @@ static int commit_send_packet(quicly_conn_t *conn, quicly_send_context_t *s, enu assert(datagram_size <= conn->egress.max_udp_payload_size); if (mode != QUICLY_COMMIT_SEND_PACKET_MODE_DETACHED) { - conn->super.ctx->crypto_engine->encrypt_packet(conn->super.ctx->crypto_engine, conn, s->target.cipher->header_protection, - s->target.cipher->aead, - ptls_iovec_init(s->payload_buf.datagram, datagram_size), - s->target.first_byte_at - s->payload_buf.datagram, - s->dst_payload_from - s->payload_buf.datagram, conn->egress.packet_number, - mode == QUICLY_COMMIT_SEND_PACKET_MODE_COALESCED); + conn->super.ctx->crypto_engine->encrypt_packet( + conn->super.ctx->crypto_engine, conn, s->target.cipher->header_protection, s->target.cipher->aead, + ptls_iovec_init(s->payload_buf.datagram, datagram_size), s->target.first_byte_at - s->payload_buf.datagram, + s->dst_payload_from - s->payload_buf.datagram, conn->egress.packet_number, + mode == QUICLY_COMMIT_SEND_PACKET_MODE_COALESCED); } /* update CC, commit sentmap */ @@ -3205,7 +3204,7 @@ int quicly_can_send_stream_data(quicly_conn_t *conn, quicly_send_context_t *s) static __thread struct st_quicly_send_stream_detach_ctx_t { quicly_conn_t *conn; quicly_send_context_t *send_ctx; -} *send_stream_detach_ctx; /* becomes NULL when detached */ +} * send_stream_detach_ctx; /* becomes NULL when detached */ void quicly_stream_on_send_emit_detach_packet(quicly_detached_send_packet_t *detached) { From 96c6b7ca3514247902366f9c493e3e1fe99c9529 Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Mon, 10 Aug 2020 13:57:45 +0900 Subject: [PATCH 07/10] add function for obtaining the protocol version of the connection --- include/quicly.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/include/quicly.h b/include/quicly.h index 74dc0ddaf..ef36dadd4 100644 --- a/include/quicly.h +++ b/include/quicly.h @@ -796,6 +796,10 @@ static struct sockaddr *quicly_get_peername(quicly_conn_t *conn); * */ int quicly_get_stats(quicly_conn_t *conn, quicly_stats_t *stats); +/** + * + */ +static uint32_t quicly_get_version(quicly_conn_t *conn); /** * */ @@ -1155,6 +1159,12 @@ inline struct sockaddr *quicly_get_peername(quicly_conn_t *conn) return &c->remote.address.sa; } +inline uint32_t quicly_get_version(quicly_conn_t *conn) +{ + struct _st_quicly_conn_public_t *c = (struct _st_quicly_conn_public_t *)conn; + return c->version; +} + inline void **quicly_get_data(quicly_conn_t *conn) { struct _st_quicly_conn_public_t *c = (struct _st_quicly_conn_public_t *)conn; From 065fe7e1e29048315aa18d8c926eb2b035b3aa81 Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Thu, 25 Nov 2021 10:07:05 +0900 Subject: [PATCH 08/10] adjust the detached API so that packets can become full-sized when necessary --- include/quicly.h | 24 ++++++++++++++++---- lib/quicly.c | 59 ++++++++++++++++++++++++++++++++++-------------- 2 files changed, 61 insertions(+), 22 deletions(-) diff --git a/include/quicly.h b/include/quicly.h index afdc2b1d4..218aaf2a5 100644 --- a/include/quicly.h +++ b/include/quicly.h @@ -801,10 +801,19 @@ typedef struct st_quicly_detached_send_packet_t { ptls_cipher_suite_t *cipher; const void *header_protection_secret; const void *aead_secret; - const void *datagram; - size_t first_byte_at; - size_t payload_from; uint64_t packet_number; + /** + * the datagram being built partially + */ + ptls_iovec_t datagram; + /** + * beginning of the partialy-built QUIC packet + */ + uint16_t packet_from; + /** + * beginning of the AEAD payload of the partially-built QUIC packet + */ + uint16_t packet_payload_from; } quicly_detached_send_packet_t; /** @@ -950,9 +959,14 @@ int quicly_stream_can_send(quicly_stream_t *stream, int at_stream_level); int quicly_can_send_data(quicly_conn_t *conn, quicly_send_context_t *s); /** * Detaches the packet being built, so that the stream data can be incorporate by a different process. This function can only be - * called from the quicly_stream_callbacks_t::on_send_emit callback. + * called from the `quicly_stream_callbacks_t::on_send_emit` callback. + * @param dst dst being provided to the `on_send_emit` callback + * @param len length of the STREAM frame payload + * @param len_built first portion of the STREAM frame payload that has already been built + * */ -void quicly_stream_on_send_emit_detach_packet(quicly_detached_send_packet_t *detached); +void quicly_stream_on_send_emit_detach_packet(quicly_detached_send_packet_t *detached, quicly_stream_t *stream, uint8_t *dst, + size_t len, size_t len_built); /** * Sends data of given stream. Called by stream scheduler. Only streams that can send some data or EOS should be specified. It is * the responsibilty of the stream scheduler to maintain a list of such streams. diff --git a/lib/quicly.c b/lib/quicly.c index 0235c9230..44754edc8 100644 --- a/lib/quicly.c +++ b/lib/quicly.c @@ -3094,12 +3094,15 @@ static int commit_send_packet(quicly_conn_t *conn, quicly_send_context_t *s, enu *s->dst++ = QUICLY_FRAME_TYPE_PADDING; if (mode != QUICLY_COMMIT_SEND_PACKET_MODE_COALESCED && s->target.full_size) { - FIXME mode != QUICLY_COMMIT_SEND_PACKET_MODE_DETACHED; assert(s->num_datagrams == 0 || s->datagrams[s->num_datagrams - 1].iov_len == conn->egress.max_udp_payload_size); - const size_t max_size = conn->egress.max_udp_payload_size - QUICLY_AEAD_TAG_SIZE; - assert(s->dst - s->payload_buf.datagram <= max_size); - memset(s->dst, QUICLY_FRAME_TYPE_PADDING, s->payload_buf.datagram + max_size - s->dst); - s->dst = s->payload_buf.datagram + max_size; + size_t max_size = conn->egress.max_udp_payload_size - QUICLY_AEAD_TAG_SIZE; + if (mode == QUICLY_COMMIT_SEND_PACKET_MODE_DETACHED) { + assert(s->dst == s->payload_buf.datagram + max_size); + } else { + assert(s->dst <= s->payload_buf.datagram + max_size); + memset(s->dst, QUICLY_FRAME_TYPE_PADDING, s->payload_buf.datagram + max_size - s->dst); + s->dst = s->payload_buf.datagram + max_size; + } } /* encode packet size */ @@ -3560,25 +3563,47 @@ int quicly_can_send_data(quicly_conn_t *conn, quicly_send_context_t *s) return s->num_datagrams < s->max_datagrams; } +/** + * stream for which on_send_emit has been invoked; becomes NULL when detached + */ static __thread struct st_quicly_send_stream_detach_ctx_t { - quicly_conn_t *conn; quicly_send_context_t *send_ctx; -} * send_stream_detach_ctx; /* becomes NULL when detached */ + uint8_t *frame_type_at; +} * send_stream_detach_ctx; -void quicly_stream_on_send_emit_detach_packet(quicly_detached_send_packet_t *detached) +void quicly_stream_on_send_emit_detach_packet(quicly_detached_send_packet_t *detached, quicly_stream_t *stream, uint8_t *dst, + size_t len, size_t len_built) { assert(send_stream_detach_ctx != NULL); - quicly_conn_t *conn = send_stream_detach_ctx->conn; + + quicly_conn_t *conn = stream->conn; quicly_send_context_t *s = send_stream_detach_ctx->send_ctx; + + /* When the packet has to be padded to full size, and if the stream payload is too small, prepend PADDING frames and adjust + * `dst`. */ + if (s->target.full_size) { + uint8_t *dst_end = s->payload_buf.datagram + conn->egress.max_udp_payload_size - QUICLY_AEAD_TAG_SIZE; + assert(dst + len < dst_end); + size_t delta = dst_end - (dst + len); + if (delta != 0) { + uint8_t *frame_type_at = send_stream_detach_ctx->frame_type_at; + memmove(frame_type_at + delta, frame_type_at, (dst + len_built) - frame_type_at); + memset(frame_type_at, QUICLY_FRAME_TYPE_PADDING, delta); + dst += delta; + } + } + send_stream_detach_ctx = NULL; - detached->cipher = ptls_get_cipher(conn->crypto.tls); - detached->header_protection_secret = conn->application->cipher.egress.header_protection_secret; - detached->aead_secret = conn->application->cipher.egress.aead_secret; - detached->datagram = s->payload_buf.datagram; - detached->first_byte_at = s->target.first_byte_at - s->payload_buf.datagram; - detached->payload_from = s->dst_payload_from - s->payload_buf.datagram; - detached->packet_number = conn->egress.packet_number; + *detached = (quicly_detached_send_packet_t){ + .cipher = ptls_get_cipher(conn->crypto.tls), + .header_protection_secret = conn->application->cipher.egress.header_protection_secret, + .aead_secret = conn->application->cipher.egress.aead_secret, + .packet_number = conn->egress.packet_number, + .datagram = ptls_iovec_init(s->payload_buf.datagram, (dst + len_built) - s->payload_buf.datagram), + .packet_from = s->target.first_byte_at - s->payload_buf.datagram, + .packet_payload_from = s->dst_payload_from - s->payload_buf.datagram, + }; } int quicly_send_stream(quicly_stream_t *stream, quicly_send_context_t *s) @@ -3658,7 +3683,7 @@ int quicly_send_stream(quicly_stream_t *stream, quicly_send_context_t *s) size_t emit_off = (size_t)(off - stream->sendstate.acked.ranges[0].end), len = capacity; QUICLY_PROBE(STREAM_ON_SEND_EMIT, stream->conn, stream->conn->stash.now, stream, emit_off, len); { - struct st_quicly_send_stream_detach_ctx_t detach_ctx = {stream->conn, s}; + struct st_quicly_send_stream_detach_ctx_t detach_ctx = {s, frame_type_at}; send_stream_detach_ctx = &detach_ctx; stream->callbacks->on_send_emit(stream, emit_off, s->dst, &len, &wrote_all); if (send_stream_detach_ctx == NULL) { From 51ad309cc44f4e35e79b329c0ace58d4ead4bcb2 Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Wed, 14 Sep 2022 12:27:29 +0900 Subject: [PATCH 09/10] calculate CWND correctly --- lib/quicly.c | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/quicly.c b/lib/quicly.c index 611a44e0f..837419e61 100644 --- a/lib/quicly.c +++ b/lib/quicly.c @@ -3751,6 +3751,7 @@ int quicly_send_stream(quicly_stream_t *stream, quicly_send_context_t *s) if (detached) { /* Detached: current frame is going to be the last frame, and FIN is to be sent separately, using a non-detached frame. */ + dst += len; is_fin = 0; } else { /* Not detached: adjust frame layout then adjust FIN. */ From c07a615592ef0aabe32fa0323cd8ca059acc30f2 Mon Sep 17 00:00:00 2001 From: Kazuho Oku Date: Wed, 14 Sep 2022 13:34:52 +0900 Subject: [PATCH 10/10] C++ requires that the orders match --- include/quicly/loss.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/quicly/loss.h b/include/quicly/loss.h index 2fac428f3..cbc307a5c 100644 --- a/include/quicly/loss.h +++ b/include/quicly/loss.h @@ -250,8 +250,8 @@ inline void quicly_loss_init(quicly_loss_t *r, const quicly_loss_conf_t *conf, u *r = (quicly_loss_t){.conf = conf, .max_ack_delay = max_ack_delay, .ack_delay_exponent = ack_delay_exponent, - .pto_count = 0, .thresholds = {.use_packet_based = 1, .time_based_percentile = 1024 / 8 /* start from 1/8 RTT */}, + .pto_count = 0, .time_of_last_packet_sent = 0, .largest_acked_packet_plus1 = {0}, .total_bytes_sent = 0,