From d87f99be5d1813013851ce74ed7d22743fa33f21 Mon Sep 17 00:00:00 2001 From: Damien Neil Date: Mon, 30 Oct 2023 10:51:45 -0700 Subject: [PATCH] quic: idle timeouts, handshake timeouts, and keepalive Negotiate the connection idle timeout based on the sent and received max_idle_timeout transport parameter values. Set a configurable limit on how long a handshake can take to complete. Add a configuration option to send keep-alive PING frames to avoid connection closure due to the idle timeout. RFC 9000, Section 10.1. For golang/go#58547 Change-Id: If6a611090ab836cd6937fcfbb1360a0f07425102 Reviewed-on: https://go-review.googlesource.com/c/net/+/540895 LUCI-TryBot-Result: Go LUCI Reviewed-by: Jonathan Amsterdam --- internal/quic/config.go | 36 ++++- internal/quic/conn.go | 33 ++--- internal/quic/conn_close.go | 270 ++++++++++++++++++++++-------------- internal/quic/conn_recv.go | 11 +- internal/quic/conn_send.go | 14 +- internal/quic/conn_test.go | 5 +- internal/quic/idle.go | 170 +++++++++++++++++++++++ internal/quic/idle_test.go | 225 ++++++++++++++++++++++++++++++ internal/quic/loss.go | 9 +- internal/quic/qlog.go | 12 +- internal/quic/qlog_test.go | 70 ++++++++++ internal/quic/quic.go | 6 + 12 files changed, 721 insertions(+), 140 deletions(-) create mode 100644 internal/quic/idle.go create mode 100644 internal/quic/idle_test.go diff --git a/internal/quic/config.go b/internal/quic/config.go index b10ecc79e9..b045b7b92c 100644 --- a/internal/quic/config.go +++ b/internal/quic/config.go @@ -9,6 +9,8 @@ package quic import ( "crypto/tls" "log/slog" + "math" + "time" ) // A Config structure configures a QUIC endpoint. @@ -74,6 +76,26 @@ type Config struct { // If this field is left as zero, stateless reset is disabled. StatelessResetKey [32]byte + // HandshakeTimeout is the maximum time in which a connection handshake must complete. + // If zero, the default of 10 seconds is used. + // If negative, there is no handshake timeout. + HandshakeTimeout time.Duration + + // MaxIdleTimeout is the maximum time after which an idle connection will be closed. + // If zero, the default of 30 seconds is used. + // If negative, idle connections are never closed. + // + // The idle timeout for a connection is the minimum of the maximum idle timeouts + // of the endpoints. + MaxIdleTimeout time.Duration + + // KeepAlivePeriod is the time after which a packet will be sent to keep + // an idle connection alive. + // If zero, keep alive packets are not sent. + // If greater than zero, the keep alive period is the smaller of KeepAlivePeriod and + // half the connection idle timeout. + KeepAlivePeriod time.Duration + // QLogLogger receives qlog events. // // Events currently correspond to the definitions in draft-ietf-qlog-quic-events-03. @@ -85,7 +107,7 @@ type Config struct { QLogLogger *slog.Logger } -func configDefault(v, def, limit int64) int64 { +func configDefault[T ~int64](v, def, limit T) T { switch { case v == 0: return def @@ -115,3 +137,15 @@ func (c *Config) maxStreamWriteBufferSize() int64 { func (c *Config) maxConnReadBufferSize() int64 { return configDefault(c.MaxConnReadBufferSize, 1<<20, maxVarint) } + +func (c *Config) handshakeTimeout() time.Duration { + return configDefault(c.HandshakeTimeout, defaultHandshakeTimeout, math.MaxInt64) +} + +func (c *Config) maxIdleTimeout() time.Duration { + return configDefault(c.MaxIdleTimeout, defaultMaxIdleTimeout, math.MaxInt64) +} + +func (c *Config) keepAlivePeriod() time.Duration { + return configDefault(c.KeepAlivePeriod, defaultKeepAlivePeriod, math.MaxInt64) +} diff --git a/internal/quic/conn.go b/internal/quic/conn.go index cca11166ca..b2b6a0877a 100644 --- a/internal/quic/conn.go +++ b/internal/quic/conn.go @@ -26,22 +26,17 @@ type Conn struct { testHooks connTestHooks peerAddr netip.AddrPort - msgc chan any - donec chan struct{} // closed when conn loop exits - exited bool // set to make the conn loop exit immediately + msgc chan any + donec chan struct{} // closed when conn loop exits w packetWriter acks [numberSpaceCount]ackState // indexed by number space lifetime lifetimeState + idle idleState connIDState connIDState loss lossState streams streamsState - // idleTimeout is the time at which the connection will be closed due to inactivity. - // https://www.rfc-editor.org/rfc/rfc9000#section-10.1 - maxIdleTimeout time.Duration - idleTimeout time.Time - // Packet protection keys, CRYPTO streams, and TLS state. keysInitial fixedKeyPair keysHandshake fixedKeyPair @@ -105,8 +100,6 @@ func newConn(now time.Time, side connSide, cids newServerConnIDs, peerAddr netip peerAddr: peerAddr, msgc: make(chan any, 1), donec: make(chan struct{}), - maxIdleTimeout: defaultMaxIdleTimeout, - idleTimeout: now.Add(defaultMaxIdleTimeout), peerAckDelayExponent: -1, } defer func() { @@ -151,6 +144,7 @@ func newConn(now time.Time, side connSide, cids newServerConnIDs, peerAddr netip c.loss.init(c.side, maxDatagramSize, now) c.streamsInit() c.lifetimeInit() + c.restartIdleTimer(now) if err := c.startTLS(now, initialConnID, transportParameters{ initialSrcConnID: c.connIDState.srcConnID(), @@ -202,6 +196,7 @@ func (c *Conn) confirmHandshake(now time.Time) { // don't need to send anything. c.handshakeConfirmed.setReceived() } + c.restartIdleTimer(now) c.loss.confirmHandshake() // "An endpoint MUST discard its Handshake keys when the TLS handshake is confirmed" // https://www.rfc-editor.org/rfc/rfc9001#section-4.9.2-1 @@ -232,6 +227,7 @@ func (c *Conn) receiveTransportParameters(p transportParameters) error { c.streams.peerInitialMaxStreamDataBidiLocal = p.initialMaxStreamDataBidiLocal c.streams.peerInitialMaxStreamDataRemote[bidiStream] = p.initialMaxStreamDataBidiRemote c.streams.peerInitialMaxStreamDataRemote[uniStream] = p.initialMaxStreamDataUni + c.receivePeerMaxIdleTimeout(p.maxIdleTimeout) c.peerAckDelayExponent = p.ackDelayExponent c.loss.setMaxAckDelay(p.maxAckDelay) if err := c.connIDState.setPeerActiveConnIDLimit(c, p.activeConnIDLimit); err != nil { @@ -248,7 +244,6 @@ func (c *Conn) receiveTransportParameters(p transportParameters) error { return err } } - // TODO: max_idle_timeout // TODO: stateless_reset_token // TODO: max_udp_payload_size // TODO: disable_active_migration @@ -261,6 +256,8 @@ type ( wakeEvent struct{} ) +var errIdleTimeout = errors.New("idle timeout") + // loop is the connection main loop. // // Except where otherwise noted, all connection state is owned by the loop goroutine. @@ -288,14 +285,14 @@ func (c *Conn) loop(now time.Time) { defer timer.Stop() } - for !c.exited { + for c.lifetime.state != connStateDone { sendTimeout := c.maybeSend(now) // try sending // Note that we only need to consider the ack timer for the App Data space, // since the Initial and Handshake spaces always ack immediately. nextTimeout := sendTimeout - nextTimeout = firstTime(nextTimeout, c.idleTimeout) - if !c.isClosingOrDraining() { + nextTimeout = firstTime(nextTimeout, c.idle.nextTimeout) + if c.isAlive() { nextTimeout = firstTime(nextTimeout, c.loss.timer) nextTimeout = firstTime(nextTimeout, c.acks[appDataSpace].nextAck) } else { @@ -329,11 +326,9 @@ func (c *Conn) loop(now time.Time) { m.recycle() case timerEvent: // A connection timer has expired. - if !now.Before(c.idleTimeout) { - // "[...] the connection is silently closed and - // its state is discarded [...]" - // https://www.rfc-editor.org/rfc/rfc9000#section-10.1-1 - c.exited = true + if c.idleAdvance(now) { + // The connection idle timer has expired. + c.abortImmediately(now, errIdleTimeout) return } c.loss.advance(now, c.handleAckOrLoss) diff --git a/internal/quic/conn_close.go b/internal/quic/conn_close.go index a9ef0db5e3..246a126382 100644 --- a/internal/quic/conn_close.go +++ b/internal/quic/conn_close.go @@ -12,33 +12,54 @@ import ( "time" ) +// connState is the state of a connection. +type connState int + +const ( + // A connection is alive when it is first created. + connStateAlive = connState(iota) + + // The connection has received a CONNECTION_CLOSE frame from the peer, + // and has not yet sent a CONNECTION_CLOSE in response. + // + // We will send a CONNECTION_CLOSE, and then enter the draining state. + connStatePeerClosed + + // The connection is in the closing state. + // + // We will send CONNECTION_CLOSE frames to the peer + // (once upon entering the closing state, and possibly again in response to peer packets). + // + // If we receive a CONNECTION_CLOSE from the peer, we will enter the draining state. + // Otherwise, we will eventually time out and move to the done state. + // + // https://www.rfc-editor.org/rfc/rfc9000#section-10.2.1 + connStateClosing + + // The connection is in the draining state. + // + // We will neither send packets nor process received packets. + // When the drain timer expires, we move to the done state. + // + // https://www.rfc-editor.org/rfc/rfc9000#section-10.2.2 + connStateDraining + + // The connection is done, and the conn loop will exit. + connStateDone +) + // lifetimeState tracks the state of a connection. // // This is fairly coupled to the rest of a Conn, but putting it in a struct of its own helps // reason about operations that cause state transitions. type lifetimeState struct { - readyc chan struct{} // closed when TLS handshake completes - drainingc chan struct{} // closed when entering the draining state + state connState + + readyc chan struct{} // closed when TLS handshake completes + donec chan struct{} // closed when finalErr is set - // Possible states for the connection: - // - // Alive: localErr and finalErr are both nil. - // - // Closing: localErr is non-nil and finalErr is nil. - // We have sent a CONNECTION_CLOSE to the peer or are about to - // (if connCloseSentTime is zero) and are waiting for the peer to respond. - // drainEndTime is set to the time the closing state ends. - // https://www.rfc-editor.org/rfc/rfc9000.html#section-10.2.1 - // - // Draining: finalErr is non-nil. - // If localErr is nil, we're waiting for the user to provide us with a final status - // to send to the peer. - // Otherwise, we've either sent a CONNECTION_CLOSE to the peer or are about to - // (if connCloseSentTime is zero). - // drainEndTime is set to the time the draining state ends. - // https://www.rfc-editor.org/rfc/rfc9000.html#section-10.2.2 localErr error // error sent to the peer - finalErr error // error sent by the peer, or transport error; always set before draining + finalErr error // error sent by the peer, or transport error; set before closing donec connCloseSentTime time.Time // send time of last CONNECTION_CLOSE frame connCloseDelay time.Duration // delay until next CONNECTION_CLOSE frame sent @@ -47,7 +68,7 @@ type lifetimeState struct { func (c *Conn) lifetimeInit() { c.lifetime.readyc = make(chan struct{}) - c.lifetime.drainingc = make(chan struct{}) + c.lifetime.donec = make(chan struct{}) } var errNoPeerResponse = errors.New("peer did not respond to CONNECTION_CLOSE") @@ -60,13 +81,25 @@ func (c *Conn) lifetimeAdvance(now time.Time) (done bool) { // The connection drain period has ended, and we can shut down. // https://www.rfc-editor.org/rfc/rfc9000.html#section-10.2-7 c.lifetime.drainEndTime = time.Time{} - if c.lifetime.finalErr == nil { - // The peer never responded to our CONNECTION_CLOSE. - c.enterDraining(now, errNoPeerResponse) + if c.lifetime.state != connStateDraining { + // We were in the closing state, waiting for a CONNECTION_CLOSE from the peer. + c.setFinalError(errNoPeerResponse) } + c.setState(now, connStateDone) return true } +// setState sets the conn state. +func (c *Conn) setState(now time.Time, state connState) { + switch state { + case connStateClosing, connStateDraining: + if c.lifetime.drainEndTime.IsZero() { + c.lifetime.drainEndTime = now.Add(3 * c.loss.ptoBasePeriod()) + } + } + c.lifetime.state = state +} + // confirmHandshake is called when the TLS handshake completes. func (c *Conn) handshakeDone() { close(c.lifetime.readyc) @@ -81,44 +114,66 @@ func (c *Conn) handshakeDone() { // // https://www.rfc-editor.org/rfc/rfc9000.html#section-10.2.2 func (c *Conn) isDraining() bool { - return c.lifetime.finalErr != nil + switch c.lifetime.state { + case connStateDraining, connStateDone: + return true + } + return false } -// isClosingOrDraining reports whether the conn is in the closing or draining states. -func (c *Conn) isClosingOrDraining() bool { - return c.lifetime.localErr != nil || c.lifetime.finalErr != nil +// isAlive reports whether the conn is handling packets. +func (c *Conn) isAlive() bool { + return c.lifetime.state == connStateAlive } // sendOK reports whether the conn can send frames at this time. func (c *Conn) sendOK(now time.Time) bool { - if !c.isClosingOrDraining() { + switch c.lifetime.state { + case connStateAlive: return true - } - // We are closing or draining. - if c.lifetime.localErr == nil { - // We're waiting for the user to close the connection, providing us with - // a final status to send to the peer. + case connStatePeerClosed: + if c.lifetime.localErr == nil { + // We're waiting for the user to close the connection, providing us with + // a final status to send to the peer. + return false + } + // We should send a CONNECTION_CLOSE. + return true + case connStateClosing: + if c.lifetime.connCloseSentTime.IsZero() { + return true + } + maxRecvTime := c.acks[initialSpace].maxRecvTime + if t := c.acks[handshakeSpace].maxRecvTime; t.After(maxRecvTime) { + maxRecvTime = t + } + if t := c.acks[appDataSpace].maxRecvTime; t.After(maxRecvTime) { + maxRecvTime = t + } + if maxRecvTime.Before(c.lifetime.connCloseSentTime.Add(c.lifetime.connCloseDelay)) { + // After sending CONNECTION_CLOSE, ignore packets from the peer for + // a delay. On the next packet received after the delay, send another + // CONNECTION_CLOSE. + return false + } + return true + case connStateDraining: + // We are in the draining state, and will send no more packets. return false + case connStateDone: + return false + default: + panic("BUG: unhandled connection state") } - // Past this point, returning true will result in the conn sending a CONNECTION_CLOSE - // due to localErr being set. - if c.lifetime.drainEndTime.IsZero() { - // The closing and draining states should last for at least three times - // the current PTO interval. We currently use exactly that minimum. - // https://www.rfc-editor.org/rfc/rfc9000.html#section-10.2-5 - // - // The drain period begins when we send or receive a CONNECTION_CLOSE, - // whichever comes first. - // https://www.rfc-editor.org/rfc/rfc9000.html#section-10.2.2-3 - c.lifetime.drainEndTime = now.Add(3 * c.loss.ptoBasePeriod()) +} + +// sendConnectionClose reports that the conn has sent a CONNECTION_CLOSE to the peer. +func (c *Conn) sentConnectionClose(now time.Time) { + switch c.lifetime.state { + case connStatePeerClosed: + c.enterDraining(now) } if c.lifetime.connCloseSentTime.IsZero() { - // We haven't sent a CONNECTION_CLOSE yet. Do so. - // Either we're initiating an immediate close - // (and will enter the closing state as soon as we send CONNECTION_CLOSE), - // or we've read a CONNECTION_CLOSE from our peer - // (and may send one CONNECTION_CLOSE before entering the draining state). - // // Set the initial delay before we will send another CONNECTION_CLOSE. // // RFC 9000 states that we should rate limit CONNECTION_CLOSE frames, @@ -126,65 +181,56 @@ func (c *Conn) sendOK(now time.Time) bool { // with the same delay as the PTO timer (RFC 9002, Section 6.2.1), // not including max_ack_delay, and double it on every CONNECTION_CLOSE sent. c.lifetime.connCloseDelay = c.loss.rtt.smoothedRTT + max(4*c.loss.rtt.rttvar, timerGranularity) - c.lifetime.drainEndTime = now.Add(3 * c.loss.ptoBasePeriod()) - return true - } - if c.isDraining() { - // We are in the draining state, and will send no more packets. - return false - } - maxRecvTime := c.acks[initialSpace].maxRecvTime - if t := c.acks[handshakeSpace].maxRecvTime; t.After(maxRecvTime) { - maxRecvTime = t - } - if t := c.acks[appDataSpace].maxRecvTime; t.After(maxRecvTime) { - maxRecvTime = t - } - if maxRecvTime.Before(c.lifetime.connCloseSentTime.Add(c.lifetime.connCloseDelay)) { - // After sending CONNECTION_CLOSE, ignore packets from the peer for - // a delay. On the next packet received after the delay, send another - // CONNECTION_CLOSE. - return false + } else if !c.lifetime.connCloseSentTime.Equal(now) { + // If connCloseSentTime == now, we're sending two CONNECTION_CLOSE frames + // coalesced into the same datagram. We only want to increase the delay once. + c.lifetime.connCloseDelay *= 2 } c.lifetime.connCloseSentTime = now - c.lifetime.connCloseDelay *= 2 - return true } -// enterDraining enters the draining state. -func (c *Conn) enterDraining(now time.Time, err error) { - if c.isDraining() { - return +// handlePeerConnectionClose handles a CONNECTION_CLOSE from the peer. +func (c *Conn) handlePeerConnectionClose(now time.Time, err error) { + c.setFinalError(err) + switch c.lifetime.state { + case connStateAlive: + c.setState(now, connStatePeerClosed) + case connStatePeerClosed: + // Duplicate CONNECTION_CLOSE, ignore. + case connStateClosing: + if c.lifetime.connCloseSentTime.IsZero() { + c.setState(now, connStatePeerClosed) + } else { + c.setState(now, connStateDraining) + } + case connStateDraining: + case connStateDone: } - if err == errStatelessReset { - // If we've received a stateless reset, then we must not send a CONNECTION_CLOSE. - // Setting connCloseSentTime here prevents us from doing so. - c.lifetime.finalErr = errStatelessReset - c.lifetime.localErr = errStatelessReset - c.lifetime.connCloseSentTime = now - } else if e, ok := c.lifetime.localErr.(localTransportError); ok && e.code != errNo { - // If we've terminated the connection due to a peer protocol violation, - // record the final error on the connection as our reason for termination. - c.lifetime.finalErr = c.lifetime.localErr - } else { - c.lifetime.finalErr = err +} + +// setFinalError records the final connection status we report to the user. +func (c *Conn) setFinalError(err error) { + select { + case <-c.lifetime.donec: + return // already set + default: } - close(c.lifetime.drainingc) - c.streams.queue.close(c.lifetime.finalErr) + c.lifetime.finalErr = err + close(c.lifetime.donec) } func (c *Conn) waitReady(ctx context.Context) error { select { case <-c.lifetime.readyc: return nil - case <-c.lifetime.drainingc: + case <-c.lifetime.donec: return c.lifetime.finalErr default: } select { case <-c.lifetime.readyc: return nil - case <-c.lifetime.drainingc: + case <-c.lifetime.donec: return c.lifetime.finalErr case <-ctx.Done(): return ctx.Err() @@ -199,7 +245,7 @@ func (c *Conn) waitReady(ctx context.Context) error { // err := conn.Wait(context.Background()) func (c *Conn) Close() error { c.Abort(nil) - <-c.lifetime.drainingc + <-c.lifetime.donec return c.lifetime.finalErr } @@ -213,7 +259,7 @@ func (c *Conn) Close() error { // containing the peer's error code and reason. // If the peer closes the connection with any other status, Wait returns a non-nil error. func (c *Conn) Wait(ctx context.Context) error { - if err := c.waitOnDone(ctx, c.lifetime.drainingc); err != nil { + if err := c.waitOnDone(ctx, c.lifetime.donec); err != nil { return err } return c.lifetime.finalErr @@ -229,30 +275,46 @@ func (c *Conn) Abort(err error) { err = localTransportError{code: errNo} } c.sendMsg(func(now time.Time, c *Conn) { - c.abort(now, err) + c.enterClosing(now, err) }) } // abort terminates a connection with an error. func (c *Conn) abort(now time.Time, err error) { - if c.lifetime.localErr != nil { - return // already closing - } - c.lifetime.localErr = err + c.setFinalError(err) // this error takes precedence over the peer's CONNECTION_CLOSE + c.enterClosing(now, err) } // abortImmediately terminates a connection. // The connection does not send a CONNECTION_CLOSE, and skips the draining period. func (c *Conn) abortImmediately(now time.Time, err error) { - c.abort(now, err) - c.enterDraining(now, err) - c.exited = true + c.setFinalError(err) + c.setState(now, connStateDone) +} + +// enterClosing starts an immediate close. +// We will send a CONNECTION_CLOSE to the peer and wait for their response. +func (c *Conn) enterClosing(now time.Time, err error) { + switch c.lifetime.state { + case connStateAlive: + c.lifetime.localErr = err + c.setState(now, connStateClosing) + case connStatePeerClosed: + c.lifetime.localErr = err + } +} + +// enterDraining moves directly to the draining state, without sending a CONNECTION_CLOSE. +func (c *Conn) enterDraining(now time.Time) { + switch c.lifetime.state { + case connStateAlive, connStatePeerClosed, connStateClosing: + c.setState(now, connStateDraining) + } } // exit fully terminates a connection immediately. func (c *Conn) exit() { c.sendMsg(func(now time.Time, c *Conn) { - c.enterDraining(now, errors.New("connection closed")) - c.exited = true + c.abortImmediately(now, errors.New("connection closed")) }) } diff --git a/internal/quic/conn_recv.go b/internal/quic/conn_recv.go index 896c6d74ed..156ef5dd50 100644 --- a/internal/quic/conn_recv.go +++ b/internal/quic/conn_recv.go @@ -61,7 +61,7 @@ func (c *Conn) handleDatagram(now time.Time, dgram *datagram) { // Invalid data at the end of a datagram is ignored. break } - c.idleTimeout = now.Add(c.maxIdleTimeout) + c.idleHandlePacketReceived(now) buf = buf[n:] } } @@ -525,7 +525,7 @@ func (c *Conn) handleConnectionCloseTransportFrame(now time.Time, payload []byte if n < 0 { return -1 } - c.enterDraining(now, peerTransportError{code: code, reason: reason}) + c.handlePeerConnectionClose(now, peerTransportError{code: code, reason: reason}) return n } @@ -534,7 +534,7 @@ func (c *Conn) handleConnectionCloseApplicationFrame(now time.Time, payload []by if n < 0 { return -1 } - c.enterDraining(now, &ApplicationError{Code: code, Reason: reason}) + c.handlePeerConnectionClose(now, &ApplicationError{Code: code, Reason: reason}) return n } @@ -548,7 +548,7 @@ func (c *Conn) handleHandshakeDoneFrame(now time.Time, space numberSpace, payloa }) return -1 } - if !c.isClosingOrDraining() { + if c.isAlive() { c.confirmHandshake(now) } return 1 @@ -560,5 +560,6 @@ func (c *Conn) handleStatelessReset(now time.Time, resetToken statelessResetToke if !c.connIDState.isValidStatelessResetToken(resetToken) { return } - c.enterDraining(now, errStatelessReset) + c.setFinalError(errStatelessReset) + c.enterDraining(now) } diff --git a/internal/quic/conn_send.go b/internal/quic/conn_send.go index 22e780479f..e45dc8af3d 100644 --- a/internal/quic/conn_send.go +++ b/internal/quic/conn_send.go @@ -77,6 +77,7 @@ func (c *Conn) maybeSend(now time.Time) (next time.Time) { } sentInitial = c.w.finishProtectedLongHeaderPacket(pnumMaxAcked, c.keysInitial.w, p) if sentInitial != nil { + c.idleHandlePacketSent(now, sentInitial) // Client initial packets and ack-eliciting server initial packaets // need to be sent in a datagram padded to at least 1200 bytes. // We can't add the padding yet, however, since we may want to @@ -104,6 +105,7 @@ func (c *Conn) maybeSend(now time.Time) (next time.Time) { logSentPacket(c, packetTypeHandshake, pnum, p.srcConnID, p.dstConnID, c.w.payload()) } if sent := c.w.finishProtectedLongHeaderPacket(pnumMaxAcked, c.keysHandshake.w, p); sent != nil { + c.idleHandlePacketSent(now, sent) c.loss.packetSent(now, handshakeSpace, sent) if c.side == clientSide { // "[...] a client MUST discard Initial keys when it first @@ -131,6 +133,7 @@ func (c *Conn) maybeSend(now time.Time) (next time.Time) { logSentPacket(c, packetType1RTT, pnum, nil, dstConnID, c.w.payload()) } if sent := c.w.finish1RTTPacket(pnum, pnumMaxAcked, dstConnID, &c.keysAppData); sent != nil { + c.idleHandlePacketSent(now, sent) c.loss.packetSent(now, appDataSpace, sent) } } @@ -261,6 +264,10 @@ func (c *Conn) appendFrames(now time.Time, space numberSpace, pnum packetNumber, if !c.appendStreamFrames(&c.w, pnum, pto) { return } + + if !c.appendKeepAlive(now) { + return + } } // If this is a PTO probe and we haven't added an ack-eliciting frame yet, @@ -325,7 +332,7 @@ func (c *Conn) appendAckFrame(now time.Time, space numberSpace) bool { } func (c *Conn) appendConnectionCloseFrame(now time.Time, space numberSpace, err error) { - c.lifetime.connCloseSentTime = now + c.sentConnectionClose(now) switch e := err.(type) { case localTransportError: c.w.appendConnectionCloseTransportFrame(e.code, 0, e.reason) @@ -342,11 +349,12 @@ func (c *Conn) appendConnectionCloseFrame(now time.Time, space numberSpace, err // TLS alerts are sent using error codes [0x0100,0x01ff). // https://www.rfc-editor.org/rfc/rfc9000#section-20.1-2.36.1 var alert tls.AlertError - if errors.As(err, &alert) { + switch { + case errors.As(err, &alert): // tls.AlertError is a uint8, so this can't exceed 0x01ff. code := errTLSBase + transportError(alert) c.w.appendConnectionCloseTransportFrame(code, 0, "") - } else { + default: c.w.appendConnectionCloseTransportFrame(errInternal, 0, "") } } diff --git a/internal/quic/conn_test.go b/internal/quic/conn_test.go index 514a8775e9..70ba7b3926 100644 --- a/internal/quic/conn_test.go +++ b/internal/quic/conn_test.go @@ -25,6 +25,7 @@ var testVV = flag.Bool("vv", false, "even more verbose test output") func TestConnTestConn(t *testing.T) { tc := newTestConn(t, serverSide) + tc.handshake() if got, want := tc.timeUntilEvent(), defaultMaxIdleTimeout; got != want { t.Errorf("new conn timeout=%v, want %v (max_idle_timeout)", got, want) } @@ -49,8 +50,8 @@ func TestConnTestConn(t *testing.T) { tc.wait() tc.advanceToTimer() - if !tc.conn.exited { - t.Errorf("after advancing to idle timeout, exited = false, want true") + if got := tc.conn.lifetime.state; got != connStateDone { + t.Errorf("after advancing to idle timeout, conn state = %v, want done", got) } } diff --git a/internal/quic/idle.go b/internal/quic/idle.go new file mode 100644 index 0000000000..f5b2422adb --- /dev/null +++ b/internal/quic/idle.go @@ -0,0 +1,170 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.21 + +package quic + +import ( + "time" +) + +// idleState tracks connection idle events. +// +// Before the handshake is confirmed, the idle timeout is Config.HandshakeTimeout. +// +// After the handshake is confirmed, the idle timeout is +// the minimum of Config.MaxIdleTimeout and the peer's max_idle_timeout transport parameter. +// +// If KeepAlivePeriod is set, keep-alive pings are sent. +// Keep-alives are only sent after the handshake is confirmed. +// +// https://www.rfc-editor.org/rfc/rfc9000#section-10.1 +type idleState struct { + // idleDuration is the negotiated idle timeout for the connection. + idleDuration time.Duration + + // idleTimeout is the time at which the connection will be closed due to inactivity. + idleTimeout time.Time + + // nextTimeout is the time of the next idle event. + // If nextTimeout == idleTimeout, this is the idle timeout. + // Otherwise, this is the keep-alive timeout. + nextTimeout time.Time + + // sentSinceLastReceive is set if we have sent an ack-eliciting packet + // since the last time we received and processed a packet from the peer. + sentSinceLastReceive bool +} + +// receivePeerMaxIdleTimeout handles the peer's max_idle_timeout transport parameter. +func (c *Conn) receivePeerMaxIdleTimeout(peerMaxIdleTimeout time.Duration) { + localMaxIdleTimeout := c.config.maxIdleTimeout() + switch { + case localMaxIdleTimeout == 0: + c.idle.idleDuration = peerMaxIdleTimeout + case peerMaxIdleTimeout == 0: + c.idle.idleDuration = localMaxIdleTimeout + default: + c.idle.idleDuration = min(localMaxIdleTimeout, peerMaxIdleTimeout) + } +} + +func (c *Conn) idleHandlePacketReceived(now time.Time) { + if !c.handshakeConfirmed.isSet() { + return + } + // "An endpoint restarts its idle timer when a packet from its peer is + // received and processed successfully." + // https://www.rfc-editor.org/rfc/rfc9000#section-10.1-3 + c.idle.sentSinceLastReceive = false + c.restartIdleTimer(now) +} + +func (c *Conn) idleHandlePacketSent(now time.Time, sent *sentPacket) { + // "An endpoint also restarts its idle timer when sending an ack-eliciting packet + // if no other ack-eliciting packets have been sent since + // last receiving and processing a packet." + // https://www.rfc-editor.org/rfc/rfc9000#section-10.1-3 + if c.idle.sentSinceLastReceive || !sent.ackEliciting || !c.handshakeConfirmed.isSet() { + return + } + c.idle.sentSinceLastReceive = true + c.restartIdleTimer(now) +} + +func (c *Conn) restartIdleTimer(now time.Time) { + if !c.isAlive() { + // Connection is closing, disable timeouts. + c.idle.idleTimeout = time.Time{} + c.idle.nextTimeout = time.Time{} + return + } + var idleDuration time.Duration + if c.handshakeConfirmed.isSet() { + idleDuration = c.idle.idleDuration + } else { + idleDuration = c.config.handshakeTimeout() + } + if idleDuration == 0 { + c.idle.idleTimeout = time.Time{} + } else { + // "[...] endpoints MUST increase the idle timeout period to be + // at least three times the current Probe Timeout (PTO)." + // https://www.rfc-editor.org/rfc/rfc9000#section-10.1-4 + idleDuration = max(idleDuration, 3*c.loss.ptoPeriod()) + c.idle.idleTimeout = now.Add(idleDuration) + } + // Set the time of our next event: + // The idle timer if no keep-alive is set, or the keep-alive timer if one is. + c.idle.nextTimeout = c.idle.idleTimeout + keepAlive := c.config.keepAlivePeriod() + switch { + case !c.handshakeConfirmed.isSet(): + // We do not send keep-alives before the handshake is complete. + case keepAlive <= 0: + // Keep-alives are not enabled. + case c.idle.sentSinceLastReceive: + // We have sent an ack-eliciting packet to the peer. + // If they don't acknowledge it, loss detection will follow up with PTO probes, + // which will function as keep-alives. + // We don't need to send further pings. + case idleDuration == 0: + // The connection does not have a negotiated idle timeout. + // Send keep-alives anyway, since they may be required to keep middleboxes + // from losing state. + c.idle.nextTimeout = now.Add(keepAlive) + default: + // Schedule our next keep-alive. + // If our configured keep-alive period is greater than half the negotiated + // connection idle timeout, we reduce the keep-alive period to half + // the idle timeout to ensure we have time for the ping to arrive. + c.idle.nextTimeout = now.Add(min(keepAlive, idleDuration/2)) + } +} + +func (c *Conn) appendKeepAlive(now time.Time) bool { + if c.idle.nextTimeout.IsZero() || c.idle.nextTimeout.After(now) { + return true // timer has not expired + } + if c.idle.nextTimeout.Equal(c.idle.idleTimeout) { + return true // no keepalive timer set, only idle + } + if c.idle.sentSinceLastReceive { + return true // already sent an ack-eliciting packet + } + if c.w.sent.ackEliciting { + return true // this packet is already ack-eliciting + } + // Send an ack-eliciting PING frame to the peer to keep the connection alive. + return c.w.appendPingFrame() +} + +var errHandshakeTimeout error = localTransportError{ + code: errConnectionRefused, + reason: "handshake timeout", +} + +func (c *Conn) idleAdvance(now time.Time) (shouldExit bool) { + if c.idle.idleTimeout.IsZero() || now.Before(c.idle.idleTimeout) { + return false + } + c.idle.idleTimeout = time.Time{} + c.idle.nextTimeout = time.Time{} + if !c.handshakeConfirmed.isSet() { + // Handshake timeout has expired. + // If we're a server, we're refusing the too-slow client. + // If we're a client, we're giving up. + // In either case, we're going to send a CONNECTION_CLOSE frame and + // enter the closing state rather than unceremoniously dropping the connection, + // since the peer might still be trying to complete the handshake. + c.abort(now, errHandshakeTimeout) + return false + } + // Idle timeout has expired. + // + // "[...] the connection is silently closed and its state is discarded [...]" + // https://www.rfc-editor.org/rfc/rfc9000#section-10.1-1 + return true +} diff --git a/internal/quic/idle_test.go b/internal/quic/idle_test.go new file mode 100644 index 0000000000..18f6a690a4 --- /dev/null +++ b/internal/quic/idle_test.go @@ -0,0 +1,225 @@ +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build go1.21 + +package quic + +import ( + "context" + "crypto/tls" + "fmt" + "testing" + "time" +) + +func TestHandshakeTimeoutExpiresServer(t *testing.T) { + const timeout = 5 * time.Second + tc := newTestConn(t, serverSide, func(c *Config) { + c.HandshakeTimeout = timeout + }) + tc.ignoreFrame(frameTypeAck) + tc.ignoreFrame(frameTypeNewConnectionID) + tc.writeFrames(packetTypeInitial, + debugFrameCrypto{ + data: tc.cryptoDataIn[tls.QUICEncryptionLevelInitial], + }) + // Server starts its end of the handshake. + // Client acks these packets to avoid starting the PTO timer. + tc.wantFrameType("server sends Initial CRYPTO flight", + packetTypeInitial, debugFrameCrypto{}) + tc.writeAckForAll() + tc.wantFrameType("server sends Handshake CRYPTO flight", + packetTypeHandshake, debugFrameCrypto{}) + tc.writeAckForAll() + + if got, want := tc.timerDelay(), timeout; got != want { + t.Errorf("connection timer = %v, want %v (handshake timeout)", got, want) + } + + // Client sends a packet, but this does not extend the handshake timer. + tc.advance(1 * time.Second) + tc.writeFrames(packetTypeHandshake, debugFrameCrypto{ + data: tc.cryptoDataIn[tls.QUICEncryptionLevelHandshake][:1], // partial data + }) + tc.wantIdle("handshake is not complete") + + tc.advance(timeout - 1*time.Second) + tc.wantFrame("server closes connection after handshake timeout", + packetTypeHandshake, debugFrameConnectionCloseTransport{ + code: errConnectionRefused, + }) +} + +func TestHandshakeTimeoutExpiresClient(t *testing.T) { + const timeout = 5 * time.Second + tc := newTestConn(t, clientSide, func(c *Config) { + c.HandshakeTimeout = timeout + }) + tc.ignoreFrame(frameTypeAck) + tc.ignoreFrame(frameTypeNewConnectionID) + // Start the handshake. + // The client always sets a PTO timer until it gets an ack for a handshake packet + // or confirms the handshake, so proceed far enough through the handshake to + // let us not worry about PTO. + tc.wantFrameType("client sends Initial CRYPTO flight", + packetTypeInitial, debugFrameCrypto{}) + tc.writeAckForAll() + tc.writeFrames(packetTypeInitial, + debugFrameCrypto{ + data: tc.cryptoDataIn[tls.QUICEncryptionLevelInitial], + }) + tc.writeFrames(packetTypeHandshake, + debugFrameCrypto{ + data: tc.cryptoDataIn[tls.QUICEncryptionLevelHandshake], + }) + tc.wantFrameType("client sends Handshake CRYPTO flight", + packetTypeHandshake, debugFrameCrypto{}) + tc.writeAckForAll() + tc.wantIdle("client is waiting for end of handshake") + + if got, want := tc.timerDelay(), timeout; got != want { + t.Errorf("connection timer = %v, want %v (handshake timeout)", got, want) + } + tc.advance(timeout) + tc.wantFrame("client closes connection after handshake timeout", + packetTypeHandshake, debugFrameConnectionCloseTransport{ + code: errConnectionRefused, + }) +} + +func TestIdleTimeoutExpires(t *testing.T) { + for _, test := range []struct { + localMaxIdleTimeout time.Duration + peerMaxIdleTimeout time.Duration + wantTimeout time.Duration + }{{ + localMaxIdleTimeout: 10 * time.Second, + peerMaxIdleTimeout: 20 * time.Second, + wantTimeout: 10 * time.Second, + }, { + localMaxIdleTimeout: 20 * time.Second, + peerMaxIdleTimeout: 10 * time.Second, + wantTimeout: 10 * time.Second, + }, { + localMaxIdleTimeout: 0, + peerMaxIdleTimeout: 10 * time.Second, + wantTimeout: 10 * time.Second, + }, { + localMaxIdleTimeout: 10 * time.Second, + peerMaxIdleTimeout: 0, + wantTimeout: 10 * time.Second, + }} { + name := fmt.Sprintf("local=%v/peer=%v", test.localMaxIdleTimeout, test.peerMaxIdleTimeout) + t.Run(name, func(t *testing.T) { + tc := newTestConn(t, serverSide, func(p *transportParameters) { + p.maxIdleTimeout = test.peerMaxIdleTimeout + }, func(c *Config) { + c.MaxIdleTimeout = test.localMaxIdleTimeout + }) + tc.handshake() + if got, want := tc.timeUntilEvent(), test.wantTimeout; got != want { + t.Errorf("new conn timeout=%v, want %v (idle timeout)", got, want) + } + tc.advance(test.wantTimeout - 1) + tc.wantIdle("connection is idle and alive prior to timeout") + ctx := canceledContext() + if err := tc.conn.Wait(ctx); err != context.Canceled { + t.Fatalf("conn.Wait() = %v, want Canceled", err) + } + tc.advance(1) + tc.wantIdle("connection exits after timeout") + if err := tc.conn.Wait(ctx); err != errIdleTimeout { + t.Fatalf("conn.Wait() = %v, want errIdleTimeout", err) + } + }) + } +} + +func TestIdleTimeoutKeepAlive(t *testing.T) { + for _, test := range []struct { + idleTimeout time.Duration + keepAlive time.Duration + wantTimeout time.Duration + }{{ + idleTimeout: 30 * time.Second, + keepAlive: 10 * time.Second, + wantTimeout: 10 * time.Second, + }, { + idleTimeout: 10 * time.Second, + keepAlive: 30 * time.Second, + wantTimeout: 5 * time.Second, + }, { + idleTimeout: -1, // disabled + keepAlive: 30 * time.Second, + wantTimeout: 30 * time.Second, + }} { + name := fmt.Sprintf("idle_timeout=%v/keepalive=%v", test.idleTimeout, test.keepAlive) + t.Run(name, func(t *testing.T) { + tc := newTestConn(t, serverSide, func(c *Config) { + c.MaxIdleTimeout = test.idleTimeout + c.KeepAlivePeriod = test.keepAlive + }) + tc.handshake() + if got, want := tc.timeUntilEvent(), test.wantTimeout; got != want { + t.Errorf("new conn timeout=%v, want %v (keepalive timeout)", got, want) + } + tc.advance(test.wantTimeout - 1) + tc.wantIdle("connection is idle prior to timeout") + tc.advance(1) + tc.wantFrameType("keep-alive ping is sent", packetType1RTT, + debugFramePing{}) + }) + } +} + +func TestIdleLongTermKeepAliveSent(t *testing.T) { + // This test examines a connection sitting idle and sending periodic keep-alive pings. + const keepAlivePeriod = 30 * time.Second + tc := newTestConn(t, clientSide, func(c *Config) { + c.KeepAlivePeriod = keepAlivePeriod + c.MaxIdleTimeout = -1 + }) + tc.handshake() + // The handshake will have completed a little bit after the point at which the + // keepalive timer was set. Send two PING frames to the conn, triggering an immediate ack + // and resetting the timer. + tc.writeFrames(packetType1RTT, debugFramePing{}) + tc.writeFrames(packetType1RTT, debugFramePing{}) + tc.wantFrameType("conn acks received pings", packetType1RTT, debugFrameAck{}) + for i := 0; i < 10; i++ { + tc.wantIdle("conn has nothing more to send") + if got, want := tc.timeUntilEvent(), keepAlivePeriod; got != want { + t.Errorf("i=%v conn timeout=%v, want %v (keepalive timeout)", i, got, want) + } + tc.advance(keepAlivePeriod) + tc.wantFrameType("keep-alive ping is sent", packetType1RTT, + debugFramePing{}) + tc.writeAckForAll() + } +} + +func TestIdleLongTermKeepAliveReceived(t *testing.T) { + // This test examines a connection sitting idle, but receiving periodic peer + // traffic to keep the connection alive. + const idleTimeout = 30 * time.Second + tc := newTestConn(t, serverSide, func(c *Config) { + c.MaxIdleTimeout = idleTimeout + }) + tc.handshake() + for i := 0; i < 10; i++ { + tc.advance(idleTimeout - 1*time.Second) + tc.writeFrames(packetType1RTT, debugFramePing{}) + if got, want := tc.timeUntilEvent(), maxAckDelay-timerGranularity; got != want { + t.Errorf("i=%v conn timeout=%v, want %v (max_ack_delay)", i, got, want) + } + tc.advanceToTimer() + tc.wantFrameType("conn acks received ping", packetType1RTT, debugFrameAck{}) + } + // Connection is still alive. + ctx := canceledContext() + if err := tc.conn.Wait(ctx); err != context.Canceled { + t.Fatalf("conn.Wait() = %v, want Canceled", err) + } +} diff --git a/internal/quic/loss.go b/internal/quic/loss.go index c0f915b422..4a0767bd0b 100644 --- a/internal/quic/loss.go +++ b/internal/quic/loss.go @@ -431,12 +431,15 @@ func (c *lossState) scheduleTimer(now time.Time) { c.timer = time.Time{} return } - // https://www.rfc-editor.org/rfc/rfc9002.html#section-6.2.1 - pto := c.ptoBasePeriod() << c.ptoBackoffCount - c.timer = last.Add(pto) + c.timer = last.Add(c.ptoPeriod()) c.ptoTimerArmed = true } +func (c *lossState) ptoPeriod() time.Duration { + // https://www.rfc-editor.org/rfc/rfc9002.html#section-6.2.1 + return c.ptoBasePeriod() << c.ptoBackoffCount +} + func (c *lossState) ptoBasePeriod() time.Duration { // https://www.rfc-editor.org/rfc/rfc9002.html#section-6.2.1 pto := c.rtt.smoothedRTT + max(4*c.rtt.rttvar, timerGranularity) diff --git a/internal/quic/qlog.go b/internal/quic/qlog.go index 29875693ec..c8ee429fec 100644 --- a/internal/quic/qlog.go +++ b/internal/quic/qlog.go @@ -119,8 +119,13 @@ func (c *Conn) logConnectionClosed() { // TODO: Distinguish between peer and locally-initiated close. trigger = "application" case localTransportError: - if e.code == errNo { - trigger = "clean" + switch err { + case errHandshakeTimeout: + trigger = "handshake_timeout" + default: + if e.code == errNo { + trigger = "clean" + } } case peerTransportError: if e.code == errNo { @@ -128,10 +133,11 @@ func (c *Conn) logConnectionClosed() { } default: switch err { + case errIdleTimeout: + trigger = "idle_timeout" case errStatelessReset: trigger = "stateless_reset" } - // TODO: idle_timeout, handshake_timeout } // https://www.ietf.org/archive/id/draft-ietf-quic-qlog-quic-events-03.html#section-4.3 c.log.LogAttrs(context.Background(), QLogLevelEndpoint, diff --git a/internal/quic/qlog_test.go b/internal/quic/qlog_test.go index 5a2858b8bd..119f5d16af 100644 --- a/internal/quic/qlog_test.go +++ b/internal/quic/qlog_test.go @@ -14,6 +14,7 @@ import ( "log/slog" "reflect" "testing" + "time" "golang.org/x/net/internal/quic/qlog" ) @@ -54,6 +55,75 @@ func TestQLogHandshake(t *testing.T) { }) } +func TestQLogConnectionClosedTrigger(t *testing.T) { + for _, test := range []struct { + trigger string + connOpts []any + f func(*testConn) + }{{ + trigger: "clean", + f: func(tc *testConn) { + tc.handshake() + tc.conn.Abort(nil) + }, + }, { + trigger: "handshake_timeout", + connOpts: []any{ + func(c *Config) { + c.HandshakeTimeout = 5 * time.Second + }, + }, + f: func(tc *testConn) { + tc.ignoreFrame(frameTypeCrypto) + tc.ignoreFrame(frameTypeAck) + tc.ignoreFrame(frameTypePing) + tc.advance(5 * time.Second) + }, + }, { + trigger: "idle_timeout", + connOpts: []any{ + func(c *Config) { + c.MaxIdleTimeout = 5 * time.Second + }, + }, + f: func(tc *testConn) { + tc.handshake() + tc.advance(5 * time.Second) + }, + }, { + trigger: "error", + f: func(tc *testConn) { + tc.handshake() + tc.writeFrames(packetType1RTT, debugFrameConnectionCloseTransport{ + code: errProtocolViolation, + }) + tc.conn.Abort(nil) + }, + }} { + t.Run(test.trigger, func(t *testing.T) { + qr := &qlogRecord{} + tc := newTestConn(t, clientSide, append(test.connOpts, qr.config)...) + test.f(tc) + fr, ptype := tc.readFrame() + switch fr := fr.(type) { + case debugFrameConnectionCloseTransport: + tc.writeFrames(ptype, fr) + case nil: + default: + t.Fatalf("unexpected frame: %v", fr) + } + tc.wantIdle("connection should be idle while closing") + tc.advance(5 * time.Second) // long enough for the drain timer to expire + qr.wantEvents(t, jsonEvent{ + "name": "connectivity:connection_closed", + "data": map[string]any{ + "trigger": test.trigger, + }, + }) + }) + } +} + type nopCloseWriter struct { io.Writer } diff --git a/internal/quic/quic.go b/internal/quic/quic.go index 084887be67..6b60db869a 100644 --- a/internal/quic/quic.go +++ b/internal/quic/quic.go @@ -54,6 +54,12 @@ const ( maxPeerActiveConnIDLimit = 4 ) +// Time limit for completing the handshake. +const defaultHandshakeTimeout = 10 * time.Second + +// Keep-alive ping frequency. +const defaultKeepAlivePeriod = 0 + // Local timer granularity. // https://www.rfc-editor.org/rfc/rfc9002.html#section-6.1.2-6 const timerGranularity = 1 * time.Millisecond