core: quic listener will manage the underlying socket by itself.

listeners.go

@@ -465,12 +465,70 @@ func ListenPacket(network, addr string) (net.PacketConn, error) {
 	return ln.(net.PacketConn), nil
 }
 
+// ListenQUIC returns a quic.EarlyListener suitable for use in a Caddy module.
+// The network will be transformed into a QUIC-compatible type (if unix, then
+// unixgram will be used; otherwise, udp will be used).
+//
+// NOTE: This API is EXPERIMENTAL and may be changed or removed.
+func (na NetworkAddress) ListenQUIC(ctx context.Context, portOffset uint, config net.ListenConfig, tlsConf *tls.Config, activeRequests *int64) (http3.QUICEarlyListener, error) {
+	lnKey := listenerKey("quic"+na.Network, na.JoinHostPort(portOffset))
+
+	sharedEarlyListener, _, err := listenerPool.LoadOrNew(lnKey, func() (Destructor, error) {
+		lnAny, err := na.Listen(ctx, portOffset, config)
+		if err != nil {
+			return nil, err
+		}
+
+		ln := lnAny.(net.PacketConn)
+
+		h3ln := ln
+		// retrieve the underlying socket, so quic-go can optimize.
+		if unwrapper, ok := ln.(interface{ Unwrap() any }); ok {
+			h3ln = unwrapper.Unwrap().(net.PacketConn)
+		}
+
+		sqtc := newSharedQUICTLSConfig(tlsConf)
+		// http3.ConfigureTLSConfig only uses this field and tls App sets this field as well
+		//nolint:gosec
+		quicTlsConfig := &tls.Config{GetConfigForClient: sqtc.getConfigForClient}
+		earlyLn, err := quic.ListenEarly(h3ln, http3.ConfigureTLSConfig(quicTlsConfig), &quic.Config{
+			Allow0RTT: true,
+			RequireAddressValidation: func(clientAddr net.Addr) bool {
+				var highLoad bool
+				if activeRequests != nil {
+					highLoad = atomic.LoadInt64(activeRequests) > 1000 // TODO: make tunable?
+				}
+				return highLoad
+			},
+		})
+		if err != nil {
+			return nil, err
+		}
+		// using the original net.PacketConn to close them properly
+		return &sharedQuicListener{EarlyListener: earlyLn, packetConn: ln, sqtc: sqtc, key: lnKey}, nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	sql := sharedEarlyListener.(*sharedQuicListener)
+	// add current tls.Config to sqtc, so GetConfigForClient will always return the latest tls.Config in case of context cancellation
+	ctx, cancel := sql.sqtc.addTLSConfig(tlsConf)
+
+	return &fakeCloseQuicListener{
+		sharedQuicListener: sql,
+		context:            ctx,
+		contextCancel:      cancel,
+	}, nil
+}
+
 // ListenQUIC returns a quic.EarlyListener suitable for use in a Caddy module.
 // The network will be transformed into a QUIC-compatible type (if unix, then
 // unixgram will be used; otherwise, udp will be used).
 //
 // NOTE: This API is EXPERIMENTAL and may be changed or removed.
 //
+// DEPRECATED: Use NetworkAddress.ListenQUIC instead. This function will likely be changed or removed in the future.
 // TODO: See if we can find a more elegant solution closer to the new NetworkAddress.Listen API.
 func ListenQUIC(ln net.PacketConn, tlsConf *tls.Config, activeRequests *int64) (http3.QUICEarlyListener, error) {
 	lnKey := listenerKey("quic+"+ln.LocalAddr().Network(), ln.LocalAddr().String())
@@ -597,13 +655,17 @@ func (sqtc *sharedQUICTLSConfig) addTLSConfig(tlsConfig *tls.Config) (context.Co
 // sharedQuicListener is like sharedListener, but for quic.EarlyListeners.
 type sharedQuicListener struct {
 	*quic.EarlyListener
-	sqtc *sharedQUICTLSConfig
-	key  string
+	packetConn net.PacketConn // we have to hold these because quic-go won't close listeners it didn't create
+	sqtc       *sharedQUICTLSConfig
+	key        string
 }
 
-// Destruct closes the underlying QUIC listener.
+// Destruct closes the underlying QUIC listener and its associated net.PacketConn.
 func (sql *sharedQuicListener) Destruct() error {
-	return sql.EarlyListener.Close()
+	// close EarlyListener first to stop any operations being done to the net.PacketConn
+	_ = sql.EarlyListener.Close()
+	// then close the net.PacketConn
+	return sql.packetConn.Close()
 }
 
 // sharedPacketConn is like sharedListener, but for net.PacketConns.
@@ -652,6 +714,11 @@ var _ quic.OOBCapablePacketConn = (*fakeClosePacketConn)(nil)
 // but doesn't actually use these methods, the only methods needed are `ReadMsgUDP` and `SyscallConn`.
 var _ net.Conn = (*fakeClosePacketConn)(nil)
 
+// Unwrap returns the underlying net.UDPConn for quic-go optimization
+func (fcpc *fakeClosePacketConn) Unwrap() any {
+	return fcpc.UDPConn
+}
+
 // Close won't close the underlying socket unless there is no more reference, then listenerPool will close it.
 func (fcpc *fakeClosePacketConn) Close() error {
 	if atomic.CompareAndSwapInt32(&fcpc.closed, 0, 1) {

modules/caddyhttp/app.go

@@ -617,17 +617,6 @@ func (app *App) Stop() error {
 				zap.Error(err),
 				zap.Strings("addresses", server.Listen))
 		}
-
-		// TODO: we have to manually close our listeners because quic-go won't
-		// close listeners it didn't create along with the server itself...
-		// see https://github.com/quic-go/quic-go/issues/3560
-		for _, el := range server.h3listeners {
-			if err := el.Close(); err != nil {
-				app.logger.Error("HTTP/3 listener close",
-					zap.Error(err),
-					zap.String("address", el.LocalAddr().String()))
-			}
-		}
 	}
 	stopH2Listener := func(server *Server) {
 		defer finishedShutdown.Done()

modules/caddyhttp/server.go

@@ -228,7 +228,6 @@ type Server struct {
 
 	server      *http.Server
 	h3server    *http3.Server
-	h3listeners []net.PacketConn // TODO: we have to hold these because quic-go won't close listeners it didn't create
 	h2listeners []*http2Listener
 	addresses   []caddy.NetworkAddress
 
@@ -555,13 +554,7 @@ func (s *Server) findLastRouteWithHostMatcher() int {
 // the listener, with Server s as the handler.
 func (s *Server) serveHTTP3(addr caddy.NetworkAddress, tlsCfg *tls.Config) error {
 	addr.Network = getHTTP3Network(addr.Network)
-	lnAny, err := addr.Listen(s.ctx, 0, net.ListenConfig{})
-	if err != nil {
-		return err
-	}
-	ln := lnAny.(net.PacketConn)
-
-	h3ln, err := caddy.ListenQUIC(ln, tlsCfg, &s.activeRequests)
+	h3ln, err := addr.ListenQUIC(s.ctx, 0, net.ListenConfig{}, tlsCfg, &s.activeRequests)
 	if err != nil {
 		return fmt.Errorf("starting HTTP/3 QUIC listener: %v", err)
 	}
@@ -579,8 +572,6 @@ func (s *Server) serveHTTP3(addr caddy.NetworkAddress, tlsCfg *tls.Config) error
 		}
 	}
 
-	s.h3listeners = append(s.h3listeners, ln)
-
 	//nolint:errcheck
 	go s.h3server.ServeListener(h3ln)