Call UpdateQeo from Connection

src/core/connection.c

@@ -485,6 +485,45 @@ QuicConnUninitialize(
         QuicBindingRemoveConnection(Connection->Paths[0].Binding, Connection);
     }
 
+    CXPLAT_DBG_ASSERT(Connection->PathsCount >= 1);
+    QUIC_PATH* Path = &Connection->Paths[0];
+    if (Path->EncryptionOffloading) {
+        QUIC_CID_HASH_ENTRY* SourceCid =
+            CXPLAT_CONTAINING_RECORD(Connection->SourceCids.Next, QUIC_CID_HASH_ENTRY, Link);
+        CXPLAT_QEO_CONNECTION Offloads[] = {
+            {
+                CXPLAT_QEO_OPERATION_REMOVE,
+                CXPLAT_QEO_DIRECTION_TRANSMIT,
+                0,
+                0,
+                0, // Reserved:0
+                0,
+                0,
+                0,
+                Path->DestCid->CID.Length,
+            },
+            {
+                CXPLAT_QEO_OPERATION_REMOVE,
+                CXPLAT_QEO_DIRECTION_RECEIVE,
+                0,
+                0,
+                0, // Reserved:0
+                0,
+                0,
+                0,
+                SourceCid->CID.Length,
+            }
+        };
+
+        memcpy(Offloads[0].ConnectionId, Path->DestCid->CID.Data, Path->DestCid->CID.Length);
+        memcpy(Offloads[1].ConnectionId, SourceCid->CID.Data, SourceCid->CID.Length);
+        if (QUIC_SUCCEEDED(CxPlatSocketUpdateQeo(Path->Binding->Socket, Offloads, 2))) {
+            // Log
+        }
+        Connection->Stats.EncryptionOffloaded = FALSE;
+        Path->EncryptionOffloading = FALSE;
+    }
+
     //
     // Clean up the packet space first, to return any deferred received
     // packets back to the binding.
@@ -3933,7 +3972,7 @@ QuicConnRecvHeader(
         } else {
             Packet->KeyType = QuicPacketTypeToKeyTypeV1(Packet->LH->Type);
         }
-        Packet->Encrypted = TRUE;
+        Packet->Encrypted = !Connection->Paths[0].EncryptionOffloading;
 
     } else {
 
@@ -6716,6 +6755,7 @@ QuicConnGetV2Statistics(
     Stats->ResumptionAttempted = Connection->Stats.ResumptionAttempted;
     Stats->ResumptionSucceeded = Connection->Stats.ResumptionSucceeded;
     Stats->GreaseBitNegotiated = Connection->Stats.GreaseBitNegotiated;
+    Stats->EncryptionOffloaded = Connection->Stats.EncryptionOffloaded;
     Stats->EcnCapable = Path->EcnValidationState == ECN_VALIDATION_CAPABLE;
     Stats->Rtt = Path->SmoothedRtt;
     Stats->MinRtt = Path->MinRtt;
@@ -7230,6 +7270,12 @@ QuicConnApplyNewSettings(
         }
     }
 
+    if (Connection->State.Started &&
+        (Connection->Settings.EncryptionOffloadAllowed ^ Connection->Paths[0].EncryptionOffloading)) {
+        // TODO: enable/disable after start
+        CXPLAT_FRE_ASSERT(FALSE);
+    }
+
     uint8_t PeerStreamType =
         QuicConnIsServer(Connection) ?
             STREAM_ID_FLAG_IS_CLIENT : STREAM_ID_FLAG_IS_SERVER;

src/core/connection.h

@@ -243,6 +243,7 @@ typedef struct QUIC_CONN_STATS {
     uint32_t ResumptionAttempted    : 1;
     uint32_t ResumptionSucceeded    : 1;
     uint32_t GreaseBitNegotiated    : 1;
+    uint32_t EncryptionOffloaded    : 1;
 
     //
     // QUIC protocol version used. Network byte order.

src/core/crypto.c

@@ -1362,6 +1362,15 @@ QuicConnReceiveTP(
     return TRUE;
 }
 
+_Success_(return==TRUE)
+BOOLEAN
+QuicPacketKeyCreateOffload(
+    _Inout_ CXPLAT_TLS* TlsContext,
+    _In_z_ const char* const SecretName,
+    _Out_ CXPLAT_QEO_CONNECTION* Offloads,
+    _In_ uint32_t OffloadCount
+    );
+
 _IRQL_requires_max_(PASSIVE_LEVEL)
 void
 QuicCryptoProcessTlsCompletion(
@@ -1647,12 +1656,52 @@ QuicCryptoProcessTlsCompletion(
         }
         Connection->Stats.ResumptionSucceeded = Crypto->TlsState.SessionResumed;
 
+        CXPLAT_DBG_ASSERT(Connection->PathsCount == 1);
+        QUIC_PATH* Path = &Connection->Paths[0];
+
+        if (Path->IsActive && Connection->Settings.IsSet.EncryptionOffloadAllowed)
+        {
+            QUIC_CID_HASH_ENTRY* SourceCid =
+                CXPLAT_CONTAINING_RECORD(Connection->SourceCids.Next, QUIC_CID_HASH_ENTRY, Link);
+            CXPLAT_QEO_CONNECTION Offloads[] = {
+                {
+                    CXPLAT_QEO_OPERATION_ADD,
+                    CXPLAT_QEO_DIRECTION_TRANSMIT,
+                    CXPLAT_QEO_DECRYPT_FAILURE_ACTION_DROP,
+                    0, // KeyPhase
+                    0, // Reserved:0
+                    CXPLAT_QEO_CIPHER_TYPE_AEAD_AES_256_GCM,
+                    Connection->Send.NextPacketNumber,
+                    QuicAddrGetFamily(&Path->Route.LocalAddress),
+                    Path->DestCid->CID.Length,
+                },
+                {
+                    CXPLAT_QEO_OPERATION_ADD,
+                    CXPLAT_QEO_DIRECTION_RECEIVE,
+                    CXPLAT_QEO_DECRYPT_FAILURE_ACTION_DROP,
+                    0, // KeyPhase
+                    0, // Reserved:0
+                    CXPLAT_QEO_CIPHER_TYPE_AEAD_AES_256_GCM,
+                    0,
+                    QuicAddrGetFamily(&Path->Route.LocalAddress),
+                    SourceCid->CID.Length,
+                }
+            };
+            memcpy(Offloads[0].ConnectionId, Path->DestCid->CID.Data, Path->DestCid->CID.Length);
+            memcpy(Offloads[1].ConnectionId, SourceCid->CID.Data, SourceCid->CID.Length);
+            // TODO: query QEO capability and use before enabling
+            if (QuicPacketKeyCreateOffload(Crypto->TLS, "testing", Offloads, 2)) {
+                if (QUIC_SUCCEEDED(CxPlatSocketUpdateQeo(Path->Binding->Socket, Offloads, 2))) {
+                    Connection->Stats.EncryptionOffloaded = TRUE;
+                    Connection->Paths[0].EncryptionOffloading = TRUE;
+                }
+            }
+        }
+
         //
         // A handshake complete means the peer has been validated. Trigger MTU
         // discovery on path.
         //
-        CXPLAT_DBG_ASSERT(Connection->PathsCount == 1);
-        QUIC_PATH* Path = &Connection->Paths[0];
         QuicMtuDiscoveryPeerValidated(&Path->MtuDiscovery, Connection);
 
         if (QuicConnIsServer(Connection) &&

src/core/path.h

@@ -173,6 +173,11 @@ typedef struct QUIC_PATH {
     //
     uint32_t PathValidationStartTime;
 
+    //
+    // Indicates whether this connection offloads encryption workload to HW
+    //
+    BOOLEAN EncryptionOffloading : 1;
+
 } QUIC_PATH;
 
 #if DEBUG

src/core/quicdef.h

@@ -516,6 +516,11 @@ CXPLAT_STATIC_ASSERT(
 //
 #define QUIC_DEFAULT_HYSTART_ENABLED                FALSE
 
+//
+// The default settings for allowing QEO support.
+//
+#define QUIC_DEFAULT_ENCRYPTION_OFFLOAD_ALLOWED      FALSE
+
 //
 // The number of rounds in Cubic Slow Start to sample RTT.
 //
@@ -614,6 +619,7 @@ CXPLAT_STATIC_ASSERT(
 #define QUIC_SETTING_GREASE_QUIC_BIT_ENABLED        "GreaseQuicBitEnabled"
 #define QUIC_SETTING_ECN_ENABLED                    "EcnEnabled"
 #define QUIC_SETTING_HYSTART_ENABLED                "HyStartEnabled"
+#define QUIC_SETTING_ENCRYPTION_OFFLOAD_ALLOWED     "EncryptionOffloadAllowed"
 
 #define QUIC_SETTING_INITIAL_WINDOW_PACKETS         "InitialWindowPackets"
 #define QUIC_SETTING_SEND_IDLE_TIMEOUT_MS           "SendIdleTimeoutMs"

src/core/settings.c

@@ -144,6 +144,10 @@ QuicSettingsSetDefault(
     if (!Settings->IsSet.HyStartEnabled) {
         Settings->HyStartEnabled = QUIC_DEFAULT_HYSTART_ENABLED;
     }
+    if (!Settings->IsSet.EncryptionOffloadAllowed) {
+        Settings->EncryptionOffloadAllowed = QUIC_DEFAULT_ENCRYPTION_OFFLOAD_ALLOWED;
+    }
+
 }
 
 _IRQL_requires_max_(PASSIVE_LEVEL)
@@ -288,6 +292,9 @@ QuicSettingsCopy(
     if (!Destination->IsSet.HyStartEnabled) {
         Destination->HyStartEnabled = Source->HyStartEnabled;
     }
+    if (!Destination->IsSet.EncryptionOffloadAllowed) {
+        Destination->EncryptionOffloadAllowed = Source->EncryptionOffloadAllowed;
+    }
 }
 
 _IRQL_requires_max_(PASSIVE_LEVEL)
@@ -599,6 +606,11 @@ QuicSettingApply(
         return FALSE;
     }
 
+    if (Source->IsSet.EncryptionOffloadAllowed && (!Destination->IsSet.EncryptionOffloadAllowed || OverWrite)) {
+        Destination->EncryptionOffloadAllowed = Source->EncryptionOffloadAllowed;
+        Destination->IsSet.EncryptionOffloadAllowed = TRUE;
+    }
+
     return TRUE;
 }
 
@@ -1190,6 +1202,16 @@ QuicSettingsLoad(
             &ValueLen);
         Settings->HyStartEnabled = !!Value;
     }
+    if (!Settings->IsSet.EncryptionOffloadAllowed) {
+        Value = QUIC_DEFAULT_ENCRYPTION_OFFLOAD_ALLOWED;
+        ValueLen = sizeof(Value);
+        CxPlatStorageReadValue(
+            Storage,
+            QUIC_SETTING_ENCRYPTION_OFFLOAD_ALLOWED,
+            (uint8_t*)&Value,
+            &ValueLen);
+        Settings->EncryptionOffloadAllowed = !!Value;
+    }
 }
 
 _IRQL_requires_max_(PASSIVE_LEVEL)
@@ -1252,6 +1274,7 @@ QuicSettingsDump(
     QuicTraceLogVerbose(SettingGreaseQuicBitEnabled,        "[sett] GreaseQuicBitEnabled   = %hhu", Settings->GreaseQuicBitEnabled);
     QuicTraceLogVerbose(SettingEcnEnabled,                  "[sett] EcnEnabled             = %hhu", Settings->EcnEnabled);
     QuicTraceLogVerbose(SettingHyStartEnabled,              "[sett] HyStartEnabled         = %hhu", Settings->HyStartEnabled);
+    QuicTraceLogVerbose(SettingEncryptionOffloadAllowed,    "[sett] EncryptionOffloadAllowed = %hhu", Settings->EncryptionOffloadAllowed);
 }
 
 _IRQL_requires_max_(PASSIVE_LEVEL)
@@ -1392,6 +1415,9 @@ QuicSettingsDumpNew(
     if (Settings->IsSet.HyStartEnabled) {
         QuicTraceLogVerbose(SettingHyStartEnabled,                  "[sett] HyStartEnabled         = %hhu", Settings->HyStartEnabled);
     }
+    if (Settings->IsSet.EncryptionOffloadAllowed) {
+        QuicTraceLogVerbose(SettingEncryptionOffloadAllowed,        "[sett] EncryptionOffloadAllowed   = %hhu", Settings->EncryptionOffloadAllowed);
+    }
 }
 
 #define SETTINGS_SIZE_THRU_FIELD(SettingsType, Field) \
@@ -1595,6 +1621,14 @@ QuicSettingsSettingsToInternal(
         SettingsSize,
         InternalSettings);
 
+    SETTING_COPY_FLAG_TO_INTERNAL_SIZED(
+        Flags,
+        EncryptionOffloadAllowed,
+        QUIC_SETTINGS,
+        Settings,
+        SettingsSize,
+        InternalSettings);
+
     return QUIC_STATUS_SUCCESS;
 }
 
@@ -1703,6 +1737,14 @@ QuicSettingsGetSettings(
         *SettingsLength,
         InternalSettings);
 
+    SETTING_COPY_FLAG_FROM_INTERNAL_SIZED(
+        Flags,
+        EncryptionOffloadAllowed,
+        QUIC_SETTINGS,
+        Settings,
+        *SettingsLength,
+        InternalSettings);
+
     *SettingsLength = CXPLAT_MIN(*SettingsLength, sizeof(QUIC_SETTINGS));
 
     return QUIC_STATUS_SUCCESS;

src/core/settings.h

@@ -54,7 +54,8 @@ typedef struct QUIC_SETTINGS_INTERNAL {
             uint64_t GreaseQuicBitEnabled                   : 1;
             uint64_t EcnEnabled                             : 1;
             uint64_t HyStartEnabled                         : 1;
-            uint64_t RESERVED                               : 24;
+            uint64_t EncryptionOffloadAllowed               : 1;
+            uint64_t RESERVED                               : 23;
         } IsSet;
     };
 
@@ -97,6 +98,7 @@ typedef struct QUIC_SETTINGS_INTERNAL {
     uint8_t GreaseQuicBitEnabled            : 1;
     uint8_t EcnEnabled                      : 1;
     uint8_t HyStartEnabled                  : 1;
+    uint8_t EncryptionOffloadAllowed        : 1;
     uint8_t MtuDiscoveryMissingProbeCount;
 
 } QUIC_SETTINGS_INTERNAL;

src/core/unittest/SettingsTest.cpp

@@ -117,6 +117,7 @@ TEST(SettingsTest, TestAllSettingsFieldsSet)
     SETTINGS_FEATURE_SET_TEST(GreaseQuicBitEnabled, QuicSettingsSettingsToInternal);
     SETTINGS_FEATURE_SET_TEST(EcnEnabled, QuicSettingsSettingsToInternal);
     SETTINGS_FEATURE_SET_TEST(HyStartEnabled, QuicSettingsSettingsToInternal);
+    SETTINGS_FEATURE_SET_TEST(EncryptionOffloadAllowed, QuicSettingsSettingsToInternal);
 
     Settings.IsSetFlags = 0;
     Settings.IsSet.RESERVED = ~Settings.IsSet.RESERVED;
@@ -195,6 +196,7 @@ TEST(SettingsTest, TestAllSettingsFieldsGet)
     SETTINGS_FEATURE_GET_TEST(GreaseQuicBitEnabled, QuicSettingsGetSettings);
     SETTINGS_FEATURE_GET_TEST(EcnEnabled, QuicSettingsGetSettings);
     SETTINGS_FEATURE_GET_TEST(HyStartEnabled, QuicSettingsGetSettings);
+    SETTINGS_FEATURE_GET_TEST(EncryptionOffloadAllowed, QuicSettingsGetSettings);
 
     Settings.IsSetFlags = 0;
     Settings.IsSet.RESERVED = ~Settings.IsSet.RESERVED;

src/inc/msquic.h

@@ -500,7 +500,8 @@ typedef struct QUIC_STATISTICS_V2 {
     uint32_t ResumptionSucceeded    : 1;
     uint32_t GreaseBitNegotiated    : 1;    // Set if we negotiated the GREASE bit.
     uint32_t EcnCapable             : 1;
-    uint32_t RESERVED               : 26;
+    uint32_t EncryptionOffloaded    : 1;    // At least one path successfully offloaded encryption
+    uint32_t RESERVED               : 25;
     uint32_t Rtt;                           // In microseconds
     uint32_t MinRtt;                        // In microseconds
     uint32_t MaxRtt;                        // In microseconds
@@ -663,7 +664,12 @@ typedef struct QUIC_SETTINGS {
             uint64_t GreaseQuicBitEnabled                   : 1;
             uint64_t EcnEnabled                             : 1;
             uint64_t HyStartEnabled                         : 1;
+#ifdef QUIC_API_ENABLE_PREVIEW_FEATURES
+            uint64_t EncryptionOffloadAllowed               : 1;
+            uint64_t RESERVED                               : 28;
+#else
             uint64_t RESERVED                               : 29;
+#endif
         } IsSet;
     };
 
@@ -704,12 +710,16 @@ typedef struct QUIC_SETTINGS {
     union {
         uint64_t Flags;
         struct {
-            uint64_t HyStartEnabled : 1;
-            uint64_t ReservedFlags : 63;
+            uint64_t HyStartEnabled            : 1;
+#ifdef QUIC_API_ENABLE_PREVIEW_FEATURES
+            uint64_t EncryptionOffloadAllowed  : 1;
+            uint64_t ReservedFlags             : 62;
+#else
+            uint64_t ReservedFlags             : 63;
+#endif
         };
     };
 
-
 } QUIC_SETTINGS;
 
 //

src/inc/msquic.hpp

@@ -439,6 +439,9 @@ class MsQuicSettings : public QUIC_SETTINGS {
     MsQuicSettings& SetDestCidUpdateIdleTimeoutMs(uint32_t Value) { DestCidUpdateIdleTimeoutMs = Value; IsSet.DestCidUpdateIdleTimeoutMs = TRUE; return *this; }
     MsQuicSettings& SetGreaseQuicBitEnabled(bool Value) { GreaseQuicBitEnabled = Value; IsSet.GreaseQuicBitEnabled = TRUE; return *this; }
     MsQuicSettings& SetEcnEnabled(bool Value) { EcnEnabled = Value; IsSet.EcnEnabled = TRUE; return *this; }
+#ifdef QUIC_API_ENABLE_PREVIEW_FEATURES
+    MsQuicSettings& SetEncryptionOffloadAllowed(bool Value) { EncryptionOffloadAllowed = Value; IsSet.EncryptionOffloadAllowed = TRUE; return *this; }
+#endif
 
     QUIC_STATUS
     SetGlobal() const noexcept {

src/inc/quic_crypt.h

@@ -231,6 +231,16 @@ QuicPacketKeyDerive(
     _Out_ QUIC_PACKET_KEY **NewKey
     );
 
+#include "quic_datapath.h"
+_IRQL_requires_max_(DISPATCH_LEVEL)
+QUIC_STATUS
+QuicPacketKeyDeriveOffload(
+    _In_ const QUIC_HKDF_LABELS* HkdfLabels,
+    _In_ const CXPLAT_SECRET* const Secret,
+    _In_z_ const char* const SecretName,
+    _Inout_ CXPLAT_QEO_CONNECTION* Offload
+    );
+
 _IRQL_requires_max_(DISPATCH_LEVEL)
 void
 CxPlatKeyFree(

src/inc/quic_datapath.h

@@ -270,7 +270,7 @@ typedef struct CXPLAT_QEO_CONNECTION {
     uint32_t RESERVED             : 12; // Must be set to 0. Don't read.
     uint32_t CipherType           : 16; // CXPLAT_QEO_CIPHER_TYPE
     uint64_t NextPacketNumber;
-    QUIC_ADDR Address;
+    QUIC_ADDRESS_FAMILY Address;
     uint8_t ConnectionIdLength;
     uint8_t ConnectionId[20]; // QUIC v1 and v2 max CID size
     uint8_t PayloadKey[32];   // Length determined by CipherType