Ensure that CipherSuite and Cert algorithms match

Filter the list of local certificate based CipherSuites so
that they match the signature algorithm of the chosen certificate.

From rfc5246#section-7.4.3

```
In addition, the hash and signature algorithms MUST be compatible
with the key in the server's end-entity certificate.  RSA keys MAY be
used with any permitted hash algorithm, subject to restrictions in
the certificate, if any.
```

AUTHORS.txt

@@ -27,6 +27,8 @@ Jozef Kralik <[email protected]>
 Julien Salleyron <[email protected]>
 Kegan Dougal <[email protected]>
 Lander Noterman <[email protected]>
+Len <[email protected]>
+Lukas Lihotzki <[email protected]>
 Michael Zabka <[email protected]>
 Michiel De Backker <[email protected]>
 Robert Eperjesi <[email protected]>

cipher_suite.go

@@ -1,6 +1,10 @@
 package dtls
 
 import (
+	"crypto/ecdsa"
+	"crypto/ed25519"
+	"crypto/rsa"
+	"crypto/tls"
 	"fmt"
 	"hash"
 
@@ -222,3 +226,24 @@ func parseCipherSuites(userSelectedSuites []CipherSuiteID, customCipherSuites fu
 
 	return cipherSuites[:i], nil
 }
+
+func filterCipherSuitesForCertificate(cert *tls.Certificate, cipherSuites []CipherSuite) []CipherSuite {
+	if cert == nil || cert.PrivateKey == nil {
+		return cipherSuites
+	}
+	var certType clientcertificate.Type
+	switch cert.PrivateKey.(type) {
+	case ed25519.PrivateKey, *ecdsa.PrivateKey:
+		certType = clientcertificate.ECDSASign
+	case *rsa.PrivateKey:
+		certType = clientcertificate.RSASign
+	}
+
+	filtered := []CipherSuite{}
+	for _, c := range cipherSuites {
+		if c.AuthenticationType() != CipherSuiteAuthenticationTypeCertificate || certType == c.CertificateType() {
+			filtered = append(filtered, c)
+		}
+	}
+	return filtered
+}

conn.go

@@ -176,6 +176,12 @@ func createConn(ctx context.Context, nextConn net.Conn, config *Config, isClient
 		sessionStore:                config.SessionStore,
 	}
 
+	cert, err := hsCfg.getCertificate(serverName)
+	if err != nil && !errors.Is(err, errNoCertificates) {
+		return nil, err
+	}
+	hsCfg.localCipherSuites = filterCipherSuitesForCertificate(cert, cipherSuites)
+
 	var initialFlight flightVal
 	var initialFSMState handshakeState
 

conn_test.go

@@ -3,7 +3,11 @@ package dtls
 import (
 	"bytes"
 	"context"
+	"crypto"
+	"crypto/ecdsa"
+	cryptoElliptic "crypto/elliptic"
 	"crypto/rand"
+	"crypto/rsa"
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/hex"
@@ -2299,3 +2303,84 @@ func (ms *memSessStore) Del(key []byte) error {
 
 	return nil
 }
+
+// Assert that the server only uses CipherSuites with a hash+signature that matches
+// the certificate. As specified in rfc5246#section-7.4.3
+func TestCipherSuiteMatchesCertificateType(t *testing.T) {
+	// Limit runtime in case of deadlocks
+	lim := test.TimeOut(time.Second * 20)
+	defer lim.Stop()
+
+	// Check for leaking routines
+	report := test.CheckRoutines(t)
+	defer report()
+
+	for _, test := range []struct {
+		Name           string
+		cipherList     []CipherSuiteID
+		expectedCipher CipherSuiteID
+		generateRSA    bool
+	}{
+		{
+			Name:           "ECDSA Certificate with RSA CipherSuite first",
+			cipherList:     []CipherSuiteID{TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256},
+			expectedCipher: TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
+		},
+		{
+			Name:           "RSA Certificate with ECDSA CipherSuite first",
+			cipherList:     []CipherSuiteID{TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256},
+			expectedCipher: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
+			generateRSA:    true,
+		},
+	} {
+		test := test
+		t.Run(test.Name, func(t *testing.T) {
+			clientErr := make(chan error, 1)
+			client := make(chan *Conn, 1)
+
+			ca, cb := dpipe.Pipe()
+			go func() {
+				c, err := testClient(context.TODO(), ca, &Config{CipherSuites: test.cipherList}, false)
+				clientErr <- err
+				client <- c
+			}()
+
+			var (
+				priv crypto.PrivateKey
+				err  error
+			)
+
+			if test.generateRSA {
+				if priv, err = rsa.GenerateKey(rand.Reader, 2048); err != nil {
+					t.Fatal(err)
+				}
+			} else {
+				if priv, err = ecdsa.GenerateKey(cryptoElliptic.P256(), rand.Reader); err != nil {
+					t.Fatal(err)
+				}
+			}
+
+			serverCert, err := selfsign.SelfSign(priv)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if s, err := testServer(context.TODO(), cb, &Config{
+				CipherSuites: test.cipherList,
+				Certificates: []tls.Certificate{serverCert},
+			}, false); err != nil {
+				t.Fatal(err)
+			} else if err = s.Close(); err != nil {
+				t.Fatal(err)
+			}
+
+			if c, err := <-client, <-clientErr; err != nil {
+				t.Fatal(err)
+			} else if err := c.Close(); err != nil {
+				t.Fatal(err)
+			} else if c.ConnectionState().cipherSuite.ID() != test.expectedCipher {
+				t.Fatalf("Expected(%s) and Actual(%s) CipherSuite do not match", test.expectedCipher, c.ConnectionState().cipherSuite.ID())
+			}
+		})
+	}
+}

e2e/e2e_openssl_test.go

@@ -174,7 +174,7 @@ func ciphersOpenSSL(cfg *dtls.Config) string {
 		dtls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384:   "ECDHE-RSA-AES256-GCM-SHA384",
 
 		dtls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA: "ECDHE-ECDSA-AES256-SHA",
-		dtls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA:   "ECDHE-RSA-AES128-SHA",
+		dtls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA:   "ECDHE-RSA-AES256-SHA",
 
 		dtls.TLS_PSK_WITH_AES_128_CCM:        "PSK-AES128-CCM",
 		dtls.TLS_PSK_WITH_AES_128_CCM_8:      "PSK-AES128-CCM8",

e2e/e2e_v113_test.go

@@ -30,7 +30,6 @@ func testPionE2ESimpleED25519(t *testing.T, server, client func(*comm)) {
 		dtls.TLS_ECDHE_ECDSA_WITH_AES_128_CCM_8,
 		dtls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
 		dtls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
-		dtls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
 		dtls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,
 	} {
 		cipherSuite := cipherSuite

pkg/crypto/selfsign/selfsign.go

@@ -7,6 +7,7 @@ import (
 	"crypto/ed25519"
 	"crypto/elliptic"
 	"crypto/rand"
+	"crypto/rsa"
 	"crypto/tls"
 	"crypto/x509"
 	"encoding/hex"
@@ -54,6 +55,8 @@ func WithDNS(key crypto.PrivateKey, cn string, sans ...string) (tls.Certificate,
 		pubKey = k.Public()
 	case *ecdsa.PrivateKey:
 		pubKey = k.Public()
+	case *rsa.PrivateKey:
+		pubKey = k.Public()
 	default:
 		return tls.Certificate{}, errInvalidPrivateKey
 	}