diff --git a/crypto/chacha/asm/chacha-armv8.pl b/crypto/chacha/asm/chacha-armv8.pl index a519b5f207..6818da2145 100755 --- a/crypto/chacha/asm/chacha-armv8.pl +++ b/crypto/chacha/asm/chacha-armv8.pl @@ -122,9 +122,6 @@ sub ROUND { $code.=<<___; #include -.extern OPENSSL_armcap_P -.hidden OPENSSL_armcap_P - .section .rodata .align 5 @@ -136,24 +133,10 @@ sub ROUND { .text -.globl ChaCha20_ctr32 -.type ChaCha20_ctr32,%function +.globl ChaCha20_ctr32_nohw +.type ChaCha20_ctr32_nohw,%function .align 5 -ChaCha20_ctr32: - AARCH64_VALID_CALL_TARGET - cbz $len,.Labort -#if defined(OPENSSL_HWASAN) && __clang_major__ >= 10 - adrp @x[0],:pg_hi21_nc:OPENSSL_armcap_P -#else - adrp @x[0],:pg_hi21:OPENSSL_armcap_P -#endif - cmp $len,#192 - b.lo .Lshort - ldr w17,[@x[0],:lo12:OPENSSL_armcap_P] - tst w17,#ARMV7_NEON - b.ne ChaCha20_neon - -.Lshort: +ChaCha20_ctr32_nohw: AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-96]! add x29,sp,#0 @@ -276,7 +259,6 @@ sub ROUND { ldp x27,x28,[x29,#80] ldp x29,x30,[sp],#96 AARCH64_VALIDATE_LINK_REGISTER -.Labort: ret .align 4 @@ -334,7 +316,7 @@ sub ROUND { ldp x29,x30,[sp],#96 AARCH64_VALIDATE_LINK_REGISTER ret -.size ChaCha20_ctr32,.-ChaCha20_ctr32 +.size ChaCha20_ctr32_nohw,.-ChaCha20_ctr32_nohw ___ {{{ @@ -375,9 +357,10 @@ sub NEONROUND { $code.=<<___; -.type ChaCha20_neon,%function +.globl ChaCha20_ctr32_neon +.type ChaCha20_ctr32_neon,%function .align 5 -ChaCha20_neon: +ChaCha20_ctr32_neon: AARCH64_SIGN_LINK_REGISTER stp x29,x30,[sp,#-96]! add x29,sp,#0 @@ -690,7 +673,7 @@ sub NEONROUND { ldp x29,x30,[sp],#96 AARCH64_VALIDATE_LINK_REGISTER ret -.size ChaCha20_neon,.-ChaCha20_neon +.size ChaCha20_ctr32_neon,.-ChaCha20_ctr32_neon ___ { my ($T0,$T1,$T2,$T3,$T4,$T5)=@K; diff --git a/crypto/chacha/chacha.c b/crypto/chacha/chacha.c index a4d88c0a40..52102ab06b 100644 --- a/crypto/chacha/chacha.c +++ b/crypto/chacha/chacha.c @@ -60,7 +60,22 @@ void CRYPTO_hchacha20(uint8_t out[32], const uint8_t key[32], OPENSSL_memcpy(&out[16], &x[12], sizeof(uint32_t) * 4); } -#if defined(CHACHA20_ASM) +#if defined(CHACHA20_ASM_NOHW) +static void ChaCha20_ctr32(uint8_t *out, const uint8_t *in, size_t in_len, + const uint32_t key[8], const uint32_t counter[4]) { +#if defined(CHACHA20_ASM_NEON) + if (ChaCha20_ctr32_neon_capable(in_len)) { + ChaCha20_ctr32_neon(out, in, in_len, key, counter); + return; + } +#endif + if (in_len > 0) { + ChaCha20_ctr32_nohw(out, in, in_len, key, counter); + } +} +#endif + +#if defined(CHACHA20_ASM) || defined(CHACHA20_ASM_NOHW) void CRYPTO_chacha_20(uint8_t *out, const uint8_t *in, size_t in_len, const uint8_t key[32], const uint8_t nonce[12], diff --git a/crypto/chacha/chacha_test.cc b/crypto/chacha/chacha_test.cc index d4e5332b40..6c03502b8f 100644 --- a/crypto/chacha/chacha_test.cc +++ b/crypto/chacha/chacha_test.cc @@ -347,7 +347,25 @@ TEST(ChaChaTest, CounterOverflow) { } } -#if defined(CHACHA20_ASM) && defined(SUPPORTS_ABI_TEST) +#if defined(SUPPORTS_ABI_TEST) + +static void check_abi(uint8_t *out, const uint8_t *in, size_t in_len, + const uint32_t key[8], const uint32_t counter[4]) { +#if defined(CHACHA20_ASM) + CHECK_ABI(ChaCha20_ctr32, out, in, in_len, key, counter); +#endif +#if defined(CHACHA20_ASM_NEON) + if (ChaCha20_ctr32_neon_capable(in_len)) { + CHECK_ABI(ChaCha20_ctr32_neon, out, in, in_len, key, counter); + } +#endif +#if defined(CHACHA20_ASM_NOHW) + if (in_len > 0) { + CHECK_ABI(ChaCha20_ctr32_nohw, out, in, in_len, key, counter); + } +#endif +} + TEST(ChaChaTest, ABI) { uint32_t key[8]; OPENSSL_memcpy(key, kKey, sizeof(key)); @@ -357,14 +375,15 @@ TEST(ChaChaTest, ABI) { auto buf = std::make_unique(sizeof(kInput)); for (size_t len = 0; len <= 32; len++) { SCOPED_TRACE(len); - CHECK_ABI(ChaCha20_ctr32, buf.get(), kInput, len, key, kCounterNonce); + check_abi(buf.get(), kInput, len, key, kCounterNonce); } for (size_t len : {32 * 2, 32 * 4, 32 * 8, 32 * 16, 32 * 24}) { SCOPED_TRACE(len); - CHECK_ABI(ChaCha20_ctr32, buf.get(), kInput, len, key, kCounterNonce); + check_abi(buf.get(), kInput, len, key, kCounterNonce); // Cover the partial block paths. - CHECK_ABI(ChaCha20_ctr32, buf.get(), kInput, len + 15, key, kCounterNonce); + check_abi(buf.get(), kInput, len + 15, key, kCounterNonce); } } -#endif // CHACHA20_ASM && SUPPORTS_ABI_TEST + +#endif // SUPPORTS_ABI_TEST diff --git a/crypto/chacha/internal.h b/crypto/chacha/internal.h index 5f442ec461..a8ae3cb84b 100644 --- a/crypto/chacha/internal.h +++ b/crypto/chacha/internal.h @@ -17,6 +17,8 @@ #include +#include "../internal.h" + #if defined(__cplusplus) extern "C" { #endif @@ -27,11 +29,24 @@ extern "C" { void CRYPTO_hchacha20(uint8_t out[32], const uint8_t key[32], const uint8_t nonce[16]); -#if !defined(OPENSSL_NO_ASM) && \ - (defined(OPENSSL_X86) || defined(OPENSSL_X86_64) || \ - defined(OPENSSL_ARM) || defined(OPENSSL_AARCH64)) +#if !defined(OPENSSL_NO_ASM) && \ + (defined(OPENSSL_X86) || defined(OPENSSL_X86_64) || defined(OPENSSL_ARM)) + #define CHACHA20_ASM +#elif !defined(OPENSSL_NO_ASM) && defined(OPENSSL_AARCH64) + +#define CHACHA20_ASM_NOHW + +#define CHACHA20_ASM_NEON +OPENSSL_INLINE int ChaCha20_ctr32_neon_capable(size_t len) { + return (len >= 192) && CRYPTO_is_NEON_capable(); +} +void ChaCha20_ctr32_neon(uint8_t *out, const uint8_t *in, size_t in_len, + const uint32_t key[8], const uint32_t counter[4]); +#endif + +#if defined(CHACHA20_ASM) // ChaCha20_ctr32 encrypts |in_len| bytes from |in| and writes the result to // |out|. If |in| and |out| alias, they must be equal. // @@ -44,6 +59,12 @@ void ChaCha20_ctr32(uint8_t *out, const uint8_t *in, size_t in_len, const uint32_t key[8], const uint32_t counter[4]); #endif +#if defined(CHACHA20_ASM_NOHW) +// ChaCha20_ctr32_nohw is like |ChaCha20_ctr32| except |in_len| must be nonzero. +void ChaCha20_ctr32_nohw(uint8_t *out, const uint8_t *in, size_t in_len, + const uint32_t key[8], const uint32_t counter[4]); +#endif + #if defined(__cplusplus) } // extern C