From 72917f02411eb9ec388fa97293e6117b24b61d18 Mon Sep 17 00:00:00 2001 From: Jack Lloyd Date: Wed, 27 Mar 2024 15:53:39 -0400 Subject: [PATCH] Modify mp algos to be templatized over the word type, and constexpr --- src/lib/math/bigint/bigint.cpp | 2 +- src/lib/math/mp/mp_asmi.h | 714 +++++++++++++++----------- src/lib/math/mp/mp_core.h | 308 ++++++----- src/lib/math/numbertheory/reducer.cpp | 2 +- src/lib/utils/ct_utils.h | 68 +-- src/lib/utils/mul128.h | 7 - src/lib/utils/types.h | 7 + src/tests/test_mp.cpp | 28 +- 8 files changed, 643 insertions(+), 493 deletions(-) diff --git a/src/lib/math/bigint/bigint.cpp b/src/lib/math/bigint/bigint.cpp index fef58d986a4..41f0817344c 100644 --- a/src/lib/math/bigint/bigint.cpp +++ b/src/lib/math/bigint/bigint.cpp @@ -469,7 +469,7 @@ void BigInt::ct_cond_swap(bool predicate, BigInt& other) { grow_to(max_words); other.grow_to(max_words); - bigint_cnd_swap(predicate, this->mutable_data(), other.mutable_data(), max_words); + bigint_cnd_swap(static_cast(predicate), this->mutable_data(), other.mutable_data(), max_words); } void BigInt::cond_flip_sign(bool predicate) { diff --git a/src/lib/math/mp/mp_asmi.h b/src/lib/math/mp/mp_asmi.h index 3aa6d60cc6d..307645c40f8 100644 --- a/src/lib/math/mp/mp_asmi.h +++ b/src/lib/math/mp/mp_asmi.h @@ -11,115 +11,138 @@ #include -#if BOTAN_MP_WORD_BITS == 64 +#if !defined(BOTAN_TARGET_HAS_NATIVE_UINT128) #include - #include #endif namespace Botan { -#if BOTAN_MP_WORD_BITS == 32 -typedef uint64_t dword; - #define BOTAN_HAS_NATIVE_DWORD - -#elif BOTAN_MP_WORD_BITS == 64 - #if defined(BOTAN_TARGET_HAS_NATIVE_UINT128) -typedef uint128_t dword; - #define BOTAN_HAS_NATIVE_DWORD - #else -typedef donna128 dword; - #endif - -#else - #error BOTAN_MP_WORD_BITS must be 32 or 64 -#endif - #if defined(BOTAN_USE_GCC_INLINE_ASM) - - #if defined(BOTAN_TARGET_ARCH_IS_X86_32) && (BOTAN_MP_WORD_BITS == 32) + #if defined(BOTAN_TARGET_ARCH_IS_X86_FAMILY) #define BOTAN_MP_USE_X86_32_ASM - #elif defined(BOTAN_TARGET_ARCH_IS_X86_64) && (BOTAN_MP_WORD_BITS == 64) + #endif + + #if defined(BOTAN_TARGET_ARCH_IS_X86_64) #define BOTAN_MP_USE_X86_64_ASM #endif +#endif +/* +* Concept for allowed multiprecision word types +*/ +template +concept WordType = (std::same_as || std::same_as); + +template +struct DwordType {}; + +template <> +struct DwordType { + public: + typedef uint64_t type; + static const bool is_native = true; +}; + +template <> +struct DwordType { + public: +#if defined(BOTAN_TARGET_HAS_NATIVE_UINT128) + typedef uint128_t type; + static const bool is_native = true; +#else + typedef donna128 type; + static const bool is_native = false; #endif +}; /* * Word Multiply/Add */ -inline word word_madd2(word a, word b, word* c) { +template +inline constexpr auto word_madd2(W a, W b, W* c) -> W { #if defined(BOTAN_MP_USE_X86_32_ASM) - asm(R"( - mull %[b] - addl %[c],%[a] - adcl $0,%[carry] - )" - : [a] "=a"(a), [b] "=rm"(b), [carry] "=&d"(*c) - : "0"(a), "1"(b), [c] "g"(*c) - : "cc"); - - return a; - -#elif defined(BOTAN_MP_USE_X86_64_ASM) - asm(R"( - mulq %[b] - addq %[c],%[a] - adcq $0,%[carry] - )" - : [a] "=a"(a), [b] "=rm"(b), [carry] "=&d"(*c) - : "0"(a), "1"(b), [c] "g"(*c) - : "cc"); - - return a; + if(std::same_as && !std::is_constant_evaluated()) { + asm(R"( + mull %[b] + addl %[c],%[a] + adcl $0,%[carry] + )" + : [a] "=a"(a), [b] "=rm"(b), [carry] "=&d"(*c) + : "0"(a), "1"(b), [c] "g"(*c) + : "cc"); + + return a; + } +#endif -#else - const dword s = static_cast(a) * b + *c; - *c = static_cast(s >> BOTAN_MP_WORD_BITS); - return static_cast(s); +#if defined(BOTAN_MP_USE_X86_64_ASM) + if(std::same_as && !std::is_constant_evaluated()) { + asm(R"( + mulq %[b] + addq %[c],%[a] + adcq $0,%[carry] + )" + : [a] "=a"(a), [b] "=rm"(b), [carry] "=&d"(*c) + : "0"(a), "1"(b), [c] "g"(*c) + : "cc"); + + return a; + } #endif + + typedef typename DwordType::type dword; + const dword s = dword(a) * b + *c; + *c = static_cast(s >> (sizeof(W) * 8)); + return static_cast(s); } /* * Word Multiply/Add */ -inline word word_madd3(word a, word b, word c, word* d) { +template +inline constexpr auto word_madd3(W a, W b, W c, W* d) -> W { #if defined(BOTAN_MP_USE_X86_32_ASM) - asm(R"( - mull %[b] - - addl %[c],%[a] - adcl $0,%[carry] - - addl %[d],%[a] - adcl $0,%[carry] - )" - : [a] "=a"(a), [b] "=rm"(b), [carry] "=&d"(*d) - : "0"(a), "1"(b), [c] "g"(c), [d] "g"(*d) - : "cc"); - - return a; - -#elif defined(BOTAN_MP_USE_X86_64_ASM) - asm(R"( - mulq %[b] + if(std::same_as && !std::is_constant_evaluated()) { + asm(R"( + mull %[b] + + addl %[c],%[a] + adcl $0,%[carry] + + addl %[d],%[a] + adcl $0,%[carry] + )" + : [a] "=a"(a), [b] "=rm"(b), [carry] "=&d"(*d) + : "0"(a), "1"(b), [c] "g"(c), [d] "g"(*d) + : "cc"); + + return a; + } +#endif - addq %[c],%[a] - adcq $0,%[carry] +#if defined(BOTAN_MP_USE_X86_64_ASM) + if(std::same_as && !std::is_constant_evaluated()) { + asm(R"( + mulq %[b] - addq %[d],%[a] - adcq $0,%[carry] - )" - : [a] "=a"(a), [b] "=rm"(b), [carry] "=&d"(*d) - : "0"(a), "1"(b), [c] "g"(c), [d] "g"(*d) - : "cc"); + addq %[c],%[a] + adcq $0,%[carry] - return a; + addq %[d],%[a] + adcq $0,%[carry] + )" + : [a] "=a"(a), [b] "=rm"(b), [carry] "=&d"(*d) + : "0"(a), "1"(b), [c] "g"(c), [d] "g"(*d) + : "cc"); -#else - const dword s = static_cast(a) * b + c + *d; - *d = static_cast(s >> BOTAN_MP_WORD_BITS); - return static_cast(s); + return a; + } #endif + + typedef typename DwordType::type dword; + const dword s = dword(a) * b + c + *d; + *d = static_cast(s >> (sizeof(W) * 8)); + return static_cast(s); } #if defined(BOTAN_MP_USE_X86_32_ASM) @@ -213,49 +236,60 @@ inline word word_madd3(word a, word b, word c, word* d) { /* * Word Addition */ -inline word word_add(word x, word y, word* carry) { +template +inline constexpr auto word_add(W x, W y, W* carry) -> W { #if defined(BOTAN_MP_USE_X86_32_ASM) - asm(ADD_OR_SUBTRACT(ASM("adcl %[y],%[x]")) - : [x] "=r"(x), [carry] "=r"(*carry) - : "0"(x), [y] "rm"(y), "1"(*carry) - : "cc"); - return x; - -#elif defined(BOTAN_MP_USE_X86_64_ASM) + if(std::same_as && !std::is_constant_evaluated()) { + asm(ADD_OR_SUBTRACT(ASM("adcl %[y],%[x]")) + : [x] "=r"(x), [carry] "=r"(*carry) + : "0"(x), [y] "rm"(y), "1"(*carry) + : "cc"); + return x; + } +#endif - asm(ADD_OR_SUBTRACT(ASM("adcq %[y],%[x]")) - : [x] "=r"(x), [carry] "=r"(*carry) - : "0"(x), [y] "rm"(y), "1"(*carry) - : "cc"); - return x; +#if defined(BOTAN_MP_USE_X86_64_ASM) + if(std::same_as && !std::is_constant_evaluated()) { + asm(ADD_OR_SUBTRACT(ASM("adcq %[y],%[x]")) + : [x] "=r"(x), [carry] "=r"(*carry) + : "0"(x), [y] "rm"(y), "1"(*carry) + : "cc"); + return x; + } +#endif -#else - word z = x + y; - word c1 = (z < x); + W z = x + y; + W c1 = (z < x); z += *carry; *carry = c1 | (z < *carry); return z; -#endif } /* * Eight Word Block Addition, Two Argument */ -inline word word8_add2(word x[8], const word y[8], word carry) { +template +inline constexpr auto word8_add2(W x[8], const W y[8], W carry) -> W { #if defined(BOTAN_MP_USE_X86_32_ASM) - asm(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB2_OP, "adcl")) - : [carry] "=r"(carry) - : [x] "r"(x), [y] "r"(y), "0"(carry) - : "cc", "memory"); - -#elif defined(BOTAN_MP_USE_X86_64_ASM) + if(std::same_as && !std::is_constant_evaluated()) { + asm(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB2_OP, "adcl")) + : [carry] "=r"(carry) + : [x] "r"(x), [y] "r"(y), "0"(carry) + : "cc", "memory"); + return carry; + } +#endif - asm(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB2_OP, "adcq")) - : [carry] "=r"(carry) - : [x] "r"(x), [y] "r"(y), "0"(carry) - : "cc", "memory"); +#if defined(BOTAN_MP_USE_X86_64_ASM) + if(std::same_as && !std::is_constant_evaluated()) { + asm(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB2_OP, "adcq")) + : [carry] "=r"(carry) + : [x] "r"(x), [y] "r"(y), "0"(carry) + : "cc", "memory"); + return carry; + } +#endif -#else x[0] = word_add(x[0], y[0], &carry); x[1] = word_add(x[1], y[1], &carry); x[2] = word_add(x[2], y[2], &carry); @@ -264,28 +298,34 @@ inline word word8_add2(word x[8], const word y[8], word carry) { x[5] = word_add(x[5], y[5], &carry); x[6] = word_add(x[6], y[6], &carry); x[7] = word_add(x[7], y[7], &carry); -#endif - return carry; } /* * Eight Word Block Addition, Three Argument */ -inline word word8_add3(word z[8], const word x[8], const word y[8], word carry) { +template +inline constexpr auto word8_add3(W z[8], const W x[8], const W y[8], W carry) -> W { #if defined(BOTAN_MP_USE_X86_32_ASM) - asm(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB3_OP, "adcl")) - : [carry] "=r"(carry) - : [x] "r"(x), [y] "r"(y), [z] "r"(z), "0"(carry) - : "cc", "memory"); + if(std::same_as && !std::is_constant_evaluated()) { + asm(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB3_OP, "adcl")) + : [carry] "=r"(carry) + : [x] "r"(x), [y] "r"(y), [z] "r"(z), "0"(carry) + : "cc", "memory"); + return carry; + } +#endif -#elif defined(BOTAN_MP_USE_X86_64_ASM) - asm(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB3_OP, "adcq")) - : [carry] "=r"(carry) - : [x] "r"(x), [y] "r"(y), [z] "r"(z), "0"(carry) - : "cc", "memory"); +#if defined(BOTAN_MP_USE_X86_64_ASM) + if(std::same_as && !std::is_constant_evaluated()) { + asm(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB3_OP, "adcq")) + : [carry] "=r"(carry) + : [x] "r"(x), [y] "r"(y), [z] "r"(z), "0"(carry) + : "cc", "memory"); + return carry; + } +#endif -#else z[0] = word_add(x[0], y[0], &carry); z[1] = word_add(x[1], y[1], &carry); z[2] = word_add(x[2], y[2], &carry); @@ -294,55 +334,66 @@ inline word word8_add3(word z[8], const word x[8], const word y[8], word carry) z[5] = word_add(x[5], y[5], &carry); z[6] = word_add(x[6], y[6], &carry); z[7] = word_add(x[7], y[7], &carry); -#endif - return carry; } /* * Word Subtraction */ -inline word word_sub(word x, word y, word* carry) { +template +inline constexpr auto word_sub(W x, W y, W* carry) -> W { #if defined(BOTAN_MP_USE_X86_32_ASM) - asm(ADD_OR_SUBTRACT(ASM("sbbl %[y],%[x]")) - : [x] "=r"(x), [carry] "=r"(*carry) - : "0"(x), [y] "rm"(y), "1"(*carry) - : "cc"); - return x; + if(std::same_as && !std::is_constant_evaluated()) { + asm(ADD_OR_SUBTRACT(ASM("sbbl %[y],%[x]")) + : [x] "=r"(x), [carry] "=r"(*carry) + : "0"(x), [y] "rm"(y), "1"(*carry) + : "cc"); + return x; + } +#endif -#elif defined(BOTAN_MP_USE_X86_64_ASM) - asm(ADD_OR_SUBTRACT(ASM("sbbq %[y],%[x]")) - : [x] "=r"(x), [carry] "=r"(*carry) - : "0"(x), [y] "rm"(y), "1"(*carry) - : "cc"); - return x; +#if defined(BOTAN_MP_USE_X86_64_ASM) + if(std::same_as && !std::is_constant_evaluated()) { + asm(ADD_OR_SUBTRACT(ASM("sbbq %[y],%[x]")) + : [x] "=r"(x), [carry] "=r"(*carry) + : "0"(x), [y] "rm"(y), "1"(*carry) + : "cc"); + return x; + } +#endif -#else - word t0 = x - y; - word c1 = (t0 > x); - word z = t0 - *carry; + W t0 = x - y; + W c1 = (t0 > x); + W z = t0 - *carry; *carry = c1 | (z > t0); return z; -#endif } /* * Eight Word Block Subtraction, Two Argument */ -inline word word8_sub2(word x[8], const word y[8], word carry) { +template +inline constexpr auto word8_sub2(W x[8], const W y[8], W carry) -> W { #if defined(BOTAN_MP_USE_X86_32_ASM) - asm(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB2_OP, "sbbl")) - : [carry] "=r"(carry) - : [x] "r"(x), [y] "r"(y), "0"(carry) - : "cc", "memory"); + if(std::same_as && !std::is_constant_evaluated()) { + asm(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB2_OP, "sbbl")) + : [carry] "=r"(carry) + : [x] "r"(x), [y] "r"(y), "0"(carry) + : "cc", "memory"); + return carry; + } +#endif -#elif defined(BOTAN_MP_USE_X86_64_ASM) - asm(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB2_OP, "sbbq")) - : [carry] "=r"(carry) - : [x] "r"(x), [y] "r"(y), "0"(carry) - : "cc", "memory"); +#if defined(BOTAN_MP_USE_X86_64_ASM) + if(std::same_as && !std::is_constant_evaluated()) { + asm(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB2_OP, "sbbq")) + : [carry] "=r"(carry) + : [x] "r"(x), [y] "r"(y), "0"(carry) + : "cc", "memory"); + return carry; + } +#endif -#else x[0] = word_sub(x[0], y[0], &carry); x[1] = word_sub(x[1], y[1], &carry); x[2] = word_sub(x[2], y[2], &carry); @@ -351,28 +402,34 @@ inline word word8_sub2(word x[8], const word y[8], word carry) { x[5] = word_sub(x[5], y[5], &carry); x[6] = word_sub(x[6], y[6], &carry); x[7] = word_sub(x[7], y[7], &carry); -#endif - return carry; } /* * Eight Word Block Subtraction, Two Argument */ -inline word word8_sub2_rev(word x[8], const word y[8], word carry) { +template +inline constexpr auto word8_sub2_rev(W x[8], const W y[8], W carry) -> W { #if defined(BOTAN_MP_USE_X86_32_ASM) - asm(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB3_OP, "sbbl")) - : [carry] "=r"(carry) - : [x] "r"(y), [y] "r"(x), [z] "r"(x), "0"(carry) - : "cc", "memory"); + if(std::same_as && !std::is_constant_evaluated()) { + asm(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB3_OP, "sbbl")) + : [carry] "=r"(carry) + : [x] "r"(y), [y] "r"(x), [z] "r"(x), "0"(carry) + : "cc", "memory"); + return carry; + } +#endif -#elif defined(BOTAN_MP_USE_X86_64_ASM) - asm(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB3_OP, "sbbq")) - : [carry] "=r"(carry) - : [x] "r"(y), [y] "r"(x), [z] "r"(x), "0"(carry) - : "cc", "memory"); +#if defined(BOTAN_MP_USE_X86_64_ASM) + if(std::same_as && !std::is_constant_evaluated()) { + asm(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB3_OP, "sbbq")) + : [carry] "=r"(carry) + : [x] "r"(y), [y] "r"(x), [z] "r"(x), "0"(carry) + : "cc", "memory"); + return carry; + } +#endif -#else x[0] = word_sub(y[0], x[0], &carry); x[1] = word_sub(y[1], x[1], &carry); x[2] = word_sub(y[2], x[2], &carry); @@ -381,28 +438,34 @@ inline word word8_sub2_rev(word x[8], const word y[8], word carry) { x[5] = word_sub(y[5], x[5], &carry); x[6] = word_sub(y[6], x[6], &carry); x[7] = word_sub(y[7], x[7], &carry); -#endif - return carry; } /* * Eight Word Block Subtraction, Three Argument */ -inline word word8_sub3(word z[8], const word x[8], const word y[8], word carry) { +template +inline constexpr auto word8_sub3(W z[8], const W x[8], const W y[8], W carry) -> W { #if defined(BOTAN_MP_USE_X86_32_ASM) - asm(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB3_OP, "sbbl")) - : [carry] "=r"(carry) - : [x] "r"(x), [y] "r"(y), [z] "r"(z), "0"(carry) - : "cc", "memory"); + if(std::same_as && !std::is_constant_evaluated()) { + asm(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB3_OP, "sbbl")) + : [carry] "=r"(carry) + : [x] "r"(x), [y] "r"(y), [z] "r"(z), "0"(carry) + : "cc", "memory"); + return carry; + } +#endif -#elif defined(BOTAN_MP_USE_X86_64_ASM) - asm(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB3_OP, "sbbq")) - : [carry] "=r"(carry) - : [x] "r"(x), [y] "r"(y), [z] "r"(z), "0"(carry) - : "cc", "memory"); +#if defined(BOTAN_MP_USE_X86_64_ASM) + if(std::same_as && !std::is_constant_evaluated()) { + asm(ADD_OR_SUBTRACT(DO_8_TIMES(ADDSUB3_OP, "sbbq")) + : [carry] "=r"(carry) + : [x] "r"(x), [y] "r"(y), [z] "r"(z), "0"(carry) + : "cc", "memory"); + return carry; + } +#endif -#else z[0] = word_sub(x[0], y[0], &carry); z[1] = word_sub(x[1], y[1], &carry); z[2] = word_sub(x[2], y[2], &carry); @@ -411,22 +474,34 @@ inline word word8_sub3(word z[8], const word x[8], const word y[8], word carry) z[5] = word_sub(x[5], y[5], &carry); z[6] = word_sub(x[6], y[6], &carry); z[7] = word_sub(x[7], y[7], &carry); -#endif - return carry; } /* * Eight Word Block Linear Multiplication */ -inline word word8_linmul2(word x[8], word y, word carry) { +template +inline constexpr auto word8_linmul2(W x[8], W y, W carry) -> W { #if defined(BOTAN_MP_USE_X86_32_ASM) - asm(DO_8_TIMES(LINMUL_OP, "x") : [carry] "=r"(carry) : [x] "r"(x), [y] "rm"(y), "0"(carry) : "cc", "%eax", "%edx"); + if(std::same_as && !std::is_constant_evaluated()) { + asm(DO_8_TIMES(LINMUL_OP, "x") + : [carry] "=r"(carry) + : [x] "r"(x), [y] "rm"(y), "0"(carry) + : "cc", "%eax", "%edx"); + return carry; + } +#endif -#elif defined(BOTAN_MP_USE_X86_64_ASM) - asm(DO_8_TIMES(LINMUL_OP, "x") : [carry] "=r"(carry) : [x] "r"(x), [y] "rm"(y), "0"(carry) : "cc", "%rax", "%rdx"); +#if defined(BOTAN_MP_USE_X86_64_ASM) + if(std::same_as && !std::is_constant_evaluated()) { + asm(DO_8_TIMES(LINMUL_OP, "x") + : [carry] "=r"(carry) + : [x] "r"(x), [y] "rm"(y), "0"(carry) + : "cc", "%rax", "%rdx"); + return carry; + } +#endif -#else x[0] = word_madd2(x[0], y, &carry); x[1] = word_madd2(x[1], y, &carry); x[2] = word_madd2(x[2], y, &carry); @@ -435,28 +510,34 @@ inline word word8_linmul2(word x[8], word y, word carry) { x[5] = word_madd2(x[5], y, &carry); x[6] = word_madd2(x[6], y, &carry); x[7] = word_madd2(x[7], y, &carry); -#endif - return carry; } /* * Eight Word Block Linear Multiplication */ -inline word word8_linmul3(word z[8], const word x[8], word y, word carry) { +template +inline constexpr auto word8_linmul3(W z[8], const W x[8], W y, W carry) -> W { #if defined(BOTAN_MP_USE_X86_32_ASM) - asm(DO_8_TIMES(LINMUL_OP, "z") - : [carry] "=r"(carry) - : [z] "r"(z), [x] "r"(x), [y] "rm"(y), "0"(carry) - : "cc", "%eax", "%edx"); + if(std::same_as && !std::is_constant_evaluated()) { + asm(DO_8_TIMES(LINMUL_OP, "z") + : [carry] "=r"(carry) + : [z] "r"(z), [x] "r"(x), [y] "rm"(y), "0"(carry) + : "cc", "%eax", "%edx"); + return carry; + } +#endif -#elif defined(BOTAN_MP_USE_X86_64_ASM) - asm(DO_8_TIMES(LINMUL_OP, "z") - : [carry] "=r"(carry) - : [z] "r"(z), [x] "r"(x), [y] "rm"(y), "0"(carry) - : "cc", "%rax", "%rdx"); +#if defined(BOTAN_MP_USE_X86_64_ASM) + if(std::same_as && !std::is_constant_evaluated()) { + asm(DO_8_TIMES(LINMUL_OP, "z") + : [carry] "=r"(carry) + : [z] "r"(z), [x] "r"(x), [y] "rm"(y), "0"(carry) + : "cc", "%rax", "%rdx"); + return carry; + } +#endif -#else z[0] = word_madd2(x[0], y, &carry); z[1] = word_madd2(x[1], y, &carry); z[2] = word_madd2(x[2], y, &carry); @@ -465,28 +546,34 @@ inline word word8_linmul3(word z[8], const word x[8], word y, word carry) { z[5] = word_madd2(x[5], y, &carry); z[6] = word_madd2(x[6], y, &carry); z[7] = word_madd2(x[7], y, &carry); -#endif - return carry; } /* * Eight Word Block Multiply/Add */ -inline word word8_madd3(word z[8], const word x[8], word y, word carry) { +template +inline constexpr auto word8_madd3(W z[8], const W x[8], W y, W carry) -> W { #if defined(BOTAN_MP_USE_X86_32_ASM) - asm(DO_8_TIMES(MULADD_OP, "") - : [carry] "=r"(carry) - : [z] "r"(z), [x] "r"(x), [y] "rm"(y), "0"(carry) - : "cc", "%eax", "%edx"); + if(std::same_as && !std::is_constant_evaluated()) { + asm(DO_8_TIMES(MULADD_OP, "") + : [carry] "=r"(carry) + : [z] "r"(z), [x] "r"(x), [y] "rm"(y), "0"(carry) + : "cc", "%eax", "%edx"); + return carry; + } +#endif -#elif defined(BOTAN_MP_USE_X86_64_ASM) - asm(DO_8_TIMES(MULADD_OP, "") - : [carry] "=r"(carry) - : [z] "r"(z), [x] "r"(x), [y] "rm"(y), "0"(carry) - : "cc", "%rax", "%rdx"); +#if defined(BOTAN_MP_USE_X86_64_ASM) + if(std::same_as && !std::is_constant_evaluated()) { + asm(DO_8_TIMES(MULADD_OP, "") + : [carry] "=r"(carry) + : [z] "r"(z), [x] "r"(x), [y] "rm"(y), "0"(carry) + : "cc", "%rax", "%rdx"); + return carry; + } +#endif -#else z[0] = word_madd3(x[0], y, z[0], &carry); z[1] = word_madd3(x[1], y, z[1], &carry); z[2] = word_madd3(x[2], y, z[2], &carry); @@ -495,8 +582,6 @@ inline word word8_madd3(word z[8], const word x[8], word y, word carry) { z[5] = word_madd3(x[5], y, z[5], &carry); z[6] = word_madd3(x[6], y, z[6], &carry); z[7] = word_madd3(x[7], y, z[7], &carry); -#endif - return carry; } @@ -504,133 +589,156 @@ inline word word8_madd3(word z[8], const word x[8], word y, word carry) { * Multiply-Add Accumulator * (w2,w1,w0) += x * y */ -inline void word3_muladd(word* w2, word* w1, word* w0, word x, word y) { +template +inline constexpr void word3_muladd(W* w2, W* w1, W* w0, W x, W y) { #if defined(BOTAN_MP_USE_X86_32_ASM) - word z0 = 0, z1 = 0; - - asm("mull %[y]" : "=a"(z0), "=d"(z1) : "a"(x), [y] "rm"(y) : "cc"); - - asm(R"( - addl %[z0],%[w0] - adcl %[z1],%[w1] - adcl $0,%[w2] - )" - : [w0] "=r"(*w0), [w1] "=r"(*w1), [w2] "=r"(*w2) - : [z0] "r"(z0), [z1] "r"(z1), "0"(*w0), "1"(*w1), "2"(*w2) - : "cc"); - -#elif defined(BOTAN_MP_USE_X86_64_ASM) - word z0 = 0, z1 = 0; - - asm("mulq %[y]" : "=a"(z0), "=d"(z1) : "a"(x), [y] "rm"(y) : "cc"); + if(std::same_as && !std::is_constant_evaluated()) { + W z0 = 0, z1 = 0; + + asm("mull %[y]" : "=a"(z0), "=d"(z1) : "a"(x), [y] "rm"(y) : "cc"); + + asm(R"( + addl %[z0],%[w0] + adcl %[z1],%[w1] + adcl $0,%[w2] + )" + : [w0] "=r"(*w0), [w1] "=r"(*w1), [w2] "=r"(*w2) + : [z0] "r"(z0), [z1] "r"(z1), "0"(*w0), "1"(*w1), "2"(*w2) + : "cc"); + return; + } +#endif - asm(R"( - addq %[z0],%[w0] - adcq %[z1],%[w1] - adcq $0,%[w2] - )" - : [w0] "=r"(*w0), [w1] "=r"(*w1), [w2] "=r"(*w2) - : [z0] "r"(z0), [z1] "r"(z1), "0"(*w0), "1"(*w1), "2"(*w2) - : "cc"); +#if defined(BOTAN_MP_USE_X86_64_ASM) + if(std::same_as && !std::is_constant_evaluated()) { + W z0 = 0, z1 = 0; + + asm("mulq %[y]" : "=a"(z0), "=d"(z1) : "a"(x), [y] "rm"(y) : "cc"); + + asm(R"( + addq %[z0],%[w0] + adcq %[z1],%[w1] + adcq $0,%[w2] + )" + : [w0] "=r"(*w0), [w1] "=r"(*w1), [w2] "=r"(*w2) + : [z0] "r"(z0), [z1] "r"(z1), "0"(*w0), "1"(*w1), "2"(*w2) + : "cc"); + return; + } +#endif -#else - word carry = *w0; + W carry = *w0; *w0 = word_madd2(x, y, &carry); *w1 += carry; *w2 += (*w1 < carry); -#endif } /* * 3-word addition * (w2,w1,w0) += x */ -inline void word3_add(word* w2, word* w1, word* w0, word x) { +template +inline constexpr void word3_add(W* w2, W* w1, W* w0, W x) { #if defined(BOTAN_MP_USE_X86_32_ASM) - asm(R"( - addl %[x],%[w0] - adcl $0,%[w1] - adcl $0,%[w2] - )" - : [w0] "=r"(*w0), [w1] "=r"(*w1), [w2] "=r"(*w2) - : [x] "r"(x), "0"(*w0), "1"(*w1), "2"(*w2) - : "cc"); + if(std::same_as && !std::is_constant_evaluated()) { + asm(R"( + addl %[x],%[w0] + adcl $0,%[w1] + adcl $0,%[w2] + )" + : [w0] "=r"(*w0), [w1] "=r"(*w1), [w2] "=r"(*w2) + : [x] "r"(x), "0"(*w0), "1"(*w1), "2"(*w2) + : "cc"); + return; + } +#endif -#elif defined(BOTAN_MP_USE_X86_64_ASM) - asm(R"( - addq %[x],%[w0] - adcq $0,%[w1] - adcq $0,%[w2] - )" - : [w0] "=r"(*w0), [w1] "=r"(*w1), [w2] "=r"(*w2) - : [x] "r"(x), "0"(*w0), "1"(*w1), "2"(*w2) - : "cc"); +#if defined(BOTAN_MP_USE_X86_64_ASM) + if(std::same_as && !std::is_constant_evaluated()) { + asm(R"( + addq %[x],%[w0] + adcq $0,%[w1] + adcq $0,%[w2] + )" + : [w0] "=r"(*w0), [w1] "=r"(*w1), [w2] "=r"(*w2) + : [x] "r"(x), "0"(*w0), "1"(*w1), "2"(*w2) + : "cc"); + return; + } +#endif -#else *w0 += x; - word c1 = (*w0 < x); + W c1 = (*w0 < x); *w1 += c1; - word c2 = (*w1 < c1); + W c2 = (*w1 < c1); *w2 += c2; -#endif } /* * Multiply-Add Accumulator * (w2,w1,w0) += 2 * x * y */ -inline void word3_muladd_2(word* w2, word* w1, word* w0, word x, word y) { +template +inline constexpr void word3_muladd_2(W* w2, W* w1, W* w0, W x, W y) { #if defined(BOTAN_MP_USE_X86_32_ASM) - word z0 = 0, z1 = 0; - - asm("mull %[y]" : "=a"(z0), "=d"(z1) : "a"(x), [y] "rm"(y) : "cc"); - - asm(R"( - addl %[z0],%[w0] - adcl %[z1],%[w1] - adcl $0,%[w2] - - addl %[z0],%[w0] - adcl %[z1],%[w1] - adcl $0,%[w2] - )" - : [w0] "=r"(*w0), [w1] "=r"(*w1), [w2] "=r"(*w2) - : [z0] "r"(z0), [z1] "r"(z1), "0"(*w0), "1"(*w1), "2"(*w2) - : "cc"); - -#elif defined(BOTAN_MP_USE_X86_64_ASM) - word z0 = 0, z1 = 0; - - asm("mulq %[y]" : "=a"(z0), "=d"(z1) : "a"(x), [y] "rm"(y) : "cc"); - - asm(R"( - addq %[z0],%[w0] - adcq %[z1],%[w1] - adcq $0,%[w2] + if(std::same_as && !std::is_constant_evaluated()) { + W z0 = 0, z1 = 0; + + asm("mull %[y]" : "=a"(z0), "=d"(z1) : "a"(x), [y] "rm"(y) : "cc"); + + asm(R"( + addl %[z0],%[w0] + adcl %[z1],%[w1] + adcl $0,%[w2] + + addl %[z0],%[w0] + adcl %[z1],%[w1] + adcl $0,%[w2] + )" + : [w0] "=r"(*w0), [w1] "=r"(*w1), [w2] "=r"(*w2) + : [z0] "r"(z0), [z1] "r"(z1), "0"(*w0), "1"(*w1), "2"(*w2) + : "cc"); + return; + } +#endif - addq %[z0],%[w0] - adcq %[z1],%[w1] - adcq $0,%[w2] - )" - : [w0] "=r"(*w0), [w1] "=r"(*w1), [w2] "=r"(*w2) - : [z0] "r"(z0), [z1] "r"(z1), "0"(*w0), "1"(*w1), "2"(*w2) - : "cc"); +#if defined(BOTAN_MP_USE_X86_64_ASM) + if(std::same_as && !std::is_constant_evaluated()) { + W z0 = 0, z1 = 0; + + asm("mulq %[y]" : "=a"(z0), "=d"(z1) : "a"(x), [y] "rm"(y) : "cc"); + + asm(R"( + addq %[z0],%[w0] + adcq %[z1],%[w1] + adcq $0,%[w2] + + addq %[z0],%[w0] + adcq %[z1],%[w1] + adcq $0,%[w2] + )" + : [w0] "=r"(*w0), [w1] "=r"(*w1), [w2] "=r"(*w2) + : [z0] "r"(z0), [z1] "r"(z1), "0"(*w0), "1"(*w1), "2"(*w2) + : "cc"); + return; + } +#endif -#else - word carry = 0; + W carry = 0; x = word_madd2(x, y, &carry); y = carry; - word top = (y >> (BOTAN_MP_WORD_BITS - 1)); + const size_t top_bit_shift = sizeof(W) * 8 - 1; + + W top = (y >> top_bit_shift); y <<= 1; - y |= (x >> (BOTAN_MP_WORD_BITS - 1)); + y |= (x >> top_bit_shift); x <<= 1; carry = 0; *w0 = word_add(*w0, x, &carry); *w1 = word_add(*w1, y, &carry); *w2 = word_add(*w2, top, &carry); -#endif } #if defined(ASM) diff --git a/src/lib/math/mp/mp_core.h b/src/lib/math/mp/mp_core.h index d04495c54b6..46c01ba0304 100644 --- a/src/lib/math/mp/mp_core.h +++ b/src/lib/math/mp/mp_core.h @@ -26,26 +26,28 @@ const word MP_WORD_MAX = ~static_cast(0); * If cond > 0, swaps x[0:size] with y[0:size] * Runs in constant time */ -inline void bigint_cnd_swap(word cnd, word x[], word y[], size_t size) { - const auto mask = CT::Mask::expand(cnd); +template +inline constexpr void bigint_cnd_swap(W cnd, W x[], W y[], size_t size) { + const auto mask = CT::Mask::expand(cnd); for(size_t i = 0; i != size; ++i) { - const word a = x[i]; - const word b = y[i]; + const W a = x[i]; + const W b = y[i]; x[i] = mask.select(b, a); y[i] = mask.select(a, b); } } -inline word bigint_cnd_add(word cnd, word x[], word x_size, const word y[], size_t y_size) { +template +inline constexpr W bigint_cnd_add(W cnd, W x[], size_t x_size, const W y[], size_t y_size) { BOTAN_ASSERT(x_size >= y_size, "Expected sizes"); - const auto mask = CT::Mask::expand(cnd); + const auto mask = CT::Mask::expand(cnd); - word carry = 0; + W carry = 0; const size_t blocks = y_size - (y_size % 8); - word z[8] = {0}; + W z[8] = {0}; for(size_t i = 0; i != blocks; i += 8) { carry = word8_add3(z, x + i, y + i, carry); @@ -58,7 +60,7 @@ inline word bigint_cnd_add(word cnd, word x[], word x_size, const word y[], size } for(size_t i = y_size; i != x_size; ++i) { - z[0] = word_add(x[i], 0, &carry); + z[0] = word_add(x[i], static_cast(0), &carry); x[i] = mask.select(z[0], x[i]); } @@ -69,7 +71,8 @@ inline word bigint_cnd_add(word cnd, word x[], word x_size, const word y[], size * If cond > 0 adds x[0:size] and y[0:size] and returns carry * Runs in constant time */ -inline word bigint_cnd_add(word cnd, word x[], const word y[], size_t size) { +template +inline constexpr W bigint_cnd_add(W cnd, W x[], const W y[], size_t size) { return bigint_cnd_add(cnd, x, size, y, size); } @@ -77,15 +80,16 @@ inline word bigint_cnd_add(word cnd, word x[], const word y[], size_t size) { * If cond > 0 subtracts x[0:size] and y[0:size] and returns borrow * Runs in constant time */ -inline word bigint_cnd_sub(word cnd, word x[], size_t x_size, const word y[], size_t y_size) { +template +inline constexpr auto bigint_cnd_sub(W cnd, W x[], size_t x_size, const W y[], size_t y_size) -> W { BOTAN_ASSERT(x_size >= y_size, "Expected sizes"); - const auto mask = CT::Mask::expand(cnd); + const auto mask = CT::Mask::expand(cnd); - word carry = 0; + W carry = 0; const size_t blocks = y_size - (y_size % 8); - word z[8] = {0}; + W z[8] = {0}; for(size_t i = 0; i != blocks; i += 8) { carry = word8_sub3(z, x + i, y + i, carry); @@ -98,7 +102,7 @@ inline word bigint_cnd_sub(word cnd, word x[], size_t x_size, const word y[], si } for(size_t i = y_size; i != x_size; ++i) { - z[0] = word_sub(x[i], 0, &carry); + z[0] = word_sub(x[i], static_cast(0), &carry); x[i] = mask.select(z[0], x[i]); } @@ -109,7 +113,8 @@ inline word bigint_cnd_sub(word cnd, word x[], size_t x_size, const word y[], si * If cond > 0 adds x[0:size] and y[0:size] and returns carry * Runs in constant time */ -inline word bigint_cnd_sub(word cnd, word x[], const word y[], size_t size) { +template +inline constexpr auto bigint_cnd_sub(W cnd, W x[], const W y[], size_t size) -> W { return bigint_cnd_sub(cnd, x, size, y, size); } @@ -120,14 +125,15 @@ inline word bigint_cnd_sub(word cnd, word x[], const word y[], size_t size) { * * Mask must be either 0 or all 1 bits */ -inline void bigint_cnd_add_or_sub(CT::Mask mask, word x[], const word y[], size_t size) { +template +inline constexpr void bigint_cnd_add_or_sub(CT::Mask mask, W x[], const W y[], size_t size) { const size_t blocks = size - (size % 8); - word carry = 0; - word borrow = 0; + W carry = 0; + W borrow = 0; - word t0[8] = {0}; - word t1[8] = {0}; + W t0[8] = {0}; + W t1[8] = {0}; for(size_t i = 0; i != blocks; i += 8) { carry = word8_add3(t0, x + i, y + i, carry); @@ -139,8 +145,8 @@ inline void bigint_cnd_add_or_sub(CT::Mask mask, word x[], const word y[], } for(size_t i = blocks; i != size; ++i) { - const word a = word_add(x[i], y[i], &carry); - const word s = word_sub(x[i], y[i], &borrow); + const W a = word_add(x[i], y[i], &carry); + const W s = word_sub(x[i], y[i], &borrow); x[i] = mask.select(a, s); } @@ -155,14 +161,15 @@ inline void bigint_cnd_add_or_sub(CT::Mask mask, word x[], const word y[], * * Returns the carry or borrow resp */ -inline word bigint_cnd_addsub(CT::Mask mask, word x[], const word y[], const word z[], size_t size) { +template +inline constexpr auto bigint_cnd_addsub(CT::Mask mask, W x[], const W y[], const W z[], size_t size) -> W { const size_t blocks = size - (size % 8); - word carry = 0; - word borrow = 0; + W carry = 0; + W borrow = 0; - word t0[8] = {0}; - word t1[8] = {0}; + W t0[8] = {0}; + W t1[8] = {0}; for(size_t i = 0; i != blocks; i += 8) { carry = word8_add3(t0, x + i, y + i, carry); @@ -187,12 +194,13 @@ inline word bigint_cnd_addsub(CT::Mask mask, word x[], const word y[], con * If cond > 0 sets x to ~x + 1 * Runs in constant time */ -inline void bigint_cnd_abs(word cnd, word x[], size_t size) { - const auto mask = CT::Mask::expand(cnd); +template +inline constexpr void bigint_cnd_abs(W cnd, W x[], size_t size) { + const auto mask = CT::Mask::expand(cnd); - word carry = mask.if_set_return(1); + W carry = mask.if_set_return(1); for(size_t i = 0; i != size; ++i) { - const word z = word_add(~x[i], 0, &carry); + const W z = word_add(~x[i], static_cast(0), &carry); x[i] = mask.select(z, x[i]); } } @@ -200,8 +208,9 @@ inline void bigint_cnd_abs(word cnd, word x[], size_t size) { /** * Two operand addition with carry out */ -inline word bigint_add2_nc(word x[], size_t x_size, const word y[], size_t y_size) { - word carry = 0; +template +inline constexpr auto bigint_add2_nc(W x[], size_t x_size, const W y[], size_t y_size) -> W { + W carry = 0; BOTAN_ASSERT(x_size >= y_size, "Expected sizes"); @@ -216,7 +225,7 @@ inline word bigint_add2_nc(word x[], size_t x_size, const word y[], size_t y_siz } for(size_t i = y_size; i != x_size; ++i) { - x[i] = word_add(x[i], 0, &carry); + x[i] = word_add(x[i], static_cast(0), &carry); } return carry; @@ -225,12 +234,13 @@ inline word bigint_add2_nc(word x[], size_t x_size, const word y[], size_t y_siz /** * Three operand addition with carry out */ -inline word bigint_add3_nc(word z[], const word x[], size_t x_size, const word y[], size_t y_size) { +template +inline constexpr auto bigint_add3_nc(W z[], const W x[], size_t x_size, const W y[], size_t y_size) -> W { if(x_size < y_size) { return bigint_add3_nc(z, y, y_size, x, x_size); } - word carry = 0; + W carry = 0; const size_t blocks = y_size - (y_size % 8); @@ -243,7 +253,7 @@ inline word bigint_add3_nc(word z[], const word x[], size_t x_size, const word y } for(size_t i = y_size; i != x_size; ++i) { - z[i] = word_add(x[i], 0, &carry); + z[i] = word_add(x[i], static_cast(0), &carry); } return carry; @@ -256,22 +266,25 @@ inline word bigint_add3_nc(word z[], const word x[], size_t x_size, const word y * @param y the second operand * @param y_size size of y (must be <= x_size) */ -inline void bigint_add2(word x[], size_t x_size, const word y[], size_t y_size) { +template +inline constexpr void bigint_add2(W x[], size_t x_size, const W y[], size_t y_size) { x[x_size] += bigint_add2_nc(x, x_size, y, y_size); } /** * Three operand addition */ -inline void bigint_add3(word z[], const word x[], size_t x_size, const word y[], size_t y_size) { +template +inline constexpr void bigint_add3(W z[], const W x[], size_t x_size, const W y[], size_t y_size) { z[x_size > y_size ? x_size : y_size] += bigint_add3_nc(z, x, x_size, y, y_size); } /** * Two operand subtraction */ -inline word bigint_sub2(word x[], size_t x_size, const word y[], size_t y_size) { - word borrow = 0; +template +inline constexpr auto bigint_sub2(W x[], size_t x_size, const W y[], size_t y_size) -> W { + W borrow = 0; BOTAN_ASSERT(x_size >= y_size, "Expected sizes"); @@ -286,7 +299,7 @@ inline word bigint_sub2(word x[], size_t x_size, const word y[], size_t y_size) } for(size_t i = y_size; i != x_size; ++i) { - x[i] = word_sub(x[i], 0, &borrow); + x[i] = word_sub(x[i], static_cast(0), &borrow); } return borrow; @@ -295,8 +308,9 @@ inline word bigint_sub2(word x[], size_t x_size, const word y[], size_t y_size) /** * Two operand subtraction, x = y - x; assumes y >= x */ -inline void bigint_sub2_rev(word x[], const word y[], size_t y_size) { - word borrow = 0; +template +inline constexpr void bigint_sub2_rev(W x[], const W y[], size_t y_size) { + W borrow = 0; const size_t blocks = y_size - (y_size % 8); @@ -318,8 +332,9 @@ inline void bigint_sub2_rev(word x[], const word y[], size_t y_size) { * * Writes to z[0:x_size] and returns borrow */ -inline word bigint_sub3(word z[], const word x[], size_t x_size, const word y[], size_t y_size) { - word borrow = 0; +template +inline constexpr auto bigint_sub3(W z[], const W x[], size_t x_size, const W y[], size_t y_size) -> W { + W borrow = 0; BOTAN_ASSERT(x_size >= y_size, "Expected sizes"); @@ -334,7 +349,7 @@ inline word bigint_sub3(word z[], const word x[], size_t x_size, const word y[], } for(size_t i = y_size; i != x_size; ++i) { - z[i] = word_sub(x[i], 0, &borrow); + z[i] = word_sub(x[i], static_cast(0), &borrow); } return borrow; @@ -352,14 +367,15 @@ inline word bigint_sub3(word z[], const word x[], size_t x_size, const word y[], * @param N length of x and y * @param ws array of at least 2*N words */ -inline CT::Mask bigint_sub_abs(word z[], const word x[], const word y[], size_t N, word ws[]) { +template +inline constexpr auto bigint_sub_abs(W z[], const W x[], const W y[], size_t N, W ws[]) -> CT::Mask { // Subtract in both direction then conditional copy out the result - word* ws0 = ws; - word* ws1 = ws + N; + W* ws0 = ws; + W* ws1 = ws + N; - word borrow0 = 0; - word borrow1 = 0; + W borrow0 = 0; + W borrow1 = 0; const size_t blocks = N - (N % 8); @@ -379,22 +395,24 @@ inline CT::Mask bigint_sub_abs(word z[], const word x[], const word y[], s /* * Shift Operations */ -inline void bigint_shl1(word x[], size_t x_size, size_t x_words, size_t word_shift, size_t bit_shift) { +template +inline constexpr void bigint_shl1(W x[], size_t x_size, size_t x_words, size_t word_shift, size_t bit_shift) { copy_mem(x + word_shift, x, x_words); clear_mem(x, word_shift); - const auto carry_mask = CT::Mask::expand(bit_shift); - const word carry_shift = carry_mask.if_set_return(BOTAN_MP_WORD_BITS - bit_shift); + const auto carry_mask = CT::Mask::expand(bit_shift); + const W carry_shift = carry_mask.if_set_return(sizeof(W) * 8 - bit_shift); - word carry = 0; + W carry = 0; for(size_t i = word_shift; i != x_size; ++i) { - const word w = x[i]; + const W w = x[i]; x[i] = (w << bit_shift) | carry; carry = carry_mask.if_set_return(w >> carry_shift); } } -inline void bigint_shr1(word x[], size_t x_size, size_t word_shift, size_t bit_shift) { +template +inline constexpr void bigint_shr1(W x[], size_t x_size, size_t word_shift, size_t bit_shift) { const size_t top = x_size >= word_shift ? (x_size - word_shift) : 0; if(top > 0) { @@ -402,45 +420,47 @@ inline void bigint_shr1(word x[], size_t x_size, size_t word_shift, size_t bit_s } clear_mem(x + top, std::min(word_shift, x_size)); - const auto carry_mask = CT::Mask::expand(bit_shift); - const word carry_shift = carry_mask.if_set_return(BOTAN_MP_WORD_BITS - bit_shift); + const auto carry_mask = CT::Mask::expand(bit_shift); + const W carry_shift = carry_mask.if_set_return(sizeof(W) * 8 - bit_shift); - word carry = 0; + W carry = 0; for(size_t i = 0; i != top; ++i) { - const word w = x[top - i - 1]; + const W w = x[top - i - 1]; x[top - i - 1] = (w >> bit_shift) | carry; carry = carry_mask.if_set_return(w << carry_shift); } } -inline void bigint_shl2(word y[], const word x[], size_t x_size, size_t word_shift, size_t bit_shift) { +template +inline constexpr void bigint_shl2(W y[], const W x[], size_t x_size, size_t word_shift, size_t bit_shift) { copy_mem(y + word_shift, x, x_size); - const auto carry_mask = CT::Mask::expand(bit_shift); - const word carry_shift = carry_mask.if_set_return(BOTAN_MP_WORD_BITS - bit_shift); + const auto carry_mask = CT::Mask::expand(bit_shift); + const W carry_shift = carry_mask.if_set_return(sizeof(W) * 8 - bit_shift); - word carry = 0; + W carry = 0; for(size_t i = word_shift; i != x_size + word_shift + 1; ++i) { - const word w = y[i]; + const W w = y[i]; y[i] = (w << bit_shift) | carry; carry = carry_mask.if_set_return(w >> carry_shift); } } -inline void bigint_shr2(word y[], const word x[], size_t x_size, size_t word_shift, size_t bit_shift) { +template +inline constexpr void bigint_shr2(W y[], const W x[], size_t x_size, size_t word_shift, size_t bit_shift) { const size_t new_size = x_size < word_shift ? 0 : (x_size - word_shift); if(new_size > 0) { copy_mem(y, x + word_shift, new_size); } - const auto carry_mask = CT::Mask::expand(bit_shift); - const word carry_shift = carry_mask.if_set_return(BOTAN_MP_WORD_BITS - bit_shift); + const auto carry_mask = CT::Mask::expand(bit_shift); + const W carry_shift = carry_mask.if_set_return(sizeof(W) * 8 - bit_shift); - word carry = 0; + W carry = 0; for(size_t i = new_size; i > 0; --i) { - word w = y[i - 1]; + W w = y[i - 1]; y[i - 1] = (w >> bit_shift) | carry; carry = carry_mask.if_set_return(w << carry_shift); } @@ -449,10 +469,11 @@ inline void bigint_shr2(word y[], const word x[], size_t x_size, size_t word_shi /* * Linear Multiply - returns the carry */ -[[nodiscard]] inline word bigint_linmul2(word x[], size_t x_size, word y) { +template +[[nodiscard]] inline constexpr auto bigint_linmul2(W x[], size_t x_size, W y) -> W { const size_t blocks = x_size - (x_size % 8); - word carry = 0; + W carry = 0; for(size_t i = 0; i != blocks; i += 8) { carry = word8_linmul2(x + i, y, carry); @@ -465,10 +486,11 @@ inline void bigint_shr2(word y[], const word x[], size_t x_size, size_t word_shi return carry; } -inline void bigint_linmul3(word z[], const word x[], size_t x_size, word y) { +template +inline constexpr void bigint_linmul3(W z[], const W x[], size_t x_size, W y) { const size_t blocks = x_size - (x_size % 8); - word carry = 0; + W carry = 0; for(size_t i = 0; i != blocks; i += 8) { carry = word8_linmul3(z + i, x + i, y, carry); @@ -487,40 +509,41 @@ inline void bigint_linmul3(word z[], const word x[], size_t x_size, word y) { * Return 0 if x == y * Return 1 if x > y */ -inline int32_t bigint_cmp(const word x[], size_t x_size, const word y[], size_t y_size) { - static_assert(sizeof(word) >= sizeof(uint32_t), "Size assumption"); +template +inline constexpr int32_t bigint_cmp(const W x[], size_t x_size, const W y[], size_t y_size) { + static_assert(sizeof(W) >= sizeof(uint32_t), "Size assumption"); - const word LT = static_cast(-1); - const word EQ = 0; - const word GT = 1; + const W LT = static_cast(-1); + const W EQ = 0; + const W GT = 1; const size_t common_elems = std::min(x_size, y_size); - word result = EQ; // until found otherwise + W result = EQ; // until found otherwise for(size_t i = 0; i != common_elems; i++) { - const auto is_eq = CT::Mask::is_equal(x[i], y[i]); - const auto is_lt = CT::Mask::is_lt(x[i], y[i]); + const auto is_eq = CT::Mask::is_equal(x[i], y[i]); + const auto is_lt = CT::Mask::is_lt(x[i], y[i]); result = is_eq.select(result, is_lt.select(LT, GT)); } if(x_size < y_size) { - word mask = 0; + W mask = 0; for(size_t i = x_size; i != y_size; i++) { mask |= y[i]; } // If any bits were set in high part of y, then x < y - result = CT::Mask::is_zero(mask).select(result, LT); + result = CT::Mask::is_zero(mask).select(result, LT); } else if(y_size < x_size) { - word mask = 0; + W mask = 0; for(size_t i = y_size; i != x_size; i++) { mask |= x[i]; } // If any bits were set in high part of x, then x > y - result = CT::Mask::is_zero(mask).select(result, GT); + result = CT::Mask::is_zero(mask).select(result, GT); } CT::unpoison(result); @@ -533,42 +556,44 @@ inline int32_t bigint_cmp(const word x[], size_t x_size, const word y[], size_t * Return ~0 if x[0:x_size] < y[0:y_size] or 0 otherwise * If lt_or_equal is true, returns ~0 also for x == y */ -inline CT::Mask bigint_ct_is_lt( - const word x[], size_t x_size, const word y[], size_t y_size, bool lt_or_equal = false) { +template +inline constexpr auto bigint_ct_is_lt(const W x[], size_t x_size, const W y[], size_t y_size, bool lt_or_equal = false) + -> CT::Mask { const size_t common_elems = std::min(x_size, y_size); - auto is_lt = CT::Mask::expand(lt_or_equal); + auto is_lt = CT::Mask::expand(lt_or_equal); for(size_t i = 0; i != common_elems; i++) { - const auto eq = CT::Mask::is_equal(x[i], y[i]); - const auto lt = CT::Mask::is_lt(x[i], y[i]); + const auto eq = CT::Mask::is_equal(x[i], y[i]); + const auto lt = CT::Mask::is_lt(x[i], y[i]); is_lt = eq.select_mask(is_lt, lt); } if(x_size < y_size) { - word mask = 0; + W mask = 0; for(size_t i = x_size; i != y_size; i++) { mask |= y[i]; } // If any bits were set in high part of y, then is_lt should be forced true - is_lt |= CT::Mask::expand(mask); + is_lt |= CT::Mask::expand(mask); } else if(y_size < x_size) { - word mask = 0; + W mask = 0; for(size_t i = y_size; i != x_size; i++) { mask |= x[i]; } // If any bits were set in high part of x, then is_lt should be false - is_lt &= CT::Mask::is_zero(mask); + is_lt &= CT::Mask::is_zero(mask); } return is_lt; } -inline CT::Mask bigint_ct_is_eq(const word x[], size_t x_size, const word y[], size_t y_size) { +template +inline constexpr auto bigint_ct_is_eq(const W x[], size_t x_size, const W y[], size_t y_size) -> CT::Mask { const size_t common_elems = std::min(x_size, y_size); - word diff = 0; + W diff = 0; for(size_t i = 0; i != common_elems; i++) { diff |= (x[i] ^ y[i]); @@ -585,7 +610,7 @@ inline CT::Mask bigint_ct_is_eq(const word x[], size_t x_size, const word } } - return CT::Mask::is_zero(diff); + return CT::Mask::is_zero(diff); } /** @@ -601,7 +626,8 @@ inline CT::Mask bigint_ct_is_eq(const word x[], size_t x_size, const word * @param y input param * @param y_size length of y */ -inline int32_t bigint_sub_abs(word z[], const word x[], size_t x_size, const word y[], size_t y_size) { +template +inline constexpr int32_t bigint_sub_abs(W z[], const W x[], size_t x_size, const W y[], size_t y_size) { const int32_t relative_size = bigint_cmp(x, x_size, y, y_size); // Swap if relative_size == -1 @@ -630,83 +656,93 @@ inline int32_t bigint_sub_abs(word z[], const word x[], size_t x_size, const wor * @param mod_sw size of t, s, and mod * @param ws workspace of size mod_sw */ -inline void bigint_mod_sub(word t[], const word s[], const word mod[], size_t mod_sw, word ws[]) { +template +inline constexpr void bigint_mod_sub(W t[], const W s[], const W mod[], size_t mod_sw, W ws[]) { // is t < s or not? const auto is_lt = bigint_ct_is_lt(t, mod_sw, s, mod_sw); // ws = p - s - const word borrow = bigint_sub3(ws, mod, mod_sw, s, mod_sw); + const W borrow = bigint_sub3(ws, mod, mod_sw, s, mod_sw); // Compute either (t - s) or (t + (p - s)) depending on mask - const word carry = bigint_cnd_addsub(is_lt, t, ws, s, mod_sw); + const W carry = bigint_cnd_addsub(is_lt, t, ws, s, mod_sw); + + if(!std::is_constant_evaluated()) { + BOTAN_DEBUG_ASSERT(borrow == 0 && carry == 0); + } - BOTAN_DEBUG_ASSERT(borrow == 0 && carry == 0); BOTAN_UNUSED(carry, borrow); } -template -inline void bigint_mod_sub_n(word t[], const word s[], const word mod[], word ws[]) { +template +inline constexpr void bigint_mod_sub_n(W t[], const W s[], const W mod[], W ws[]) { // is t < s or not? const auto is_lt = bigint_ct_is_lt(t, N, s, N); // ws = p - s - const word borrow = bigint_sub3(ws, mod, N, s, N); + const W borrow = bigint_sub3(ws, mod, N, s, N); // Compute either (t - s) or (t + (p - s)) depending on mask - const word carry = bigint_cnd_addsub(is_lt, t, ws, s, N); + const W carry = bigint_cnd_addsub(is_lt, t, ws, s, N); + + if(!std::is_constant_evaluated()) { + BOTAN_DEBUG_ASSERT(borrow == 0 && carry == 0); + } - BOTAN_DEBUG_ASSERT(borrow == 0 && carry == 0); BOTAN_UNUSED(carry, borrow); } /** * Compute ((n1< +inline constexpr auto bigint_divop_vartime(W n1, W n0, W d) -> W { if(d == 0) { throw Invalid_Argument("bigint_divop_vartime divide by zero"); } -#if defined(BOTAN_HAS_NATIVE_DWORD) - return static_cast(((static_cast(n1) << BOTAN_MP_WORD_BITS) | n0) / d); -#else + constexpr const size_t W_bits = sizeof(W) * 8; - word high = n1 % d; - word quotient = 0; + if constexpr(DwordType::is_native) { + typename DwordType::type n = n1; + n <<= W_bits; + n |= n0; + return static_cast(n / d); + } + else { + W high = n1 % d; + W quotient = 0; - for(size_t i = 0; i != BOTAN_MP_WORD_BITS; ++i) { - const word high_top_bit = high >> (BOTAN_MP_WORD_BITS - 1); + for(size_t i = 0; i != W_bits; ++i) { + const W high_top_bit = high >> (W_bits - 1); - high <<= 1; - high |= (n0 >> (BOTAN_MP_WORD_BITS - 1 - i)) & 1; - quotient <<= 1; + high <<= 1; + high |= (n0 >> (W_bits - 1 - i)) & 1; + quotient <<= 1; - if(high_top_bit || high >= d) { - high -= d; - quotient |= 1; + if(high_top_bit || high >= d) { + high -= d; + quotient |= 1; + } } - } - return quotient; -#endif + return quotient; + } } /** * Compute ((n1< +inline constexpr auto bigint_modop_vartime(W n1, W n0, W d) -> W { if(d == 0) { throw Invalid_Argument("bigint_modop_vartime divide by zero"); } -#if defined(BOTAN_HAS_NATIVE_DWORD) - return ((static_cast(n1) << BOTAN_MP_WORD_BITS) | n0) % d; -#else - word z = bigint_divop_vartime(n1, n0, d); - word dummy = 0; - z = word_madd2(z, d, &dummy); + W z = bigint_divop_vartime(n1, n0, d); + W carry = 0; + z = word_madd2(z, d, &carry); return (n0 - z); -#endif } /* diff --git a/src/lib/math/numbertheory/reducer.cpp b/src/lib/math/numbertheory/reducer.cpp index 9a359f19e70..040cc9a55bc 100644 --- a/src/lib/math/numbertheory/reducer.cpp +++ b/src/lib/math/numbertheory/reducer.cpp @@ -61,7 +61,7 @@ void cnd_rev_sub(bool cnd, BigInt& x, const word y[], size_t y_sw, secure_vector const int32_t relative_size = bigint_sub_abs(ws.data(), x.data(), x_sw, y, y_sw); x.cond_flip_sign((relative_size > 0) && cnd); - bigint_cnd_swap(cnd, x.mutable_data(), ws.data(), max_words); + bigint_cnd_swap(static_cast(cnd), x.mutable_data(), ws.data(), max_words); } } // namespace diff --git a/src/lib/utils/ct_utils.h b/src/lib/utils/ct_utils.h index e6d864d8e44..dbf6535d306 100644 --- a/src/lib/utils/ct_utils.h +++ b/src/lib/utils/ct_utils.h @@ -45,28 +45,34 @@ namespace Botan::CT { template inline void poison(const T* p, size_t n) { #if defined(BOTAN_HAS_VALGRIND) - VALGRIND_MAKE_MEM_UNDEFINED(p, n * sizeof(T)); -#else - BOTAN_UNUSED(p, n); + if(!std::is_constant_evaluated()) { + VALGRIND_MAKE_MEM_UNDEFINED(p, n * sizeof(T)); + } #endif + + BOTAN_UNUSED(p, n); } template inline void unpoison(const T* p, size_t n) { #if defined(BOTAN_HAS_VALGRIND) - VALGRIND_MAKE_MEM_DEFINED(p, n * sizeof(T)); -#else - BOTAN_UNUSED(p, n); + if(!std::is_constant_evaluated()) { + VALGRIND_MAKE_MEM_DEFINED(p, n * sizeof(T)); + } #endif + + BOTAN_UNUSED(p, n); } template inline void unpoison(T& p) { #if defined(BOTAN_HAS_VALGRIND) - VALGRIND_MAKE_MEM_DEFINED(&p, sizeof(T)); -#else - BOTAN_UNUSED(p); + if(!std::is_constant_evaluated()) { + VALGRIND_MAKE_MEM_DEFINED(&p, sizeof(T)); + } #endif + + BOTAN_UNUSED(p); } /** @@ -89,30 +95,30 @@ class Mask final { * Derive a Mask from a Mask of a larger type */ template - Mask(Mask o) : m_mask(static_cast(o.value())) { + constexpr Mask(Mask o) : m_mask(static_cast(o.value())) { static_assert(sizeof(U) > sizeof(T), "sizes ok"); } /** * Return a Mask with all bits set */ - static Mask set() { return Mask(static_cast(~0)); } + static constexpr Mask set() { return Mask(static_cast(~0)); } /** * Return a Mask with all bits cleared */ - static Mask cleared() { return Mask(0); } + static constexpr Mask cleared() { return Mask(0); } /** * Return a Mask which is set if v is != 0 */ - static Mask expand(T v) { return ~Mask::is_zero(v); } + static constexpr Mask expand(T v) { return ~Mask::is_zero(v); } /** * Return a Mask which is set if m is set */ template - static Mask expand(Mask m) { + static constexpr Mask expand(Mask m) { static_assert(sizeof(U) < sizeof(T), "sizes ok"); return ~Mask::is_zero(m.value()); } @@ -120,34 +126,34 @@ class Mask final { /** * Return a Mask which is set if v is == 0 or cleared otherwise */ - static Mask is_zero(T x) { return Mask(ct_is_zero(x)); } + static constexpr Mask is_zero(T x) { return Mask(ct_is_zero(x)); } /** * Return a Mask which is set if x == y */ - static Mask is_equal(T x, T y) { return Mask::is_zero(static_cast(x ^ y)); } + static constexpr Mask is_equal(T x, T y) { return Mask::is_zero(static_cast(x ^ y)); } /** * Return a Mask which is set if x < y */ - static Mask is_lt(T x, T y) { return Mask(expand_top_bit(x ^ ((x ^ y) | ((x - y) ^ x)))); } + static constexpr Mask is_lt(T x, T y) { return Mask(expand_top_bit(x ^ ((x ^ y) | ((x - y) ^ x)))); } /** * Return a Mask which is set if x > y */ - static Mask is_gt(T x, T y) { return Mask::is_lt(y, x); } + static constexpr Mask is_gt(T x, T y) { return Mask::is_lt(y, x); } /** * Return a Mask which is set if x <= y */ - static Mask is_lte(T x, T y) { return ~Mask::is_gt(x, y); } + static constexpr Mask is_lte(T x, T y) { return ~Mask::is_gt(x, y); } /** * Return a Mask which is set if x >= y */ - static Mask is_gte(T x, T y) { return ~Mask::is_lt(x, y); } + static constexpr Mask is_gte(T x, T y) { return ~Mask::is_lt(x, y); } - static Mask is_within_range(T v, T l, T u) { + static constexpr Mask is_within_range(T v, T l, T u) { //return Mask::is_gte(v, l) & Mask::is_lte(v, u); const T v_lt_l = v ^ ((v ^ l) | ((v - l) ^ v)); @@ -156,7 +162,7 @@ class Mask final { return ~Mask(expand_top_bit(either)); } - static Mask is_any_of(T v, std::initializer_list accepted) { + static constexpr Mask is_any_of(T v, std::initializer_list accepted) { T accept = 0; for(auto a : accepted) { @@ -210,24 +216,24 @@ class Mask final { /** * Negate this mask */ - Mask operator~() const { return Mask(~value()); } + constexpr Mask operator~() const { return Mask(~value()); } /** * Return x if the mask is set, or otherwise zero */ - T if_set_return(T x) const { return m_mask & x; } + constexpr T if_set_return(T x) const { return m_mask & x; } /** * Return x if the mask is cleared, or otherwise zero */ - T if_not_set_return(T x) const { return ~m_mask & x; } + constexpr T if_not_set_return(T x) const { return ~m_mask & x; } /** * If this mask is set, return x, otherwise return y */ - T select(T x, T y) const { return choose(value(), x, y); } + constexpr T select(T x, T y) const { return choose(value(), x, y); } - T select_and_unpoison(T x, T y) const { + constexpr T select_and_unpoison(T x, T y) const { T r = this->select(x, y); CT::unpoison(r); return r; @@ -260,7 +266,7 @@ class Mask final { /** * Return the value of the mask, unpoisoned */ - T unpoisoned_value() const { + constexpr T unpoisoned_value() const { T r = value(); CT::unpoison(r); return r; @@ -269,15 +275,15 @@ class Mask final { /** * Return true iff this mask is set */ - bool as_bool() const { return unpoisoned_value() != 0; } + constexpr bool as_bool() const { return unpoisoned_value() != 0; } /** * Return the underlying value of the mask */ - T value() const { return m_mask; } + constexpr T value() const { return m_mask; } private: - Mask(T m) : m_mask(m) {} + constexpr Mask(T m) : m_mask(m) {} T m_mask; }; diff --git a/src/lib/utils/mul128.h b/src/lib/utils/mul128.h index 9f2e2832af2..fb91f548fff 100644 --- a/src/lib/utils/mul128.h +++ b/src/lib/utils/mul128.h @@ -17,13 +17,6 @@ namespace Botan { -#if defined(__SIZEOF_INT128__) && defined(BOTAN_TARGET_CPU_HAS_NATIVE_64BIT) - #define BOTAN_TARGET_HAS_NATIVE_UINT128 - -// GCC complains if this isn't marked with __extension__ -__extension__ typedef unsigned __int128 uint128_t; -#endif - /** * Perform a 64x64->128 bit multiplication */ diff --git a/src/lib/utils/types.h b/src/lib/utils/types.h index 80baeba0e8d..c9eddb7a259 100644 --- a/src/lib/utils/types.h +++ b/src/lib/utils/types.h @@ -106,6 +106,13 @@ typedef uint64_t word; #error BOTAN_MP_WORD_BITS must be 32 or 64 #endif +#if defined(__SIZEOF_INT128__) && defined(BOTAN_TARGET_CPU_HAS_NATIVE_64BIT) + #define BOTAN_TARGET_HAS_NATIVE_UINT128 + +// GCC complains if this isn't marked with __extension__ +__extension__ typedef unsigned __int128 uint128_t; +#endif + /* * Should this assert fail on your system please contact the developers * for assistance in porting. diff --git a/src/tests/test_mp.cpp b/src/tests/test_mp.cpp index e6319b76e91..ff07e0bb763 100644 --- a/src/tests/test_mp.cpp +++ b/src/tests/test_mp.cpp @@ -36,12 +36,12 @@ class MP_Unit_Tests final : public Test { const Botan::word max = Botan::MP_WORD_MAX; Botan::word a = 2; - Botan::word c = Botan::bigint_cnd_add(0, &a, &max, 1); + Botan::word c = Botan::bigint_cnd_add(0, &a, &max, 1); result.test_int_eq(a, 2, "No op"); result.test_int_eq(c, 0, "No op"); - c = Botan::bigint_cnd_add(1, &a, &max, 1); + c = Botan::bigint_cnd_add(1, &a, &max, 1); result.test_int_eq(a, 1, "Add"); result.test_int_eq(c, 1, "Carry"); @@ -56,12 +56,12 @@ class MP_Unit_Tests final : public Test { Botan::word a = 2; Botan::word b = 3; - Botan::word c = Botan::bigint_cnd_sub(0, &a, &b, 1); + Botan::word c = Botan::bigint_cnd_sub(0, &a, &b, 1); result.test_int_eq(a, 2, "No op"); result.test_int_eq(c, 0, "No op"); - c = Botan::bigint_cnd_sub(1, &a, &b, 1); + c = Botan::bigint_cnd_sub(1, &a, &b, 1); result.test_int_eq(a, Botan::MP_WORD_MAX, "Sub"); result.test_int_eq(c, 1, "Borrow"); @@ -73,24 +73,24 @@ class MP_Unit_Tests final : public Test { Result result("bigint_cnd_abs"); Botan::word x1 = Botan::MP_WORD_MAX; - Botan::bigint_cnd_abs(1, &x1, 1); + Botan::bigint_cnd_abs(1, &x1, 1); result.test_int_eq(x1, 1, "Abs"); x1 = 0; - Botan::bigint_cnd_abs(1, &x1, 1); + Botan::bigint_cnd_abs(1, &x1, 1); result.test_int_eq(x1, 0, "Abs"); x1 = 1; - Botan::bigint_cnd_abs(1, &x1, 1); + Botan::bigint_cnd_abs(1, &x1, 1); result.test_int_eq(x1, Botan::MP_WORD_MAX, "Abs"); x1 = 1; - Botan::bigint_cnd_abs(0, &x1, 1); + Botan::bigint_cnd_abs(0, &x1, 1); result.test_int_eq(x1, 1, "No change"); Botan::word x2[2] = {Botan::MP_WORD_MAX, Botan::MP_WORD_MAX}; - Botan::bigint_cnd_abs(1, x2, 2); + Botan::bigint_cnd_abs(1, x2, 2); result.test_int_eq(x2[0], 1, "Abs"); result.test_int_eq(x2[1], 0, "Abs"); @@ -101,21 +101,21 @@ class MP_Unit_Tests final : public Test { Result result("bigint_cnd_swap"); // null with zero length is ok - Botan::bigint_cnd_swap(0, nullptr, nullptr, 0); - Botan::bigint_cnd_swap(1, nullptr, nullptr, 0); + Botan::bigint_cnd_swap(0, nullptr, nullptr, 0); + Botan::bigint_cnd_swap(1, nullptr, nullptr, 0); Botan::word x1 = 5, y1 = 9; - Botan::bigint_cnd_swap(0, &x1, &y1, 1); + Botan::bigint_cnd_swap(0, &x1, &y1, 1); result.test_int_eq(x1, 5, "No swap"); - Botan::bigint_cnd_swap(1, &x1, &y1, 1); + Botan::bigint_cnd_swap(1, &x1, &y1, 1); result.test_int_eq(x1, 9, "Swap"); Botan::word x5[5] = {0, 1, 2, 3, 4}; Botan::word y5[5] = {3, 2, 1, 0, 9}; // Should only modify first four - Botan::bigint_cnd_swap(1, x5, y5, 4); + Botan::bigint_cnd_swap(1, x5, y5, 4); for(size_t i = 0; i != 4; ++i) { result.test_int_eq(x5[i], 3 - i, "Swap x5");