Skip to content

Commit

Permalink
CrypoAlg-556: Implement ARMv8 Perl Assembly function: ecp_nistz256_se…
Browse files Browse the repository at this point in the history 
…lect_w5

This function selects an entry from a table of points in constant-time
It is used in BoringSSL by the x86_64 optimized C code (nistz256).
It did not exist in OpenSSL's ARMv8 assembly that supports nistz256.
  • Loading branch information
@nebeid
nebeid committed Oct 1, 2020
1 parent 8c50bd9 commit ae08fa6
Showing 1 changed file with 345 additions and 4 deletions.
349 changes: 345 additions & 4 deletions patches/0013-start-arm-optimized-nistz256
View file
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ Index: aws-lc/third_party/boringssl/crypto/fipsmodule/ec/p256.c
#include <assert.h>
#include <string.h>

@@ -737,4 +743,276 @@ DEFINE_METHOD_FUNCTION(EC_METHOD, EC_GFp
@@ -737,4 +743,525 @@ DEFINE_METHOD_FUNCTION(EC_METHOD, EC_GFp
out->cmp_x_coordinate = ec_GFp_nistp256_cmp_x_coordinate;
}

Expand Down Expand Up @@ -213,6 +213,255 @@ Index: aws-lc/third_party/boringssl/crypto/fipsmodule/ec/p256.c
+ OPENSSL_memcpy(r->Z.words, a.Z, P256_LIMBS * sizeof(BN_ULONG));
+}
+
+typedef P256_POINT_AFFINE PRECOMP256_ROW[64];
+
+// One converted into the Montgomery domain
+static const BN_ULONG ONE[P256_LIMBS] = {
+ TOBN(0x00000000, 0x00000001), TOBN(0xffffffff, 0x00000000),
+ TOBN(0xffffffff, 0xffffffff), TOBN(0x00000000, 0xfffffffe),
+};
+
+// Precomputed tables for the default generator
+#include "p256-x86_64-table.h"
+
+// Recode window to a signed digit, see |ec_GFp_nistp_recode_scalar_bits| in
+// util.c for details
+static crypto_word_t booth_recode_w5(crypto_word_t in) {
+ crypto_word_t s, d;
+
+ s = ~((in >> 5) - 1);
+ d = (1 << 6) - in - 1;
+ d = (d & s) | (in & ~s);
+ d = (d >> 1) + (d & 1);
+
+ return (d << 1) + (s & 1);
+}
+
+static crypto_word_t booth_recode_w7(crypto_word_t in) {
+ crypto_word_t s, d;
+
+ s = ~((in >> 7) - 1);
+ d = (1 << 8) - in - 1;
+ d = (d & s) | (in & ~s);
+ d = (d >> 1) + (d & 1);
+
+ return (d << 1) + (s & 1);
+}
+
+// copy_conditional copies |src| to |dst| if |move| is one and leaves it as-is
+// if |move| is zero.
+//
+// WARNING: this breaks the usual convention of constant-time functions
+// returning masks.
+static void copy_conditional(BN_ULONG dst[P256_LIMBS],
+ const BN_ULONG src[P256_LIMBS], BN_ULONG move) {
+ BN_ULONG mask1 = ((BN_ULONG)0) - move;
+ BN_ULONG mask2 = ~mask1;
+
+ dst[0] = (src[0] & mask1) ^ (dst[0] & mask2);
+ dst[1] = (src[1] & mask1) ^ (dst[1] & mask2);
+ dst[2] = (src[2] & mask1) ^ (dst[2] & mask2);
+ dst[3] = (src[3] & mask1) ^ (dst[3] & mask2);
+ if (P256_LIMBS == 8) {
+ dst[4] = (src[4] & mask1) ^ (dst[4] & mask2);
+ dst[5] = (src[5] & mask1) ^ (dst[5] & mask2);
+ dst[6] = (src[6] & mask1) ^ (dst[6] & mask2);
+ dst[7] = (src[7] & mask1) ^ (dst[7] & mask2);
+ }
+}
+
+// r = p * p_scalar
+static void ecp_nistz256_windowed_mul(const EC_GROUP *group, P256_POINT *r,
+ const EC_RAW_POINT *p,
+ const EC_SCALAR *p_scalar) {
+ assert(p != NULL);
+ assert(p_scalar != NULL);
+ assert(group->field.width == P256_LIMBS);
+
+ static const size_t kWindowSize = 5;
+ static const crypto_word_t kMask = (1 << (5 /* kWindowSize */ + 1)) - 1;
+
+ // A |P256_POINT| is (3 * 32) = 96 bytes, and the 64-byte alignment should
+ // add no more than 63 bytes of overhead. Thus, |table| should require
+ // ~1599 ((96 * 16) + 63) bytes of stack space.
+ alignas(64) P256_POINT table[16];
+ uint8_t p_str[33];
+ OPENSSL_memcpy(p_str, p_scalar->bytes, 32);
+ p_str[32] = 0;
+
+ // table[0] is implicitly (0,0,0) (the point at infinity), therefore it is
+ // not stored. All other values are actually stored with an offset of -1 in
+ // table.
+ P256_POINT *row = table;
+ assert(group->field.width == P256_LIMBS);
+ OPENSSL_memcpy(row[1 - 1].X, p->X.words, P256_LIMBS * sizeof(BN_ULONG));
+ OPENSSL_memcpy(row[1 - 1].Y, p->Y.words, P256_LIMBS * sizeof(BN_ULONG));
+ OPENSSL_memcpy(row[1 - 1].Z, p->Z.words, P256_LIMBS * sizeof(BN_ULONG));
+
+ ecp_nistz256_point_double(&row[2 - 1], &row[1 - 1]);
+ ecp_nistz256_point_add(&row[3 - 1], &row[2 - 1], &row[1 - 1]);
+ ecp_nistz256_point_double(&row[4 - 1], &row[2 - 1]);
+ ecp_nistz256_point_double(&row[6 - 1], &row[3 - 1]);
+ ecp_nistz256_point_double(&row[8 - 1], &row[4 - 1]);
+ ecp_nistz256_point_double(&row[12 - 1], &row[6 - 1]);
+ ecp_nistz256_point_add(&row[5 - 1], &row[4 - 1], &row[1 - 1]);
+ ecp_nistz256_point_add(&row[7 - 1], &row[6 - 1], &row[1 - 1]);
+ ecp_nistz256_point_add(&row[9 - 1], &row[8 - 1], &row[1 - 1]);
+ ecp_nistz256_point_add(&row[13 - 1], &row[12 - 1], &row[1 - 1]);
+ ecp_nistz256_point_double(&row[14 - 1], &row[7 - 1]);
+ ecp_nistz256_point_double(&row[10 - 1], &row[5 - 1]);
+ ecp_nistz256_point_add(&row[15 - 1], &row[14 - 1], &row[1 - 1]);
+ ecp_nistz256_point_add(&row[11 - 1], &row[10 - 1], &row[1 - 1]);
+ ecp_nistz256_point_double(&row[16 - 1], &row[8 - 1]);
+
+ BN_ULONG tmp[P256_LIMBS];
+ alignas(32) P256_POINT h;
+ size_t index = 255;
+ crypto_word_t wvalue = p_str[(index - 1) / 8];
+ wvalue = (wvalue >> ((index - 1) % 8)) & kMask;
+
+ ecp_nistz256_select_w5(r, table, booth_recode_w5(wvalue) >> 1);
+
+ while (index >= 5) {
+ if (index != 255) {
+ size_t off = (index - 1) / 8;
+
+ wvalue = (crypto_word_t)p_str[off] | (crypto_word_t)p_str[off + 1] << 8;
+ wvalue = (wvalue >> ((index - 1) % 8)) & kMask;
+
+ wvalue = booth_recode_w5(wvalue);
+
+ ecp_nistz256_select_w5(&h, table, wvalue >> 1);
+
+ ecp_nistz256_neg(tmp, h.Y);
+ copy_conditional(h.Y, tmp, (wvalue & 1));
+
+ ecp_nistz256_point_add(r, r, &h);
+ }
+
+ index -= kWindowSize;
+
+ ecp_nistz256_point_double(r, r);
+ ecp_nistz256_point_double(r, r);
+ ecp_nistz256_point_double(r, r);
+ ecp_nistz256_point_double(r, r);
+ ecp_nistz256_point_double(r, r);
+ }
+
+ // Final window
+ wvalue = p_str[0];
+ wvalue = (wvalue << 1) & kMask;
+
+ wvalue = booth_recode_w5(wvalue);
+
+ ecp_nistz256_select_w5(&h, table, wvalue >> 1);
+
+ ecp_nistz256_neg(tmp, h.Y);
+ copy_conditional(h.Y, tmp, wvalue & 1);
+
+ ecp_nistz256_point_add(r, r, &h);
+}
+
+static void ecp_nistz256_point_mul(const EC_GROUP *group, EC_RAW_POINT *r,
+ const EC_RAW_POINT *p,
+ const EC_SCALAR *scalar) {
+ alignas(32) P256_POINT out;
+ ecp_nistz256_windowed_mul(group, &out, p, scalar);
+
+ assert(group->field.width == P256_LIMBS);
+ OPENSSL_memcpy(r->X.words, out.X, P256_LIMBS * sizeof(BN_ULONG));
+ OPENSSL_memcpy(r->Y.words, out.Y, P256_LIMBS * sizeof(BN_ULONG));
+ OPENSSL_memcpy(r->Z.words, out.Z, P256_LIMBS * sizeof(BN_ULONG));
+}
+
+typedef union {
+ P256_POINT p;
+ P256_POINT_AFFINE a;
+} p256_point_union_t;
+
+static crypto_word_t calc_first_wvalue(size_t *index, const uint8_t p_str[33]) {
+ static const size_t kWindowSize = 7;
+ static const crypto_word_t kMask = (1 << (7 /* kWindowSize */ + 1)) - 1;
+ *index = kWindowSize;
+
+ crypto_word_t wvalue = (p_str[0] << 1) & kMask;
+ return booth_recode_w7(wvalue);
+}
+
+static crypto_word_t calc_wvalue(size_t *index, const uint8_t p_str[33]) {
+ static const size_t kWindowSize = 7;
+ static const crypto_word_t kMask = (1 << (7 /* kWindowSize */ + 1)) - 1;
+
+ const size_t off = (*index - 1) / 8;
+ crypto_word_t wvalue =
+ (crypto_word_t)p_str[off] | (crypto_word_t)p_str[off + 1] << 8;
+ wvalue = (wvalue >> ((*index - 1) % 8)) & kMask;
+ *index += kWindowSize;
+
+ return booth_recode_w7(wvalue);
+}
+
+static void ecp_nistz256_points_mul_public(const EC_GROUP *group,
+ EC_RAW_POINT *r,
+ const EC_SCALAR *g_scalar,
+ const EC_RAW_POINT *p_,
+ const EC_SCALAR *p_scalar) {
+ assert(p_ != NULL && p_scalar != NULL && g_scalar != NULL);
+
+ alignas(32) p256_point_union_t t, p;
+ uint8_t p_str[33];
+ OPENSSL_memcpy(p_str, g_scalar->bytes, 32);
+ p_str[32] = 0;
+
+ // First window
+ size_t index = 0;
+ size_t wvalue = calc_first_wvalue(&index, p_str);
+
+ // Convert |p| from affine to Jacobian coordinates. We set Z to zero if |p|
+ // is infinity and |ONE| otherwise. |p| was computed from the table, so it
+ // is infinity iff |wvalue >> 1| is zero.
+ if ((wvalue >> 1) != 0) {
+ OPENSSL_memcpy(&p.a, &ecp_nistz256_precomputed[0][(wvalue >> 1) - 1],
+ sizeof(p.a));
+ OPENSSL_memcpy(&p.p.Z, ONE, sizeof(p.p.Z));
+ } else {
+ OPENSSL_memset(&p.a, 0, sizeof(p.a));
+ OPENSSL_memset(p.p.Z, 0, sizeof(p.p.Z));
+ }
+
+ if ((wvalue & 1) == 1) {
+ ecp_nistz256_neg(p.p.Y, p.p.Y);
+ }
+
+ for (int i = 1; i < 37; i++) {
+ wvalue = calc_wvalue(&index, p_str);
+
+ if ((wvalue >> 1) == 0) {
+ continue;
+ }
+
+ OPENSSL_memcpy(&t.a, &ecp_nistz256_precomputed[i][(wvalue >> 1) - 1],
+ sizeof(p.a));
+
+ if ((wvalue & 1) == 1) {
+ ecp_nistz256_neg(t.a.Y, t.a.Y);
+ }
+
+ // Note |ecp_nistz256_point_add_affine| does not work if |p.p| and |t.a|
+ // are the same non-infinity point, so it is important that we compute the
+ // |g_scalar| term before the |p_scalar| term.
+ ecp_nistz256_point_add_affine(&p.p, &p.p, &t.a);
+ }
+
+ ecp_nistz256_windowed_mul(group, &t.p, p_, p_scalar);
+ ecp_nistz256_point_add(&p.p, &p.p, &t.p);
+
+ assert(group->field.width == P256_LIMBS);
+ OPENSSL_memcpy(r->X.words, p.p.X, P256_LIMBS * sizeof(BN_ULONG));
+ OPENSSL_memcpy(r->Y.words, p.p.Y, P256_LIMBS * sizeof(BN_ULONG));
+ OPENSSL_memcpy(r->Z.words, p.p.Z, P256_LIMBS * sizeof(BN_ULONG));
+}
+
+static void ecp_nistz256_inv0_mod_ord(const EC_GROUP *group, EC_SCALAR *out,
+ const EC_SCALAR *in) {
+ // table[i] stores a power of |in| corresponding to the matching enum value.
Expand Down Expand Up @@ -340,9 +589,9 @@ Index: aws-lc/third_party/boringssl/crypto/fipsmodule/ec/p256.c
+ out->point_get_affine_coordinates = ecp_nistz256_get_affine;
+ out->add = ecp_nistz256_add;
+ out->dbl = ecp_nistz256_dbl;
+ out->mul = ec_GFp_nistp256_point_mul;
+ out->mul = ecp_nistz256_point_mul;
+ out->mul_base = ec_GFp_nistp256_point_mul_base;
+ out->mul_public = ec_GFp_nistp256_point_mul_public;
+ out->mul_public = ecp_nistz256_points_mul_public;
+ out->felem_mul = ec_GFp_mont_felem_mul;
+ out->felem_sqr = ec_GFp_mont_felem_sqr;
+ out->felem_to_bytes = ec_GFp_mont_felem_to_bytes;
Expand All @@ -359,7 +608,7 @@ Index: aws-lc/third_party/boringssl/crypto/fipsmodule/ec/asm/p256-armv8-asm.pl
===================================================================
--- /dev/null
+++ aws-lc/third_party/boringssl/crypto/fipsmodule/ec/asm/p256-armv8-asm.pl
@@ -0,0 +1,1874 @@
@@ -0,0 +1,1966 @@
+#! /usr/bin/env perl
+# Copyright 2015-2020 The OpenSSL Project Authors. All Rights Reserved.
+#
Expand Down Expand Up @@ -1990,6 +2239,98 @@ Index: aws-lc/third_party/boringssl/crypto/fipsmodule/ec/asm/p256-armv8-asm.pl
+} }
+
+########################################################################
+# select subroutines
+# These select functions are ported from p256-x86_64-asm.pl
+# The x86_64 instructions are included here as comments starting with ////
+{
+my ($val,$in_t)=map("x$_",(0..1));
+my ($index)=("w2");
+my ($Ctr,$Val_in)=("w9", "x10");
+my ($ONE,$M0,$INDEX,$TMP0)=map("v$_",(0..3));
+my ($Ra,$Rb,$Rc,$Rd,$Re,$Rf)=map("v$_",(16..21));
+my ($T0a,$T0b,$T0c,$T0d,$T0e,$T0f)=map("v$_",(22..27));
+$code.=<<___;
+// void ecp_nistz256_select_w5(uint64_t *val, uint64_t *in_t, int index);
+.globl ecp_nistz256_select_w5
+.type ecp_nistz256_select_w5,%function
+.align 4
+ecp_nistz256_select_w5:
+ // $ONE (vec_4*32) := | 1 | 1 | 1 | 1 |
+ // $INDEX (vec_4*32) := | idx | idx | idx | idx |
+ movi $ONE.4s, #1 //// movdqa .LOne(%rip), $ONE
+ dup $INDEX.4s, $index //// movd $index, $INDEX
+ //// pshufd \$0, $INDEX, $INDEX
+
+ // [$Ra-$Rf] := 0
+ eor $Ra.16b, $Ra.16b, $Ra.16b //// pxor $Ra, $Ra
+ eor $Rb.16b, $Rb.16b, $Rb.16b //// pxor $Rb, $Rb
+ eor $Rc.16b, $Rc.16b, $Rc.16b //// pxor $Rc, $Rc
+ eor $Rd.16b, $Rd.16b, $Rd.16b //// pxor $Rd, $Rd
+ eor $Re.16b, $Re.16b, $Re.16b //// pxor $Re, $Re
+ eor $Rf.16b, $Rf.16b, $Rf.16b //// pxor $Rf, $Rf
+
+ // $M0 := $ONE
+ mov $M0.16b, $ONE.16b //// movdqa $ONE, $M0
+
+ // $Val_in := $val
+ // $Ctr := 16; loop counter
+ mov $Val_in, $val
+ mov $Ctr, #16 //// mov \$16, %rax
+
+.Lselect_w5_loop:
+ // [$T0a-$T0f] := Load a (3*256-bit = 6*128-bit) table entry starting at $in_t
+ // and advance $in_t to point to the next entry
+ ld1 {$T0a.2d, $T0b.2d, $T0c.2d, $T0d.2d}, [$in_t],#64
+ ld1 {$T0e.2d, $T0f.2d}, [$in_t],#32 //// movdqa 16*0($in_t), $T0a
+ //// movdqa 16*1($in_t), $T0b
+ //// movdqa 16*2($in_t), $T0c
+ //// movdqa 16*3($in_t), $T0d
+ //// movdqa 16*4($in_t), $T0e
+ //// movdqa 16*5($in_t), $T0f
+ //// lea 16*6($in_t), $in_t
+
+ // $TMP0 = ($M0 == $INDEX)? All 1s : All 0s
+ cmeq $TMP0.4s, $M0.4s, $INDEX.4s //// movdqa $M0, $TMP0
+ //// pcmpeqd $INDEX, $TMP0
+ // Increment $M0 lanes
+ add $M0.4s, $M0.4s, $ONE.4s //// paddd $ONE, $M0
+
+ // [$T0a-$T0f] := [$T0a-$T0f] AND $TMP0
+ // values read from the table will be 0'd if $M0 != $INDEX
+ // [$Ra-$Rf] := [$Ra-$Rf] OR [$T0a-$T0b]
+ // values in output registers will remain the same if $M0 != $INDEX
+ and $T0a.16b, $T0a.16b, $TMP0.16b //// pand $TMP0, $T0a
+ and $T0b.16b, $T0b.16b, $TMP0.16b //// pand $TMP0, $T0b
+ orr $Ra.16b, $Ra.16b, $T0a.16b //// por $T0a, $Ra
+ orr $Rb.16b, $Rb.16b, $T0b.16b //// pand $TMP0, $T0c
+ //// por $T0b, $Rb
+ and $T0c.16b, $T0c.16b, $TMP0.16b //// pand $TMP0, $T0d
+ and $T0d.16b, $T0d.16b, $TMP0.16b //// por $T0c, $Rc
+ orr $Rc.16b, $Rc.16b, $T0c.16b //// pand $TMP0, $T0e
+ orr $Rd.16b, $Rd.16b, $T0d.16b //// por $T0d, $Rd
+ //// pand $TMP0, $T0f
+ and $T0e.16b, $T0e.16b, $TMP0.16b //// por $T0e, $Re
+ and $T0f.16b, $T0f.16b, $TMP0.16b //// por $T0f, $Rf
+ orr $Re.16b, $Re.16b, $T0e.16b
+ orr $Rf.16b, $Rf.16b, $T0f.16b
+
+ // Decrement loop counter; loop back if not 0
+ subs $Ctr, $Ctr, #1 //// dec %rax
+ bne .Lselect_w5_loop //// jnz .Lselect_loop_sse_w5
+
+ // Write [$Ra-$Rf] to memory at the output pointer
+ st1 {$Ra.2d, $Rb.2d, $Rc.2d, $Rd.2d}, [$Val_in],#64
+ st1 {$Re.2d, $Rf.2d}, [$Val_in] //// movdqu $Ra, 16*0($val)
+ //// movdqu $Rb, 16*1($val)
+ //// movdqu $Rc, 16*2($val)
+ //// movdqu $Rd, 16*3($val)
+ //// movdqu $Re, 16*4($val)
+ //// movdqu $Rf, 16*5($val)
+ ret
+.size ecp_nistz256_select_w5,.-ecp_nistz256_select_w5
+___
+}
+########################################################################
+# scatter-gather subroutines
+{
+my ($out,$inp,$index,$mask)=map("x$_",(0..3));
Expand Down

0 comments on commit ae08fa6

Please sign in to comment.