Add Internal SHAKE interfaces and their NIST test vectors. (#572)

Add SHAKE128 and SHAKE256 internal APIs and test vector units `SHAKE128Test.NISTTestVectors` and `SHAKE256Test.NISTTestVectors` based on NIST Test Vectors.

Co-authored-by: Anastasova <[email protected]>
Co-authored-by: dkostic <[email protected]>
Co-authored-by: torben-hansen <[email protected]>
Co-authored-by: Andrew Hopkins <[email protected]>

crypto/fipsmodule/sha/internal.h

@@ -21,6 +21,7 @@ extern "C" {
 
 // SHA3 constants, from NIST FIPS202.
 // https://nvlpubs.nist.gov/nistpubs/FIPS/NIST.FIPS.202.pdf
+#define SHA3_ROWS 5
 #define KECCAK1600_WIDTH 1600
 
 #define SHA3_224_CAPACITY_BYTES 56
@@ -39,24 +40,28 @@ extern "C" {
 #define SHA3_512_DIGEST_BITLENGTH 512
 #define SHA3_512_DIGEST_LENGTH 64
 
-// The |SHA3_MIN_CAPACITY_BYTES| corresponds to the 
-// lowest security level capacity.
-// The data block size increases when the capacity decreases.
-// The data buffer should have at least the maximum 
-// block size bytes to fit any digest length.
 #define SHA3_BLOCKSIZE(bitlen) (KECCAK1600_WIDTH - bitlen * 2) / 8
-#define SHA3_MIN_CAPACITY_BYTES SHA3_224_CAPACITY_BYTES
 #define SHA3_PAD_CHAR 0x06
-#define SHA3_ROWS 5
+
+// SHAKE constants, from NIST FIPS202.
+// https://nvlpubs.nist.gov/nistpubs/FIPS/NIST.FIPS.202.pdf
+#define SHAKE_PAD_CHAR 0x1F
+#define SHAKE128_BLOCKSIZE (KECCAK1600_WIDTH - 128 * 2) / 8
+#define SHAKE256_BLOCKSIZE (KECCAK1600_WIDTH - 256 * 2) / 8
+
+// SHAKE128 has the maximum block size among the SHA3/SHAKE algorithms.
+#define SHA3_MAX_BLOCKSIZE SHAKE128_BLOCKSIZE
 
 typedef struct keccak_st KECCAK1600_CTX;
 
+// The data buffer should have at least the maximum number of
+// block size bytes to fit any SHA3/SHAKE block length.
 struct keccak_st {
   uint64_t A[SHA3_ROWS][SHA3_ROWS];
-  size_t block_size;                                             // cached ctx->digest->block_size
-  size_t md_size;                                                // output length, variable in XOF (SHAKE)
-  size_t buf_load;                                               // used bytes in below buffer
-  uint8_t buf[KECCAK1600_WIDTH / 8 - SHA3_MIN_CAPACITY_BYTES];   // should have at least the max data block size bytes
+  size_t block_size;                               // cached ctx->digest->block_size
+  size_t md_size;                                  // output length, variable in XOF (SHAKE)
+  size_t buf_load;                                 // used bytes in below buffer
+  uint8_t buf[SHA3_MAX_BLOCKSIZE];                 // should have at least the max data block size bytes
   uint8_t pad;
 };
 
@@ -107,30 +112,40 @@ OPENSSL_EXPORT uint8_t *SHA3_384(const uint8_t *data, size_t len,
 OPENSSL_EXPORT uint8_t *SHA3_512(const uint8_t *data, size_t len,
                   uint8_t out[SHA3_512_DIGEST_LENGTH]);
 
+// SHAKE128 writes the |out_len| bytes output from |in_len| bytes |data| 
+// to |out| and returns |out| on success and NULL on failure. 
+OPENSSL_EXPORT uint8_t *SHAKE128(const uint8_t *data, const size_t in_len, 
+                                 uint8_t *out, size_t out_len);
+
+// SHAKE256 writes |out_len| bytes output from |in_len| bytes |data| 
+// to |out| and returns |out| on success and NULL on failure. 
+OPENSSL_EXPORT uint8_t *SHAKE256(const uint8_t *data, const size_t in_len, 
+                                 uint8_t *out, size_t out_len);
+
 // SHA3_Reset zeros the bitstate and the amount of processed input.
 OPENSSL_EXPORT void SHA3_Reset(KECCAK1600_CTX *ctx);
 
 // SHA3_Init initialises |ctx| fields and returns 1 on success and 0 on failure.
 OPENSSL_EXPORT int SHA3_Init(KECCAK1600_CTX *ctx, uint8_t pad,
                              size_t bitlen);
 
-// SHA3_Update processes all data blocks that don't need pad through |SHA3_Absorb| and returns 1 and 0 on failure.
+// SHA3_Update processes all data blocks that don't need pad through 
+// |SHA3_Absorb| and returns 1 and 0 on failure.
 OPENSSL_EXPORT int SHA3_Update(KECCAK1600_CTX *ctx, const void *data,
                                size_t len);
 
-// SHA3_Final pads the last block of data and proccesses it through |SHA3_Absorb|. 
+// SHA3_Final pads the last data block and processes it through |SHA3_Absorb|. 
 // It processes the data through |SHA3_Squeeze| and returns 1 and 0 on failure.
 OPENSSL_EXPORT int SHA3_Final(uint8_t *md, KECCAK1600_CTX *ctx);
 
 // SHA3_Absorb processes the largest multiple of |r| out of |len| bytes and 
 // returns the remaining number of bytes. 
-OPENSSL_EXPORT size_t SHA3_Absorb(uint64_t A[SHA3_ROWS][SHA3_ROWS], const uint8_t *data,
-                                  size_t len, size_t r);
+OPENSSL_EXPORT size_t SHA3_Absorb(uint64_t A[SHA3_ROWS][SHA3_ROWS], 
+                                  const uint8_t *data, size_t len, size_t r);
 
 // SHA3_Squeeze generate |out| hash value of |len| bytes.
-OPENSSL_EXPORT void SHA3_Squeeze(uint64_t A[SHA3_ROWS][SHA3_ROWS], uint8_t *out,
-                                 size_t len, size_t r);
-
+OPENSSL_EXPORT void SHA3_Squeeze(uint64_t A[SHA3_ROWS][SHA3_ROWS], 
+                                 uint8_t *out, size_t len, size_t r);
 
 #if defined(__cplusplus)
 }  // extern "C"

crypto/fipsmodule/sha/sha3.c

@@ -67,6 +67,42 @@ uint8_t *SHA3_512(const uint8_t *data, size_t len,
   return out;
 }
 
+uint8_t *SHAKE128(const uint8_t *data, const size_t in_len, uint8_t *out, size_t out_len) {
+  KECCAK1600_CTX ctx;
+
+  // The SHAKE block size depends on the security level of the algorithm only
+  // It is independent of the output size
+  ctx.block_size = SHAKE128_BLOCKSIZE;
+
+  int ok = (SHA3_Init(&ctx, SHAKE_PAD_CHAR, out_len) && 
+            SHA3_Update(&ctx, data, in_len) &&
+            SHA3_Final(out, &ctx));
+
+  OPENSSL_cleanse(&ctx, sizeof(ctx));
+  if (ok == 0) {
+    return NULL;
+  }
+  return out;
+}
+
+uint8_t *SHAKE256(const uint8_t *data, const size_t in_len, uint8_t *out, size_t out_len) {
+  KECCAK1600_CTX ctx;
+
+  // The SHAKE block size depends on the security level of the algorithm only
+  // It is independent of the output size
+  ctx.block_size = SHAKE256_BLOCKSIZE;
+
+  int ok = (SHA3_Init(&ctx, SHAKE_PAD_CHAR, out_len) && 
+            SHA3_Update(&ctx, data, in_len) &&
+            SHA3_Final(out, &ctx));
+
+  OPENSSL_cleanse(&ctx, sizeof(ctx));
+  if (ok == 0) {
+    return NULL;
+  }
+  return out;
+}
+
 void SHA3_Reset(KECCAK1600_CTX *ctx) {
   memset(ctx->A, 0, sizeof(ctx->A));
   ctx->buf_load = 0;
@@ -77,7 +113,22 @@ int SHA3_Init(KECCAK1600_CTX *ctx, uint8_t pad, size_t bit_len) {
     return 0;
   }
 
-  size_t block_size = SHA3_BLOCKSIZE(bit_len);
+  size_t block_size;
+
+  // The block size is computed differently depending on which algorithm
+  // is calling |SHA3_Init|:
+  //   - for SHA3 we compute it by calling SHA3_BLOCKSIZE(bit_len)
+  //     because the block size depends on the digest bit-length,
+  //   - for SHAKE we take the block size from the context.
+  // We use the given padding character to differentiate between SHA3 and SHAKE.
+  if (pad == SHA3_PAD_CHAR) {
+    block_size = SHA3_BLOCKSIZE(bit_len);
+  } else if (pad == SHAKE_PAD_CHAR) {
+    block_size = ctx->block_size;
+  } else {
+    return 0;
+  }
+
   if (block_size <= sizeof(ctx->buf)) {
     SHA3_Reset(ctx);
     ctx->block_size = block_size;

crypto/fipsmodule/sha/sha3_test.cc

@@ -24,23 +24,23 @@ class SHA3TestVector {
 
   void NISTTestVectors(const EVP_MD *algorithm) const {
     uint32_t digest_length;
-    uint8_t *digest  = new uint8_t[EVP_MD_size(algorithm)];
+    std::unique_ptr<uint8_t[]> digest(new uint8_t[EVP_MD_size(algorithm)]);
     EVP_MD_CTX* ctx = EVP_MD_CTX_new();
 
     // SHA3 is disabled by default. First test this assumption and then enable SHA3 and test it.
     ASSERT_FALSE(EVP_DigestInit(ctx, algorithm));
     ASSERT_FALSE(EVP_DigestUpdate(ctx, msg_.data(), len_ / 8));
-    ASSERT_FALSE(EVP_DigestFinal(ctx, digest, &digest_length));
+    ASSERT_FALSE(EVP_DigestFinal(ctx, digest.get(), &digest_length));
 
     // Enable SHA3
     EVP_MD_unstable_sha3_enable(true);
 
     // Test the correctness via the Init, Update and Final Digest APIs.
     ASSERT_TRUE(EVP_DigestInit(ctx, algorithm));
     ASSERT_TRUE(EVP_DigestUpdate(ctx, msg_.data(), len_ / 8));
-    ASSERT_TRUE(EVP_DigestFinal(ctx, digest, &digest_length));
+    ASSERT_TRUE(EVP_DigestFinal(ctx, digest.get(), &digest_length));
 
-    ASSERT_EQ(Bytes(digest, EVP_MD_size(algorithm)),
+    ASSERT_EQ(Bytes(digest.get(), EVP_MD_size(algorithm)),
               Bytes(digest_.data(), EVP_MD_size(algorithm)));
 
     // Disable SHA3
@@ -49,42 +49,80 @@ class SHA3TestVector {
     // Test again SHA3 when |unstable_sha3_enabled_flag| is disabled.
     ASSERT_FALSE(EVP_DigestInit(ctx, algorithm));
     ASSERT_FALSE(EVP_DigestUpdate(ctx, msg_.data(), len_ / 8));
-    ASSERT_FALSE(EVP_DigestFinal(ctx, digest, &digest_length));
+    ASSERT_FALSE(EVP_DigestFinal(ctx, digest.get(), &digest_length));
 
-    delete [] digest;
     OPENSSL_free(ctx);
   }
 
   void NISTTestVectors_SingleShot(const EVP_MD *algorithm) const {
     uint32_t digest_length;
-    uint8_t *digest  = new uint8_t[EVP_MD_size(algorithm)];
+    std::unique_ptr<uint8_t[]> digest(new uint8_t[EVP_MD_size(algorithm)]);
     EVP_MD_CTX* ctx = EVP_MD_CTX_new();
 
     // SHA3 is disabled by default. First test this assumption and then enable SHA3 and test it.
-    ASSERT_FALSE(EVP_Digest(msg_.data(), len_ / 8, digest, &digest_length, algorithm, NULL));
+    ASSERT_FALSE(EVP_Digest(msg_.data(), len_ / 8, digest.get(), &digest_length, algorithm, NULL));
 
     // Enable SHA3
     EVP_MD_unstable_sha3_enable(true);
 
     // Test the correctness via the Single-Shot EVP_Digest APIs.
-    ASSERT_TRUE(EVP_Digest(msg_.data(), len_ / 8, digest, &digest_length, algorithm, NULL));
+    ASSERT_TRUE(EVP_Digest(msg_.data(), len_ / 8, digest.get(), &digest_length, algorithm, NULL));
 
-    ASSERT_EQ(Bytes(digest, EVP_MD_size(algorithm)),
+    ASSERT_EQ(Bytes(digest.get(), EVP_MD_size(algorithm)),
               Bytes(digest_.data(), EVP_MD_size(algorithm)));
 
     // Disable SHA3
     EVP_MD_unstable_sha3_enable(false);
 
     // Test again SHA3 when |unstable_sha3_enabled_flag| is disabled.
-    ASSERT_FALSE(EVP_Digest(msg_.data(), len_ / 8, digest, &digest_length, algorithm, NULL));
+    ASSERT_FALSE(EVP_Digest(msg_.data(), len_ / 8, digest.get(), &digest_length, algorithm, NULL));
 
-    delete [] digest;
     OPENSSL_free(ctx);
+  }
+
+  void NISTTestVectors_SHAKE128() const {
+    uint32_t digest_length = out_len_ / 8;
+    std::unique_ptr<uint8_t[]> digest(new uint8_t[digest_length]);
+
+    ASSERT_FALSE(SHAKE128(msg_.data(), msg_.size() , digest.get(), out_len_));
+
+    // Enable SHA3
+    EVP_MD_unstable_sha3_enable(true);
+
+    ASSERT_TRUE(SHAKE128(msg_.data(), msg_.size() , digest.get(), out_len_));
+
+    ASSERT_EQ(Bytes(digest.get(), out_len_ / 8),
+            Bytes(digest_.data(), out_len_ / 8));
+
+    // Disable SHA3
+    EVP_MD_unstable_sha3_enable(false);
+
+    ASSERT_FALSE(SHAKE128(msg_.data(), msg_.size() , digest.get(), out_len_));
+  }
+
+  void NISTTestVectors_SHAKE256() const {
+    uint32_t digest_length = out_len_ / 8;
+    std::unique_ptr<uint8_t[]> digest(new uint8_t[digest_length]);
+
+    ASSERT_FALSE(SHAKE256(msg_.data(), msg_.size() , digest.get(), out_len_));
+
+    // Enable SHA3
+    EVP_MD_unstable_sha3_enable(true);
+
+    ASSERT_TRUE(SHAKE256(msg_.data(), msg_.size() , digest.get(), out_len_));
+
+    ASSERT_EQ(Bytes(digest.get(), out_len_ / 8),
+            Bytes(digest_.data(), out_len_ / 8));
 
+    // Disable SHA3
+    EVP_MD_unstable_sha3_enable(false);
+
+    ASSERT_FALSE(SHAKE256(msg_.data(), msg_.size() , digest.get(), out_len_));
   }
 
  private:
-  uint16_t len_;
+  uint32_t len_;
+  uint32_t out_len_;
   std::vector<uint8_t> msg_;
   std::vector<uint8_t> digest_;
 };
@@ -99,34 +137,46 @@ bool FileTestReadInt(FileTest *file_test, T *out, const std::string &key) {
 }
 
 bool SHA3TestVector::ReadFromFileTest(FileTest *t) {
-  if (!FileTestReadInt(t, &len_, "Len") ||
-      !t->GetBytes(&msg_, "Msg") ||
+   if (t->HasAttribute("Outputlen")) {
+    if (!FileTestReadInt(t, &out_len_, "Outputlen")) {
+      return false;
+    }
+  }
+
+  if (t->HasAttribute("Len")) {
+    if (!FileTestReadInt(t, &len_, "Len")) {
+      return false;
+    }
+  }
+
+  if (!t->GetBytes(&msg_, "Msg") ||
       !t->GetBytes(&digest_, "MD")) {
     return false;
   }
+
   return true;
 }
 
 TEST(SHA3Test, NISTTestVectors) {
-  FileTestGTest("crypto/fipsmodule/sha/SHA3_224ShortMsg.txt", [](FileTest *t) {
+  FileTestGTest("crypto/fipsmodule/sha/testvectors/SHA3_224ShortMsg.txt", [](FileTest *t) {
     SHA3TestVector test_vec;
     EXPECT_TRUE(test_vec.ReadFromFileTest(t));
     const EVP_MD* algorithm = EVP_sha3_224();
     test_vec.NISTTestVectors(algorithm);
   });
-  FileTestGTest("crypto/fipsmodule/sha/SHA3_256ShortMsg.txt", [](FileTest *t) {
+  FileTestGTest("crypto/fipsmodule/sha/testvectors/SHA3_256ShortMsg.txt", [](FileTest *t) {
     SHA3TestVector test_vec;
     EXPECT_TRUE(test_vec.ReadFromFileTest(t));
     const EVP_MD* algorithm = EVP_sha3_256();
     test_vec.NISTTestVectors(algorithm);
   });
-  FileTestGTest("crypto/fipsmodule/sha/SHA3_384ShortMsg.txt", [](FileTest *t) {
+  FileTestGTest("crypto/fipsmodule/sha/testvectors/SHA3_384ShortMsg.txt", [](FileTest *t) {
     SHA3TestVector test_vec;
     EXPECT_TRUE(test_vec.ReadFromFileTest(t));
     const EVP_MD* algorithm = EVP_sha3_384();
     test_vec.NISTTestVectors(algorithm);
   });
-  FileTestGTest("crypto/fipsmodule/sha/SHA3_512ShortMsg.txt", [](FileTest *t) {
+  FileTestGTest("crypto/fipsmodule/sha/testvectors/SHA3_512ShortMsg.txt", [](FileTest *t) {
     SHA3TestVector test_vec;
     EXPECT_TRUE(test_vec.ReadFromFileTest(t));
     const EVP_MD* algorithm = EVP_sha3_512();
@@ -135,28 +185,44 @@ TEST(SHA3Test, NISTTestVectors) {
 }
 
 TEST(SHA3Test, NISTTestVectors_SingleShot) {
-  FileTestGTest("crypto/fipsmodule/sha/SHA3_224ShortMsg.txt", [](FileTest *t) {
+  FileTestGTest("crypto/fipsmodule/sha/testvectors/SHA3_224ShortMsg.txt", [](FileTest *t) {
     SHA3TestVector test_vec;
     EXPECT_TRUE(test_vec.ReadFromFileTest(t));
     const EVP_MD* algorithm = EVP_sha3_224();
     test_vec.NISTTestVectors_SingleShot(algorithm);
   });
-  FileTestGTest("crypto/fipsmodule/sha/SHA3_256ShortMsg.txt", [](FileTest *t) {
+  FileTestGTest("crypto/fipsmodule/sha/testvectors/SHA3_256ShortMsg.txt", [](FileTest *t) {
     SHA3TestVector test_vec;
     EXPECT_TRUE(test_vec.ReadFromFileTest(t));
     const EVP_MD* algorithm = EVP_sha3_256();
     test_vec.NISTTestVectors_SingleShot(algorithm);
   });
-  FileTestGTest("crypto/fipsmodule/sha/SHA3_384ShortMsg.txt", [](FileTest *t) {
+  FileTestGTest("crypto/fipsmodule/sha/testvectors/SHA3_384ShortMsg.txt", [](FileTest *t) {
     SHA3TestVector test_vec;
     EXPECT_TRUE(test_vec.ReadFromFileTest(t));
     const EVP_MD* algorithm = EVP_sha3_384();
     test_vec.NISTTestVectors_SingleShot(algorithm);
   });
-  FileTestGTest("crypto/fipsmodule/sha/SHA3_512ShortMsg.txt", [](FileTest *t) {
+  FileTestGTest("crypto/fipsmodule/sha/testvectors/SHA3_512ShortMsg.txt", [](FileTest *t) {
     SHA3TestVector test_vec;
     EXPECT_TRUE(test_vec.ReadFromFileTest(t));
     const EVP_MD* algorithm = EVP_sha3_512();
     test_vec.NISTTestVectors_SingleShot(algorithm);
   });
 }
+
+TEST(SHAKE128Test, NISTTestVectors) {
+  FileTestGTest("crypto/fipsmodule/sha/testvectors/SHAKE128VariableOut.txt", [](FileTest *t) {
+    SHA3TestVector test_vec;
+    EXPECT_TRUE(test_vec.ReadFromFileTest(t));
+    test_vec.NISTTestVectors_SHAKE128();
+  });
+}
+
+TEST(SHAKE256Test, NISTTestVectors) {
+  FileTestGTest("crypto/fipsmodule/sha/testvectors/SHAKE256VariableOut.txt", [](FileTest *t) {
+    SHA3TestVector test_vec;
+    EXPECT_TRUE(test_vec.ReadFromFileTest(t));
+    test_vec.NISTTestVectors_SHAKE256();
+  });
+}