SHA3 xof shake128 and shake256

src/SHA.jl

@@ -38,15 +38,20 @@ export sha1, SHA1_CTX, update!, digest!
 export sha224, sha256, sha384, sha512
 export sha2_224, sha2_256, sha2_384, sha2_512
 export sha3_224, sha3_256, sha3_384, sha3_512
+export shake_128, shake_256
 export SHA224_CTX, SHA256_CTX, SHA384_CTX, SHA512_CTX
 export SHA2_224_CTX, SHA2_256_CTX, SHA2_384_CTX, SHA2_512_CTX
 export SHA3_224_CTX, SHA3_256_CTX, SHA3_384_CTX, SHA3_512_CTX
+export SHAKE_128_CTX, SHAKE_256_CTX
 export HMAC_CTX, hmac_sha1
 export hmac_sha224, hmac_sha256, hmac_sha384, hmac_sha512
 export hmac_sha2_224, hmac_sha2_256, hmac_sha2_384, hmac_sha2_512
 export hmac_sha3_224, hmac_sha3_256, hmac_sha3_384, hmac_sha3_512
 
 # data to be hashed:
+"""
+Union{AbstractVector{UInt8}, NTuple{N, UInt8} where N}
+"""
 const AbstractBytes = Union{AbstractVector{UInt8},NTuple{N,UInt8} where N}
 
 include("constants.jl")
@@ -55,6 +60,7 @@ include("base_functions.jl")
 include("sha1.jl")
 include("sha2.jl")
 include("sha3.jl")
+include("shake.jl")
 include("common.jl")
 include("hmac.jl")
 
@@ -71,7 +77,8 @@ for (f, ctx) in [(:sha1, :SHA1_CTX),
                  (:sha3_224, :SHA3_224_CTX),
                  (:sha3_256, :SHA3_256_CTX),
                  (:sha3_384, :SHA3_384_CTX),
-                 (:sha3_512, :SHA3_512_CTX),]
+                 (:sha3_512, :SHA3_512_CTX),
+                 (:shake_256, :SHAKE_256_CTX),]
     g = Symbol(:hmac_, f)
 
     @eval begin
@@ -138,4 +145,4 @@ for (f, ctx) in [(:sha1, :SHA1_CTX),
     end
 end
 
-end #module SHA
+end #module SHA

src/shake.jl

@@ -0,0 +1,134 @@
+abstract type SHAKE <: SHA3_CTX end 
+# note, that field property used has differend uses, depending on T<:SHAKE or T<:SHA3_CTX
+mutable struct SHAKE_128_CTX <: SHAKE
+    state::Array{UInt64,1}
+    bytecount::UInt128
+    buffer::Array{UInt8,1}
+    bc::Array{UInt64,1}
+    used::Bool
+end
+mutable struct SHAKE_256_CTX <: SHAKE
+    state::Array{UInt64,1}
+    bytecount::UInt128
+    buffer::Array{UInt8,1}
+    bc::Array{UInt64,1}
+    used::Bool
+end
+
+digestlen(::Type{SHAKE_128_CTX}) = 16
+digestlen(::Type{SHAKE_256_CTX}) = 32
+blocklen(::Type{SHAKE_128_CTX}) = UInt64(25*8 - 2*digestlen(SHAKE_128_CTX))
+blocklen(::Type{SHAKE_256_CTX}) = UInt64(25*8 - 2*digestlen(SHAKE_256_CTX))
+buffer_pointer(ctx::T) where {T<:SHAKE} = Ptr{state_type(T)}(pointer(ctx.buffer))
+
+# construct an empty SHA context
+SHAKE_128_CTX() = SHAKE_128_CTX(zeros(UInt64, 25), 0, zeros(UInt8, blocklen(SHAKE_128_CTX)), Vector{UInt64}(undef, 5), false)
+SHAKE_256_CTX() = SHAKE_256_CTX(zeros(UInt64, 25), 0, zeros(UInt8, blocklen(SHAKE_256_CTX)), Vector{UInt64}(undef, 5), false)
+
+function transform!(context::T) where {T<:SHAKE}
+    # First, update state with buffer
+    pbuf = Ptr{eltype(context.state)}(pointer(context.buffer))
+    # after SHAKE_256_MAX_READ (digestlen) is reached, simply work with context.state[idx]
+    if !context.used 
+        for idx in 1:div(blocklen(T),8)
+            context.state[idx] = context.state[idx] ⊻ unsafe_load(pbuf, idx)
+        end
+    end 
+    bc = context.bc
+    state = context.state
+    # We always assume 24 rounds
+    @inbounds for round in 0:23
+        # Theta function
+        for i in 1:5
+            bc[i] = state[i] ⊻ state[i + 5] ⊻ state[i + 10] ⊻ state[i + 15] ⊻ state[i + 20]
+        end
+        for i in 0:4
+            temp = bc[rem(i + 4, 5) + 1] ⊻ L64(1, bc[rem(i + 1, 5) + 1])
+            for j in 0:5:20
+                state[Int(i + j + 1)] = state[i + j + 1] ⊻ temp
+            end
+        end
+        # Rho Pi
+        temp = state[2]
+        for i in 1:24
+            j = SHA3_PILN[i]
+            bc[1] = state[j]
+            state[j] = L64(SHA3_ROTC[i], temp)
+            temp = bc[1]
+        end
+        # Chi
+        for j in 0:5:20
+            for i in 1:5
+                bc[i] = state[i + j]
+            end
+            for i in 0:4
+                state[j + i + 1] = state[j + i + 1] ⊻ (~bc[rem(i + 1, 5) + 1] & bc[rem(i + 2, 5) + 1])
+            end
+        end
+        # Iota
+        state[1] = state[1] ⊻ SHA3_ROUND_CONSTS[round+1]
+    end
+    return context.state
+end
+function digest!(context::T,d::UInt,p::Ptr{UInt8}) where {T<:SHAKE}
+    usedspace = context.bytecount % blocklen(T)
+    # If we have anything in the buffer still, pad and transform that data
+    if usedspace < blocklen(T) - 1
+        # Begin padding with a 0x1f
+        context.buffer[usedspace+1] = 0x1f
+        # Fill with zeros up until the last byte
+        context.buffer[usedspace+2:end-1] .= 0x00
+        # Finish it off with a 0x80
+        context.buffer[end] = 0x80
+    else
+        # Otherwise, we have to add on a whole new buffer
+        context.buffer[end] = 0x1f
+        transform!(context)
+        context.buffer[1:end-1] .= 0x0
+        context.buffer[end] = 0x80
+    end
+    # Final transform:
+    transform!(context)
+    # Return the digest:
+    # fill the given memory via pointer, if d>blocklen, update pointer and digest again.
+    if d <= blocklen(T)
+        for i = 1:d
+            unsafe_store!(p,reinterpret(UInt8, context.state)[i],i)
+        end 
+        return
+    else 
+        for i = 1:blocklen(T)
+            unsafe_store!(p,reinterpret(UInt8, context.state)[i],i)
+        end 
+        context.used = true
+        p+=blocklen(T)
+        digest!(context,d-blocklen(T),p)
+        return 
+    end
+end
+"""
+    shake128(data::AbstractBytes,d::UInt)
+
+Hash data using the `shake128` algorithm and return the first d resulting bytes.
+"""
+function shake128(data::AbstractBytes,d::UInt)
+    ctx = SHAKE_128_CTX()
+    update!(ctx, data)
+    M = Array{UInt8,1}(undef,d)     # prealloc
+    p = pointer(M)
+    digest!(ctx,d,p)
+    return M
+end
+"""
+    shake256(data::AbstractBytes,d::UInt)
+
+Hash data using the `shake258` algorithm and return the first d resulting bytes.
+"""
+function shake256(data::AbstractBytes,d::UInt)
+    ctx = SHAKE_256_CTX()
+    update!(ctx, data)
+    M = Array{UInt8,1}(undef,d)     # prealloc
+    p = pointer(M)
+    digest!(ctx,d,p)
+    return M
+end

test/constants.jl

@@ -101,4 +101,21 @@ hmac_data = (
     ("", "", hmac_sha256, "b613679a0814d9ec772f95d778c35fc5ff1697c493715653c6c712144292c5ad"),
     ("key", "The quick brown fox jumps over the lazy dog", hmac_sha1, "de7c9b85b8b78aa6bc8a7a36f70a90701c9db4d9"),
     ("key", "The quick brown fox jumps over the lazy dog", hmac_sha256, "f7bc83f430538424b13298e6aa6fb143ef4d59a14946175997479dbc2d1a3cd8"),
-)
+)
+
+# shake 128 testvectors
+SHA128test = Dict([ ("d9e8",UInt(16)) => "c7211512340734235bb8d3c4651495aa",
+                    ("3fc3dc539de2171e05909d1f89a6b01b302036f69c29756bea781b",UInt(16)) => "585739f75aaf8f980e7505e841981450",     
+                    ("ef8779a8789c6a860e8eb485b9d83714fa832aa45499bab80bddf43b2cb9e517cd78000396b5c09c81138eb8dd2fe4f799d3b39bbf6d02993fbfd8efeed75d39ce423c7f907d4b5257ac6e17bf8eb9a21eb1d4070ae2a4ac1be827a3f172a037794df5c56dda55ce1ac01d4966defd3e118457903be2e71b27c0a9fe3e70f73a5701f27d35b6243851b664f7da05f9f2fbb302df5c54150bebb026a9dcbbaa6aeec7fd4ef1f801b985f809c924edf5e763a89e9c741f74a6f01e654cd33901eec905dce0a88834793dedd4664b7c3844d1caf26f8c2d4d14bd732f8e2cd444ef6d2458dc624e059f01fd8f77579c0c32cd81c8feab65e77dda184c17ee72ba94648d39ea2731bc53599000a35134b7d9e035e6f4d9af8c08d073849021fdac0635a747b1001ca0bfce0ef193402c8260914e15f71ef382dd26ee9fa1ce5c626c68d94adaf4f4fa08e7e028ef2c6cb5f069341daf8319122564f064f1fc7724ef858d090c0ca212592d6c6d5e1edb98b0ef50c6c60f0a840813e09c29b3363ab7152a4df04f0e30d8271b8a881c6373c9ab08d8e1cbac20adba280ff0fb81ebeba018de2ec5209cad5d55451631eb8d6225c51aabbde82e3de12f85efdd4081b4acbe846d76c19d01578368b0625c2fa4e6f055c9efa7f5df105dc6ce605b4239c38607556bd77dfc9e0bb9fdb99ca1b775ef51e669cfbebcd01a99348dba77a36cd72e8fab4b82c5b95701a63d942bc25a2cd6a83149d1fba8cec8c96a4667193019c2c33e06f58530cd0752f62c15ed1e0dacb751fe76c621e1193602e55cc495b58c982def1e638cd795aaed4da66248fc9abcd1d5f542c1bc55235a809ccb251151b77fed448853dc715629dd274660606a5fb7b5959913c544377c0070576896dcc83ceab01b104464fdb2c5e0c4eea1e2470d889ff39c65ad1b6f4b2090e5acf4726ccdab454876364791ee19017b27cb1ddb355b77e60459773b4bf49d24bc605d187e7b2e85dcafdecd3f1a288e59ec8827a4114e780b11f2e9e634a7ad2ce93248e468fd4edd97676e9f2a5fb0a10f31fbbd099fdae09b347f0627d50966579902e9bcdb2ce1b58d7df893ce6866fabf5e688a9395edbfe608921650bfd072395074bdefa650d2f501319b9f3e1176db598dde52309947b08972493f57d401bb396eb5624493b64a02328bf7ba084b2df2144b03334bf9390c8e08cb59d9cec6c6bdea5b8e10293289e3ab0165b4415c33e51acf266d9f2a98ce9b1c4e479f03fd4aa5030d30f3559c2c42e2c67d201124e893dd9266ff2a361a4959be853d1ae8fd2d81485bdb5d46f59ac566ccaff3cc06fd56ff9ee2e2ed5a5cab2ffd70b28fbeccfec7faabef8c26f9d0cb07b7ab3325a261f37d0ff9c5761a392b16ff73f876c8a2137329b678a7a82b7f9800e0612ec827f86b5018c656cc2c3938b4072d0b6a94b3301e571160efff0ae5523b5f0ab8cf95d06be7f9e2b9abe2cb55133667fafd2f97517d5f391574f2587564349fba825314290d65032bdfac37ab421970c00410128cce6b1b64b1c1573002e8d71568b89f5233129f68ef21e57e39e01003d2dd0895f23cec22dffe4c4dbd849913a84ed9ae00099fb904c93b96a3932e0d8e1ee13592da969825329e923f0eccfa6b9d2b041d51ec66462ba2ba2638b224615c0434564eff0eace4b5c82d915a13d82cfb1e31f26cb7dccf83b46ad98391731ddb5ae59f11cb9b9cf9b732122aa61883b2c2e5f509f4ef3d444a187d8b9cebeccd6fe8d5b2191be3d226f49b24fb522ab9b9e161aa64a6fe4f45d82db18cf24e423a06f9cc3e543fa8de1b333059afaa76cbcc3e37502a990a6c51f0ddabafcda284e81246836d3be3a850e3360a578953bf4f959700a296df892ccef73f4bf05615352808308f5ceb4a3bb21e547152b75f116db4bc377e356210cfe891ed3fc4b59644fa8de12c288c83a8d080258a0a71bbce75cc11d773c1a6fc96e9be43552cb36c04e28c320338d0cd35d59dbbb3bef4c0c277572c036c965cea8fe683714133ab0e76b7f6e214cc2a339dfe6c1c0e654c830732024c2b87f7ccca8c8881bebf0600928b012eea1dc6b31e7326ac29b579a0c2c9e3b6c822d625e3d8d8eb69fe87c27e30da50806ba99ea22170cbd68d5002f14556a9090e00db22f3fd190a99353c82cd09ae9e5fdcda6f33d2e1df95d78fdbcde48db8f37dd8cf057f716b371857e68a17b20ea06651cdcfd4560a741830ca8a13ca9b7b56023235e45d538139f9f1908bc397d74fe9061eff9320d31bce04a36ec87b4c924393acb644891e9f0bec07ed4800aca08c6147dafee859e64fd2417af01f0e398239927c8ed6b752761f17c6ab12c660212572ee002caf839d7c8cc1a8516a54a76d0ebfa202f45b20d2e69c5395ce97909bc9c102b27cfa9e4aabca59d707ea99774806ead57cc1f34d9ffaa8516b4dc10b71dd6a4369513c84c71d6c7a8f03429f69d9cd17f88cd6d76865120d468b9bbc95caa1a71e925d7107434702a7a7892d99c0b38f0fc5b0ba6967f993075808f85b0e10a642c5eb5a94c35437f9df5668ff4404066a736b3ccc87c5a955bdf005eb331e087df0cd879525fae12db29199c4ec2944b19eeea6b64ede18d8c1392c6ec13294791deb4033f2a0c7eb2923080f4188118b4e21405142ada7396cccb50c3eb613db532dcd96b1b0b2da4bf2fd09ca8a35a08cec51c5ed982020c691e8ec34114e4e9b7b75e5ea916aa220e7c0da0f9777995f37233ad7084934d8700d372aab0a78fcbd824925abb7ecb0ea98a99818a01ed0eafcd78666ec1ea9526b14a9ec12017edb3ae9eb66616004e4cf9eee7378d6e6ed25c48418cf86411d4bac383c2d9229bde6eec32a2fa523bbabe39812c76aa928e62a5988bb12a9bbf5275f0322d3064ccd7f365fea641559759d1e5b5581218486318b1c776de812b1aca6a9ba6b1c6e39c5cb6d5a44e3a474f709b8eac457e74f00a43ecd3d060cc7639696bd03730c70e70abc47ee9486f217df904ddb523b87c02c55832b0c907e75b632f342140fbb0e7bb4790930e635252b4b476d1a667798a9c962de0185200a8ee623d1065d6262a7beb73ec0ab864a7250a022b85267b1a50132ffedd5f349a718175751e0ad0a1540cb354abf2dad3e0920a433e59e32a767bb35d967405b8832957dfe7ee42f4da47b95d909727816dd8e98930966443862c429cc234e68b99a66eb14214ef8d850900672719dd08334b4a9e1a5050fdf5a1782cef16797c3ba52489bb0348e3daed2321fdff5df0c77587d24a0c4ba24b77ae9e7b9d1102df8203c9c13de18042408bee73bc3cd390c43bc64e4eb6e8fe2b6062063d5fc7fbe9a711d3294bfb61cfc55d36d4fc2925c3d0fa958c252a941938080d7bb834273868c29f59eabdafd749de95ee1e5a0cf0e1efce4918ff93dd8000ffecbccfd7ead38c5444303f82f393e33f4c15eeace7d9064709f4c073f4f2dd6be4d003849d71bfb07cdf0981a13e1b9183756f8bf530deae435b25cadbeb0dc0088467c3bcbddd22682994326c92e82b38af5530b583f95289be97ff0ee302aadc5c74fbc53275389ab2780bb25cc96ab0362821b54cb6d251918f871a756c6ab4caa3c3435a184188fc1d841ed64fdeaeac60d560f5c7ac1f7675097b0cabae46518c80b3af26fbc7890d220ea9dbf5687487d235098c5e57cff0ec9d3330e3e58397daa2ac21c5bacbbf6c0e420e5d9211159ece127557af108ba71d639c8a36c34e277cdbb91de746bcc89a394775a0148291888b232669e",UInt(16))  => "82b41ecf981b19e4688f26f7ab366681",
+                    ("2649ca7df44dbafbcc09f6378abd4e0e",UInt(134)) => "7bd949ee2ac44c91585467876835322986d8633f835c9e519b245cfa0c7674e960085079574f70b4329619a2986e2b49720a4c58510a0f00873ec0d37b4e25b6909d4515ce50a63de498ee3e7ddba3eded3fab5b765734705d90bb35f116828b2675b0478f61c84e9e4b3f2a1473c62f904879130d75862c4f290d3697e5345a7393b12ae411"])
+
+
+
+# shake 256 testvectors
+SHA256test = Dict([ ("0f",UInt(32)) => "aabb07488ff9edd05d6a603b7791b60a16d45093608f1badc0c9cc9a9154f215",
+                    ("36c9970cbf6084743d076a68aa2039eb9061e1ff7a766467a9ee2fb10954aa74649c0ed7d4c277527ead3ab97d5978761d35f3154aca2107489e7f9fa45828616a24a67c98590895d35a34c659ba7fe737a37b0cbc78104e",UInt(32)) => "e6d737c11428a2b7e74de750839d0f1d90d91fa6045b6282f03508b40184ba78",  
+                    ("6969a27ad5d0aae6479b2b044bb4b043642375ff503ccb538e17be2f1e41f6aa88b1db991ffefd6087cfb20875920192b671be8b7381f7e1b33d8ff5213429f110fe475cbc74b3ecd2211f9b33f308fcf536e0d0abc36bd5e7756adefddd7728093730ec339c97313179b9e40e3f8e2a2a5c21f5836bf0d632a7961239a6a7f77b44dc700cdd70d8abbfc90c8dde5bc45dcaca2380df4e",UInt(32)) => "bcdec7a8776380df27a4613cb50b7221995d3f752fa55691798ac2dfa0b15599",
+                    ("74d7980949c1dc759a4a10acc3ab994b771ae6d8b5ef0005f8046233af610c36",UInt(2)) => "77cd",
+                    ("6ae23f058f0f2264a18cd609acc26dd4dbc00f5c3ee9e13ecaea2bb5a2f0bb6b",UInt(190)) => "b9b92544fb25cfe4ec6fe437d8da2bbe00f7bdaface3de97b8775a44d753c3adca3f7c6f183cc8647e229070439aa9539ae1f8f13470c9d3527fffdeef6c94f9f0520ff0c1ba8b16e16014e1af43ac6d94cb7929188cce9d7b02f81a2746f52ba16988e5f6d93298d778dfe05ea0ef256ae3728643ce3e29c794a0370e9ca6a8bf3e7a41e86770676ac106f7ae79e67027ce7b7b38efe27d253a52b5cb54d6eb4367a87736ed48cb45ef27f42683da140ed3295dfc575d3ea38cfc2a3697",
+                    ("e3ef127eadfafaf40408cebb28705df30b68d99dfa1893507ef3062d85461715",UInt(222)) => "7314002948c057006d4fc21e3e19c258fb5bdd57728fe93c9c6ef265b6d9f559ca73da32c427e135ba0db900d9003b19c9cf116f542a760418b1a435ac75ed5ab4ef151808c3849c3bce11c3cd285dd75e5c9fd0a0b32a89640a68e6e5b270f966f33911cfdffd03488b52b4c7fd1b2219de133e77519c426a63b9d8afac2ccab273ebd23765616b04446d6ac403f46ac0c147eda629eb7583c8bd00dc7c30fcd6711b36f99f80ac94b683ebb090581970ae7e696c4c0afa9b5dafe07d1ab80877cbd09b705a0147d62d72a506732459a54142a0892c56afb61359e910f1",
+                    ("8d8001e2c096f1b88e7c9224a086efd4797fbf74a8033a2d422a2b6b8f6747e4",UInt(250)) => "2e975f6a8a14f0704d51b13667d8195c219f71e6345696c49fa4b9d08e9225d3d39393425152c97e71dd24601c11abcfa0f12f53c680bd3ae757b8134a9c10d429615869217fdd5885c4db174985703a6d6de94a667eac3023443a8337ae1bc601b76d7d38ec3c34463105f0d3949d78e562a039e4469548b609395de5a4fd43c46ca9fd6ee29ada5efc07d84d553249450dab4a49c483ded250c9338f85cd937ae66bb436f3b4026e859fda1ca571432f3bfc09e7c03ca4d183b741111ca0483d0edabc03feb23b17ee48e844ba2408d9dcfd0139d2e8c7310125aee801c61ab7900d1efc47c078281766f361c5e6111346235e1dc38325666c"])

test/runtests.jl

@@ -141,3 +141,23 @@ end
         @test_throws MethodError f(UInt32[0x23467, 0x324775])
     end
 end
+
+@testset "SHAKE" begin
+    # test some official testvectors from https://csrc.nist.gov/Projects/Cryptographic-Algorithm-Validation-Program/Secure-Hashing
+    @testset "shake128" begin
+        for (k,v) in SHA128test
+            @test SHA.shake128(hex2bytes(k[1]),k[2]) == hex2bytes(v)
+        end
+        @test SHA.shake128(b"",UInt(16)) == hex2bytes("7f9c2ba4e88f827d616045507605853e")
+    end
+
+    @testset "shake256" begin
+        for (k,v) in SHA256test
+            @test SHA.shake256(hex2bytes(k[1]),k[2]) == hex2bytes(v)
+        end
+        @test SHA.shake256(b"",UInt(32)) == hex2bytes("46b9dd2b0ba88d13233b3feb743eeb243fcd52ea62b81b82b50c27646ed5762f")
+    end
+    @time SHA.shake256(b"abc",UInt(100000))
+    @time SHA.shake128(b"abc",UInt(100000))
+end
+