feat(credentials): store and processing generic app credentials

tests/all_tests_v2.nim

@@ -64,9 +64,11 @@ import
   # Utils
   ./v2/test_utils_keyfile
 
-
 ## Experimental
 
+import
+  ./v2/test_utils_credentials
+
 when defined(rln):
   import
     ./v2/test_waku_rln_relay,

tests/v2/test_utils_credentials.nim

@@ -0,0 +1,195 @@
+{.used.}
+
+import
+  std/[algorithm, options, os, strutils, sequtils, sets],
+  testutils/unittests, chronos, chronicles, stint, json,
+  stew/byteutils, 
+  ../../waku/v2/utils/credentials,
+  ../test_helpers,
+  ./test_utils
+
+from  ../../waku/v2/protocol/waku_noise/noise_utils import randomSeqByte
+
+procSuite "Credentials test suite":
+
+  # We initialize the RNG in test_helpers
+  let rng = rng()
+
+  asyncTest "Create keystore":
+
+    let filepath = "./testAppKeystore.txt"
+    defer: removeFile(filepath)
+
+    let keystore = createAppKeystore(path = filepath,
+                        application = "test",
+                        appIdentifier = "1234",
+                        version = "0.1")
+
+    check:
+      keystore.isOk()
+
+  asyncTest "Load keystore":
+
+    let filepath = "./testAppKeystore.txt"
+    defer: removeFile(filepath)
+
+    let keystore = loadAppKeystore(path = filepath,
+                                   application = "test",
+                                   appIdentifier = "1234",
+                                   version = "0.1")
+
+    check:
+      keystore.isOk()
+
+  asyncTest "Add credentials to keystore":
+
+    let filepath = "./testAppKeystore.txt"
+    defer: removeFile(filepath)
+
+    # We generate a random identity credential (inter-value constrains are not enforced, otherwise we need to load e.g. zerokit RLN keygen)
+    var
+      idTrapdoor = randomSeqByte(rng[], 32)
+      idNullifier =  randomSeqByte(rng[], 32)
+      idSecretHash = randomSeqByte(rng[], 32)
+      idCommitment = randomSeqByte(rng[], 32)
+
+    var idCredential = IdentityCredential(idTrapdoor: idTrapdoor, idNullifier: idNullifier, idSecretHash: idSecretHash, idCommitment: idCommitment)
+
+    var contract = MembershipContract(chainId: "5", address: "0x0123456789012345678901234567890123456789")
+    var index1 = MembershipIndex(1)
+    var membershipGroup1 = MembershipGroup(membershipContract: contract, treeIndex: index1)
+
+    let membershipCredentials1 = MembershipCredentials(identityCredential: idCredential,
+                                                       membershipGroups: @[membershipGroup1])
+
+    # We generate a random identity credential (inter-value constrains are not enforced, otherwise we need to load e.g. zerokit RLN keygen)
+    idTrapdoor = randomSeqByte(rng[], 32)
+    idNullifier =  randomSeqByte(rng[], 32)
+    idSecretHash = randomSeqByte(rng[], 32)
+    idCommitment = randomSeqByte(rng[], 32)
+
+    idCredential = IdentityCredential(idTrapdoor: idTrapdoor, idNullifier: idNullifier, idSecretHash: idSecretHash, idCommitment: idCommitment)
+
+    var index2 = MembershipIndex(2)
+    var membershipGroup2 = MembershipGroup(membershipContract: contract, treeIndex: index2)
+
+    let membershipCredentials2 = MembershipCredentials(identityCredential: idCredential,
+                                                       membershipGroups: @[membershipGroup2])
+
+    let password = "%m0um0ucoW%"
+
+    let keystore = addMembershipCredentials(path = filepath,
+                                            credentials = @[membershipCredentials1, membershipCredentials2],
+                                            password = password,
+                                            application = "test",
+                                            appIdentifier = "1234",
+                                            version = "0.1")
+
+    check:
+      keystore.isOk()
+
+  asyncTest "Add/retrieve credentials in keystore":
+
+    let filepath = "./testAppKeystore.txt"
+    defer: removeFile(filepath)
+
+    # We generate two random identity credentials (inter-value constrains are not enforced, otherwise we need to load e.g. zerokit RLN keygen)
+    var
+      idTrapdoor1 = randomSeqByte(rng[], 32)
+      idNullifier1 =  randomSeqByte(rng[], 32)
+      idSecretHash1 = randomSeqByte(rng[], 32)
+      idCommitment1 = randomSeqByte(rng[], 32)
+      idCredential1 = IdentityCredential(idTrapdoor: idTrapdoor1, idNullifier: idNullifier1, idSecretHash: idSecretHash1, idCommitment: idCommitment1)
+
+    var
+      idTrapdoor2 = randomSeqByte(rng[], 32)
+      idNullifier2 =  randomSeqByte(rng[], 32)
+      idSecretHash2 = randomSeqByte(rng[], 32)
+      idCommitment2 = randomSeqByte(rng[], 32)
+      idCredential2 = IdentityCredential(idTrapdoor: idTrapdoor2, idNullifier: idNullifier2, idSecretHash: idSecretHash2, idCommitment: idCommitment2)
+
+    # We generate two distinct membership groups
+    var contract1 = MembershipContract(chainId: "5", address: "0x0123456789012345678901234567890123456789")
+    var index1 = MembershipIndex(1)
+    var membershipGroup1 = MembershipGroup(membershipContract: contract1, treeIndex: index1)
+
+    var contract2 = MembershipContract(chainId: "6", address: "0x0000000000000000000000000000000000000000")
+    var index2 = MembershipIndex(2)
+    var membershipGroup2 = MembershipGroup(membershipContract: contract2, treeIndex: index2)
+
+    # We generate three membership credentials
+    let membershipCredentials1 = MembershipCredentials(identityCredential: idCredential1,
+                                                       membershipGroups: @[membershipGroup1])
+
+    let membershipCredentials2 = MembershipCredentials(identityCredential: idCredential2,
+                                                       membershipGroups: @[membershipGroup2])
+
+    let membershipCredentials3 = MembershipCredentials(identityCredential: idCredential1,
+                                                       membershipGroups: @[membershipGroup2])
+
+    # This is the same as rlnMembershipCredentials3, should not change the keystore entry of idCredential
+    let membershipCredentials4 = MembershipCredentials(identityCredential: idCredential1,
+                                                       membershipGroups: @[membershipGroup2])
+
+    let password = "%m0um0ucoW%"
+
+    # We add credentials to the keystore. Note that only 3 credentials should be effectively added, since rlnMembershipCredentials3 is equal to membershipCredentials2
+    let keystore = addMembershipCredentials(path = filepath,
+                                            credentials = @[membershipCredentials1, membershipCredentials2, membershipCredentials3, membershipCredentials4],
+                                            password = password,
+                                            application = "test",
+                                            appIdentifier = "1234",
+                                            version = "0.1")
+
+    check:
+      keystore.isOk()
+
+    # We test retrieval of credentials.
+    var expectedMembershipGroups1 = @[membershipGroup1, membershipGroup2]          
+    expectedMembershipGroups1.sort(sortMembershipGroup)
+    let expectedCredential1 = MembershipCredentials(identityCredential: idCredential1,
+                                                    membershipGroups: expectedMembershipGroups1)
+
+
+    var expectedMembershipGroups2 = @[membershipGroup2]          
+    expectedMembershipGroups2.sort(sortMembershipGroup)
+    let expectedCredential2 = MembershipCredentials(identityCredential: idCredential2,
+                                                    membershipGroups: expectedMembershipGroups2)
+
+
+    # We retrieve all credentials stored under password (no filter)
+    var recoveredCredentials = getMembershipCredentials(path = filepath,
+                                                        password = password,
+                                                        application = "test",
+                                                        appIdentifier = "1234",
+                                                        version = "0.1")
+
+    check:
+      recoveredCredentials.isOk()
+      recoveredCredentials.get() == @[expectedCredential1, expectedCredential2]
+
+
+    # We retrieve credentials by filtering on an IdentityCredential
+    recoveredCredentials = getMembershipCredentials(path = filepath,
+                                                    password = password,
+                                                    filterIdentityCredentials = @[idCredential1],
+                                                    application = "test",
+                                                    appIdentifier = "1234",
+                                                    version = "0.1")
+
+    check:
+      recoveredCredentials.isOk()
+      recoveredCredentials.get() == @[expectedCredential1]
+
+    # We retrieve credentials by filtering on multiple IdentityCredentials
+    recoveredCredentials = getMembershipCredentials(path = filepath,
+                                                    password = password,
+                                                    filterIdentityCredentials = @[idCredential1, idCredential2],
+                                                    application = "test",
+                                                    appIdentifier = "1234",
+                                                    version = "0.1")
+
+    check:  
+      recoveredCredentials.isOk()
+      recoveredCredentials.get() == @[expectedCredential1, expectedCredential2]
+

tests/v2/test_waku_rln_relay.nim

@@ -14,12 +14,14 @@ import
   ../../waku/v2/node/waku_node,
   ../../waku/v2/protocol/waku_message,
   ../../waku/v2/protocol/waku_rln_relay,
+  ../../waku/v2/utils/credentials,
   ../test_helpers
 
 const RlnRelayPubsubTopic = "waku/2/rlnrelay/proto"
 const RlnRelayContentTopic = "waku/2/rlnrelay/proto"
 
 procSuite "Waku rln relay":
+
   asyncTest "mount waku-rln-relay in the off-chain mode":
     let
       nodeKey = crypto.PrivateKey.random(Secp256k1, rng[])[]
@@ -1041,10 +1043,11 @@ suite "Waku rln relay":
 
     debug "the generated identity credential: ", idCredential
 
-    let index =  MembershipIndex(1)
+    let index = MembershipIndex(1)
 
-    let rlnMembershipCredentials = RlnMembershipCredentials(identityCredential: idCredential, 
-                                                            rlnIndex: index)
+    let rlnMembershipContract = MembershipContract(chainId: "5", address: "0x0123456789012345678901234567890123456789")
+    let rlnMembershipGroup = MembershipGroup(membershipContract: rlnMembershipContract, treeIndex: index)
+    let rlnMembershipCredentials = MembershipCredentials(identityCredential: idCredential, membershipGroups: @[rlnMembershipGroup])
 
     let password = "%m0um0ucoW%"
 
@@ -1053,19 +1056,35 @@ suite "Waku rln relay":
 
     # Write RLN credentials
     require:
-      writeRlnCredentials(filepath, rlnMembershipCredentials, password).isOk()
+      addMembershipCredentials(path = filepath,
+                                credentials = @[rlnMembershipCredentials],
+                                password = password,
+                                application = RLNKeystoreApplication,
+                                appIdentifier = RLNKeystoreAppIdentifier,
+                                version = RLNKeystoreVersion).isOk()
+
+    let readCredentialsResult = getMembershipCredentials(path = filepath,
+                                                         password = password,
+                                                         filterMembershipContracts = @[rlnMembershipContract],
+                                                         application = RLNKeystoreApplication,
+                                                         appIdentifier = RLNKeystoreAppIdentifier,
+                                                         version = RLNKeystoreVersion)
 
-    let readCredentialsResult = readRlnCredentials(filepath, password)
     require:
       readCredentialsResult.isOk()
 
-    let credentials = readCredentialsResult.get()
+    # getMembershipCredentials returns all credentials in keystore as sequence matching the filter
+    let allMatchingCredentials = readCredentialsResult.get()
+    # if any is found, we return the first credential, otherwise credentials is none 
+    var credentials = none(MembershipCredentials)
+    if allMatchingCredentials.len() > 0:
+      credentials = some(allMatchingCredentials[0])
 
     require:
       credentials.isSome()
     check:
       credentials.get().identityCredential == idCredential
-      credentials.get().rlnIndex == index
+      credentials.get().membershipGroups == @[rlnMembershipGroup]
 
   test "histogram static bucket generation":
     let buckets = generateBucketsForHistogram(10)

tests/v2/test_waku_rln_relay_onchain.nim

@@ -10,6 +10,7 @@ import
   eth/keys,
   ../../waku/v2/protocol/waku_rln_relay,
   ../../waku/v2/node/waku_node,
+  ../../waku/v2/utils/credentials,
   ../test_helpers,
   ./test_utils
 

tests/v2/test_wakunode_rln_relay.nim

@@ -17,6 +17,7 @@ import
   ../../waku/v2/node/waku_node,
   ../../waku/v2/protocol/waku_message,
   ../../waku/v2/protocol/waku_rln_relay,
+  ../../waku/v2/utils/credentials,
   ../../waku/v2/utils/peers
 
 from std/times import epochTime

waku/v2/node/waku_node.nim

@@ -285,7 +285,7 @@ proc info*(node: WakuNode): WakuInfo =
 proc connectToNodes*(node: WakuNode, nodes: seq[RemotePeerInfo] | seq[string], source = "api") {.async.} =
   ## `source` indicates source of node addrs (static config, api call, discovery, etc)
   # NOTE This is dialing on WakuRelay protocol specifically
-  await connectToNodes(node.peerManager, nodes, WakuRelayCodec, source=source)
+  await peer_manager.connectToNodes(node.peerManager, nodes, WakuRelayCodec, source=source)
 
 
 ## Waku relay

waku/v2/protocol/waku_rln_relay/constants.nim

@@ -48,3 +48,11 @@ const MaxClockGapSeconds* = 20.0 # the maximum clock difference between peers in
 
 # maximum allowed gap between the epochs of messages' RateLimitProofs
 const MaxEpochGap* = uint64(MaxClockGapSeconds/EpochUnitSeconds)
+
+# RLN Keystore defaults
+const 
+  RLNKeystoreApplication* = "nwaku-rln-relay"
+  RLNKeystoreAppIdentifier* = "01234567890abcdef"
+  RLNKeystoreVersion* = "0.1"
+  # NOTE: 256-bytes long credentials are due to the use of BN254 in RLN. Other implementations/curves might have a different byte size
+  CredentialByteSize* = 256

waku/v2/protocol/waku_rln_relay/conversion_utils.nim

@@ -10,9 +10,11 @@ import
   stew/[arrayops, results, endians2],
   stint
 import
+    ./constants,
     ./protocol_types
 import
-  ../../utils/keyfile
+  ../../utils/keyfile,
+  ../../utils/credentials
 
 export
   web3,
@@ -27,16 +29,9 @@ proc toUInt256*(idCommitment: IDCommitment): UInt256 =
   return pk
 
 proc toIDCommitment*(idCommitmentUint: UInt256): IDCommitment =
-  let pk = IDCommitment(idCommitmentUint.toBytesLE())
+  let pk = IDCommitment(@(idCommitmentUint.toBytesLE()))
   return pk
 
-proc inHex*(value: array[32, byte]): string =
-  var valueHex = (UInt256.fromBytesLE(value)).toHex()
-  # We pad leading zeroes
-  while valueHex.len < value.len * 2:
-    valueHex = "0" & valueHex
-  return valueHex
-
 proc toMembershipIndex*(v: UInt256): MembershipIndex =
   let membershipIndex: MembershipIndex = cast[MembershipIndex](v)
   return membershipIndex
@@ -115,10 +110,10 @@ proc toIdentityCredentials*(groupKeys: seq[(string, string, string, string)]): R
   for i in 0..groupKeys.len-1:
     try:
       let
-        idTrapdoor = hexToUint[IdentityTrapdoor.len*8](groupKeys[i][0]).toBytesLE()
-        idNullifier = hexToUint[IdentityNullifier.len*8](groupKeys[i][1]).toBytesLE()
-        idSecretHash = hexToUint[IdentitySecretHash.len*8](groupKeys[i][2]).toBytesLE()
-        idCommitment = hexToUint[IDCommitment.len*8](groupKeys[i][3]).toBytesLE()
+        idTrapdoor = IdentityTrapdoor(@(hexToUint[CredentialByteSize](groupKeys[i][0]).toBytesLE()))
+        idNullifier = IdentityNullifier(@(hexToUint[CredentialByteSize](groupKeys[i][1]).toBytesLE()))
+        idSecretHash = IdentitySecretHash(@(hexToUint[CredentialByteSize](groupKeys[i][2]).toBytesLE()))
+        idCommitment = IDCommitment(@(hexToUint[CredentialByteSize](groupKeys[i][3]).toBytesLE()))
       groupIdCredentials.add(IdentityCredential(idTrapdoor: idTrapdoor, idNullifier: idNullifier, idSecretHash: idSecretHash,
           idCommitment: idCommitment))
     except ValueError as err:
@@ -138,8 +133,8 @@ proc toIdentityCredentials*(groupKeys: seq[(string, string)]): RlnRelayResult[se
   for i in 0..groupKeys.len-1:
     try:
       let
-        idSecretHash = hexToUint[IdentitySecretHash.len*8](groupKeys[i][0]).toBytesLE()
-        idCommitment = hexToUint[IDCommitment.len*8](groupKeys[i][1]).toBytesLE()
+        idSecretHash = IdentitySecretHash(@(hexToUint[CredentialByteSize](groupKeys[i][0]).toBytesLE()))
+        idCommitment = IDCommitment(@(hexToUint[CredentialByteSize](groupKeys[i][1]).toBytesLE()))
       groupIdCredentials.add(IdentityCredential(idSecretHash: idSecretHash,
           idCommitment: idCommitment))
     except ValueError as err: