Merge branch 'master' into resultify-waku-rln-relay

.github/workflows/auto_assign_pr.yml

@@ -0,0 +1,12 @@
+name: Auto Assign PR to Creator
+
+on:
+  pull_request:
+    types:
+      - opened
+
+jobs:
+  assign_creator:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: toshimaru/[email protected]

CHANGELOG.md

@@ -1,3 +1,80 @@
+## v0.27.0 (2024-04-19)
+
+>  **Note:**
+
+>  - Filter v1 protocol and its REST-API access have been deprecated.
+>  - A new field of the `WakuMetadataRequest` protobuf for shards was introduced. The old shards field (2) will be deprecated in 2 releases time
+>  - CLI flags `--requestRateLimit` and `--requestRatePeriod` have been added for rate limiting configuration. Period is measured in seconds. Limits are measured per protocol per period of time. Over limit will result in TOO_MANY_REQUEST (429) response.
+
+## What's Changed
+
+Release highlights:
+
+* Introduced configurable rate limiting for lightpush and store requests
+* Sync time has been considerably reduced for node initialization
+* Significant refactors were made to node initialization and `WakuArchive` logic as work towards C-bindings and Store V3 features
+
+### Features
+
+- Added simple, configurable rate limit for lightpush and store-query ([#2390](https://github.com/waku-org/nwaku/issues/2390)) ([a00f350c](https://github.com/waku-org/nwaku/commit/a00f350c))
+- examples/golang/waku.go add new example ([#2559](https://github.com/waku-org/nwaku/issues/2559)) ([8d66a548](https://github.com/waku-org/nwaku/commit/8d66a548))
+- **c-bindings:** rln relay ([#2544](https://github.com/waku-org/nwaku/issues/2544)) ([2aa835e3](https://github.com/waku-org/nwaku/commit/2aa835e3))
+- **incentivization:** add codec for eligibility proof and status ([#2419](https://github.com/waku-org/nwaku/issues/2419)) ([65530264](https://github.com/waku-org/nwaku/commit/65530264))
+- **rest:** add support to ephemeral field ([#2525](https://github.com/waku-org/nwaku/issues/2525)) ([c734f60d](https://github.com/waku-org/nwaku/commit/c734f60d))
+- archive update for store v3 ([#2451](https://github.com/waku-org/nwaku/issues/2451)) ([505479b8](https://github.com/waku-org/nwaku/commit/505479b8))
+- **c-bindings:** add function to dealloc nodes ([#2499](https://github.com/waku-org/nwaku/issues/2499)) ([8341864d](https://github.com/waku-org/nwaku/commit/8341864d))
+
+### Bug Fixes
+
+- **rln-relay:** reduce sync time ([#2577](https://github.com/waku-org/nwaku/issues/2577)) ([480a62fa](https://github.com/waku-org/nwaku/commit/480a62fa))
+- rest store: content_topic -> contentTopic in the response ([#2584](https://github.com/waku-org/nwaku/issues/2584)) ([d2578553](https://github.com/waku-org/nwaku/commit/d2578553))
+- **c-bindings:** rln credential path key ([#2564](https://github.com/waku-org/nwaku/issues/2564)) ([3d752b11](https://github.com/waku-org/nwaku/commit/3d752b11))
+- cluster-id 0 disc5 issue ([#2562](https://github.com/waku-org/nwaku/issues/2562)) ([a76c9587](https://github.com/waku-org/nwaku/commit/a76c9587))
+- regex for rpc endpoint ([#2563](https://github.com/waku-org/nwaku/issues/2563)) ([c87545d5](https://github.com/waku-org/nwaku/commit/c87545d5))
+- **rln:** set a minimum epoch gap ([#2555](https://github.com/waku-org/nwaku/issues/2555)) ([b5e4795f](https://github.com/waku-org/nwaku/commit/b5e4795f))
+- fix regresion + remove deprecated flag ([#2556](https://github.com/waku-org/nwaku/issues/2556)) ([47ad0fb0](https://github.com/waku-org/nwaku/commit/47ad0fb0))
+- **networkmanager:** regularly disconnect from random peers ([#2553](https://github.com/waku-org/nwaku/issues/2553)) ([70c53fc0](https://github.com/waku-org/nwaku/commit/70c53fc0))
+- remove subscription queue limit ([#2551](https://github.com/waku-org/nwaku/issues/2551)) ([94ff5eab](https://github.com/waku-org/nwaku/commit/94ff5eab))
+- peer_manager - extend the number of connection requests to known peers ([#2534](https://github.com/waku-org/nwaku/issues/2534)) ([2173fe22](https://github.com/waku-org/nwaku/commit/2173fe22))
+- **2491:** Fix metadata protocol disconnecting light nodes ([#2533](https://github.com/waku-org/nwaku/issues/2533)) ([33774fad](https://github.com/waku-org/nwaku/commit/33774fad))
+- **rest:** filter/v2/subscriptions response ([#2529](https://github.com/waku-org/nwaku/issues/2529)) ([7aea2d4f](https://github.com/waku-org/nwaku/commit/7aea2d4f))
+- **store:** retention policy regex ([#2532](https://github.com/waku-org/nwaku/issues/2532)) ([23a291b3](https://github.com/waku-org/nwaku/commit/23a291b3))
+- enable autosharding in any cluster ([#2505](https://github.com/waku-org/nwaku/issues/2505)) ([5a225809](https://github.com/waku-org/nwaku/commit/5a225809))
+- introduce new field for shards in metadata protocol ([#2511](https://github.com/waku-org/nwaku/issues/2511)) ([f9f92b7d](https://github.com/waku-org/nwaku/commit/f9f92b7d))
+- **rln-relay:** handle empty metadata returned by getMetadata proc ([#2516](https://github.com/waku-org/nwaku/issues/2516)) ([1274b15d](https://github.com/waku-org/nwaku/commit/1274b15d))
+
+### Changes
+
+- adding migration script adding i_query index ([#2578](https://github.com/waku-org/nwaku/issues/2578)) ([4117fe65](https://github.com/waku-org/nwaku/commit/4117fe65))
+- bumping chronicles version ([#2583](https://github.com/waku-org/nwaku/issues/2583)) ([a04e0d99](https://github.com/waku-org/nwaku/commit/a04e0d99))
+- add ARM64 support for Linux/MacOS ([#2580](https://github.com/waku-org/nwaku/issues/2580)) ([269139cf](https://github.com/waku-org/nwaku/commit/269139cf))
+- **rln:** update submodule + rln patch version ([#2574](https://github.com/waku-org/nwaku/issues/2574)) ([24f6fed8](https://github.com/waku-org/nwaku/commit/24f6fed8))
+- bumping dependencies for 0.27.0 ([#2572](https://github.com/waku-org/nwaku/issues/2572)) ([f68ac792](https://github.com/waku-org/nwaku/commit/f68ac792))
+- **c-bindings:** node initialization ([#2547](https://github.com/waku-org/nwaku/issues/2547)) ([6d0f6d82](https://github.com/waku-org/nwaku/commit/6d0f6d82))
+- remove deprecated legacy filter protocol ([#2507](https://github.com/waku-org/nwaku/issues/2507)) ([e8613172](https://github.com/waku-org/nwaku/commit/e8613172))
+- switch wakuv2 to waku fleet ([#2519](https://github.com/waku-org/nwaku/issues/2519)) ([18a05359](https://github.com/waku-org/nwaku/commit/18a05359))
+- create nph.md ([#2536](https://github.com/waku-org/nwaku/issues/2536)) ([a576e624](https://github.com/waku-org/nwaku/commit/a576e624))
+- Better postgres duplicate insert ([#2535](https://github.com/waku-org/nwaku/issues/2535)) ([693a1778](https://github.com/waku-org/nwaku/commit/693a1778))
+- add 150 kB to msg size histogram metric ([#2430](https://github.com/waku-org/nwaku/issues/2430)) ([2c1391d3](https://github.com/waku-org/nwaku/commit/2c1391d3))
+- content_script_version_2: add simple protection and rename messages_backup if exists ([#2531](https://github.com/waku-org/nwaku/issues/2531)) ([c6c376b5](https://github.com/waku-org/nwaku/commit/c6c376b5))
+- **vendor:** update nim-libp2p path ([#2527](https://github.com/waku-org/nwaku/issues/2527)) ([3c823756](https://github.com/waku-org/nwaku/commit/3c823756))
+- adding node factory tests ([#2524](https://github.com/waku-org/nwaku/issues/2524)) ([a1b3e090](https://github.com/waku-org/nwaku/commit/a1b3e090))
+- factory cleanup ([#2523](https://github.com/waku-org/nwaku/issues/2523)) ([8d7eb3a6](https://github.com/waku-org/nwaku/commit/8d7eb3a6))
+- **rln-relay-v2:** wakunode testing + improvements ([#2501](https://github.com/waku-org/nwaku/issues/2501)) ([059cb975](https://github.com/waku-org/nwaku/commit/059cb975))
+- update CHANGELOG for v0.26.0 release ([#2518](https://github.com/waku-org/nwaku/issues/2518)) ([097cb362](https://github.com/waku-org/nwaku/commit/097cb362))
+- migrating logic from wakunode2.nim to node_factory.nim ([#2504](https://github.com/waku-org/nwaku/issues/2504)) ([dcc88ee0](https://github.com/waku-org/nwaku/commit/dcc88ee0))
+
+This release supports the following [libp2p protocols](https://docs.libp2p.io/concepts/protocols/):
+| Protocol | Spec status | Protocol id |
+| ---: | :---: | :--- |
+| [`11/WAKU2-RELAY`](https://rfc.vac.dev/spec/11/) | `stable` | `/vac/waku/relay/2.0.0` |
+| [`12/WAKU2-FILTER`](https://rfc.vac.dev/spec/12/) | `draft` | `/vac/waku/filter/2.0.0-beta1` <br />`/vac/waku/filter-subscribe/2.0.0-beta1` <br />`/vac/waku/filter-push/2.0.0-beta1` |
+| [`13/WAKU2-STORE`](https://rfc.vac.dev/spec/13/) | `draft` | `/vac/waku/store/2.0.0-beta4` |
+| [`19/WAKU2-LIGHTPUSH`](https://rfc.vac.dev/spec/19/) | `draft` | `/vac/waku/lightpush/2.0.0-beta1` |
+| [`66/WAKU2-METADATA`](https://rfc.vac.dev/spec/66/) | `raw` | `/vac/waku/metadata/1.0.0` |
+
+The Waku v1 implementation has been removed from this repository and can be found in a separate [Waku Legacy](https://github.com/waku-org/waku-legacy) repository.
+
 ## v0.26.0 (2024-03-07)
 
 > **Note:**

apps/chat2/chat2.nim

@@ -39,7 +39,7 @@ import
   ../../waku/waku_lightpush/rpc,
   ../../waku/waku_enr,
   ../../waku/waku_store,
-  ../../waku/waku_dnsdisc,
+  ../../waku/discovery/waku_dnsdisc,
   ../../waku/waku_node,
   ../../waku/node/waku_metrics,
   ../../waku/node/peer_manager,
@@ -523,7 +523,9 @@ proc processInput(rfd: AsyncFD, rng: ref HmacDrbgContext) {.async.} =
       if msg.contentTopic == chat.contentTopic:
         chat.printReceivedMessage(msg)
 
-    node.subscribe((kind: PubsubSub, topic: DefaultPubsubTopic), some(WakuRelayHandler(handler)))
+    node.subscribe(
+      (kind: PubsubSub, topic: DefaultPubsubTopic), some(WakuRelayHandler(handler))
+    )
 
     if conf.rlnRelay:
       info "WakuRLNRelay is enabled"

apps/networkmonitor/networkmonitor.nim

@@ -25,12 +25,12 @@ import
   ../../waku/node/peer_manager,
   ../../waku/waku_node,
   ../../waku/waku_enr,
-  ../../waku/waku_discv5,
-  ../../waku/waku_dnsdisc,
+  ../../waku/discovery/waku_discv5,
+  ../../waku/discovery/waku_dnsdisc,
   ../../waku/waku_relay,
   ../../waku/waku_rln_relay,
   ../../waku/factory/builder,
-  ../wakunode2/networks_config,
+  ../../waku/factory/networks_config,
   ./networkmonitor_metrics,
   ./networkmonitor_config,
   ./networkmonitor_utils
@@ -131,12 +131,12 @@ proc setConnectedPeersMetrics(
   var analyzeFuts: seq[Future[Result[string, string]]]
 
   var (inConns, outConns) = node.peer_manager.connectedPeers(WakuRelayCodec)
-  info "connected peers", inConns=inConns.len, outConns=outConns.len
+  info "connected peers", inConns = inConns.len, outConns = outConns.len
 
   shuffle(outConns)
 
-  if outConns.len >= toInt(MaxConnectedPeers/2):
-    for p in outConns[0 ..< toInt(outConns.len/2)]:
+  if outConns.len >= toInt(MaxConnectedPeers / 2):
+    for p in outConns[0 ..< toInt(outConns.len / 2)]:
       trace "Pruning Peer", Peer = $p
       asyncSpawn(node.switch.disconnect(p))
 

apps/wakunode2/wakunode2.nim

@@ -17,8 +17,8 @@ import
   ../../tools/rln_db_inspector/rln_db_inspector,
   ../../waku/common/logging,
   ../../waku/factory/external_config,
-  ./networks_config,
-  ./app
+  ../../waku/factory/networks_config,
+  ../../waku/factory/app
 
 logScope:
   topics = "wakunode main"

docs/contributors/nph.md

@@ -8,6 +8,14 @@ https://marketplace.visualstudio.com/items?itemName=arnetheduck.vscode-nph
 ### GitHub
 https://github.com/arnetheduck/nph
 
+Make sure you use a binary from the following release:
+https://github.com/arnetheduck/nph/releases/tag/v0.5.1
+
+```bash
+$ nph --version
+v0.5.1-0-gde5cd48
+```
+
 ### Installation and configuration
 1. Ask the [nwaku team](https://discord.com/channels/1110799176264056863/1111541184490393691) about the required `nph` version.
 2. Download the desired release from _GitHub_ and place the binary in the PATH env var.

examples/publisher.nim

@@ -15,7 +15,7 @@ import
   ../../../waku/waku_core,
   ../../../waku/waku_node,
   ../../../waku/waku_enr,
-  ../../../waku/waku_discv5,
+  ../../../waku/discovery/waku_discv5,
   ../../../waku/factory/builder
 
 proc now*(): Timestamp =

examples/subscriber.nim

@@ -15,7 +15,7 @@ import
   ../../../waku/waku_core,
   ../../../waku/waku_node,
   ../../../waku/waku_enr,
-  ../../../waku/waku_discv5,
+  ../../../waku/discovery/waku_discv5,
   ../../../waku/factory/builder,
   ../../../waku/waku_relay
 

library/waku_thread/inter_thread_communication/requests/node_lifecycle_request.nim

@@ -22,7 +22,7 @@ import
   ../../../../waku/waku_store,
   ../../../../waku/factory/builder,
   ../../../../waku/factory/node_factory,
-  ../../../apps/wakunode2/networks_config,
+  ../../../../waku/factory/networks_config,
   ../../../events/[json_message_event, json_base_event],
   ../../../alloc,
   ../../config

tests/node/test_wakunode_peer_exchange.nim

@@ -15,7 +15,7 @@ import
 import
   ../../../waku/[
     waku_node,
-    waku_discv5,
+    discovery/waku_discv5,
     waku_peer_exchange,
     node/peer_manager,
     waku_relay/protocol,

tests/test_waku_dnsdisc.nim

@@ -13,7 +13,7 @@ import
 import
   ../../waku/node/peer_manager,
   ../../waku/waku_node,
-  ../../waku/waku_dnsdisc,
+  ../../waku/discovery/waku_dnsdisc,
   ./testlib/common,
   ./testlib/wakucore,
   ./testlib/wakunode

tests/test_waku_metadata.nim

@@ -17,7 +17,7 @@ import
   ../../waku/waku_node,
   ../../waku/waku_core/topics,
   ../../waku/node/peer_manager,
-  ../../waku/waku_discv5,
+  ../../waku/discovery/waku_discv5,
   ../../waku/waku_metadata,
   ./testlib/wakucore,
   ./testlib/wakunode

tests/testlib/wakunode.nim

@@ -12,7 +12,7 @@ import
   ../../../waku/waku_node,
   ../../../waku/node/peer_manager,
   ../../../waku/waku_enr,
-  ../../../waku/waku_discv5,
+  ../../../waku/discovery/waku_discv5,
   ../../../waku/factory/external_config,
   ../../../waku/factory/internal_config,
   ../../../waku/factory/builder,

tests/waku_discv5/test_waku_discv5.nim

@@ -11,7 +11,7 @@ import
   eth/keys as eth_keys
 
 import
-  ../../../waku/[waku_core/topics, waku_enr, waku_discv5, common/enr],
+  ../../../waku/[waku_core/topics, waku_enr, discovery/waku_discv5, common/enr],
   ../testlib/[wakucore, testasync, assertions, futures],
   ../waku_enr/utils,
   ./utils

tests/waku_discv5/utils.nim

@@ -6,7 +6,8 @@ import
   eth/keys as eth_keys
 
 import
-  ../../../waku/[waku_core/topics, waku_enr, waku_discv5], ../testlib/[common, wakucore]
+  ../../../waku/[waku_core/topics, waku_enr, discovery/waku_discv5],
+  ../testlib/[common, wakucore]
 
 proc newTestDiscv5*(
     privKey: libp2p_keys.PrivateKey,

tests/waku_enr/test_sharding.nim

@@ -9,7 +9,7 @@ import
   eth/keys as eth_keys
 
 import
-  ../../../waku/[waku_enr, waku_discv5, waku_core],
+  ../../../waku/[waku_enr, discovery/waku_discv5, waku_core],
   ../testlib/wakucore,
   ../waku_discv5/utils,
   ./utils

tests/waku_enr/utils.nim

@@ -7,7 +7,7 @@ import
   eth/keys as eth_keys
 
 import
-  ../../../waku/[waku_core/topics, waku_enr, waku_discv5, waku_enr/sharding],
+  ../../../waku/[waku_core/topics, waku_enr, discovery/waku_discv5, waku_enr/sharding],
   ../testlib/[common, wakucore]
 
 proc newTestEnrRecord*(

tests/waku_filter_v2/test_waku_client.nim

@@ -126,8 +126,9 @@ suite "Waku Filter - End to End":
 
       asyncTest "Subscribing to an empty content topic":
         # When subscribing to an empty content topic
-        let subscribeResponse =
-          await wakuFilterClient.subscribe(serverRemotePeerInfo, pubsubTopic, newSeq[ContentTopic]())
+        let subscribeResponse = await wakuFilterClient.subscribe(
+          serverRemotePeerInfo, pubsubTopic, newSeq[ContentTopic]()
+        )
 
         # Then the subscription is not successful
         check:
@@ -1838,8 +1839,9 @@ suite "Waku Filter - End to End":
           wakuFilter.subscriptions.isSubscribed(clientPeerId)
 
         # When unsubscribing from an empty content topic
-        let unsubscribeResponse =
-          await wakuFilterClient.unsubscribe(serverRemotePeerInfo, pubsubTopic, newSeq[ContentTopic]())
+        let unsubscribeResponse = await wakuFilterClient.unsubscribe(
+          serverRemotePeerInfo, pubsubTopic, newSeq[ContentTopic]()
+        )
 
         # Then the unsubscription is not successful
         check:
@@ -2076,10 +2078,11 @@ suite "Waku Filter - End to End":
             contentTopic = contentTopic, payload = getByteSequence(100 * 1024)
           ) # 100KiB
           msg4 = fakeWakuMessage(
-            contentTopic = contentTopic, payload = getByteSequence(MaxPushSize - 1024)
+            contentTopic = contentTopic,
+            payload = getByteSequence(DefaultMaxPushSize - 1024),
           ) # Max Size (Inclusive Limit)
           msg5 = fakeWakuMessage(
-            contentTopic = contentTopic, payload = getByteSequence(MaxPushSize)
+            contentTopic = contentTopic, payload = getByteSequence(DefaultMaxPushSize)
           ) # Max Size (Exclusive Limit)
 
         # When sending the 1KiB message

tests/waku_lightpush/test_client.nim

@@ -43,6 +43,9 @@ suite "Waku Lightpush Client":
     handler = proc(
         peer: PeerId, pubsubTopic: PubsubTopic, message: WakuMessage
     ): Future[WakuLightPushResult[void]] {.async.} =
+      let msgLen = message.encode().buffer.len
+      if msgLen > int(DefaultMaxWakuMessageSize) + 64 * 1024:
+        return err("length greater than maxMessageSize")
       handlerFuture.complete((pubsubTopic, message))
       return ok()
 
@@ -209,11 +212,11 @@ suite "Waku Lightpush Client":
         ) # 100KiB
         message4 = fakeWakuMessage(
           contentTopic = contentTopic,
-          payload = getByteSequence(MaxRpcSize - overheadBytes - 1),
+          payload = getByteSequence(DefaultMaxWakuMessageSize - overheadBytes - 1),
         ) # Inclusive Limit
         message5 = fakeWakuMessage(
           contentTopic = contentTopic,
-          payload = getByteSequence(MaxRpcSize - overheadBytes),
+          payload = getByteSequence(DefaultMaxWakuMessageSize + 64 * 1024),
         ) # Exclusive Limit
 
       # When publishing the 1KiB payload

tests/waku_peer_exchange/test_protocol.nim

@@ -13,7 +13,7 @@ import
   ../../../waku/[
     waku_node,
     node/peer_manager,
-    waku_discv5,
+    discovery/waku_discv5,
     waku_peer_exchange,
     waku_peer_exchange/rpc,
     waku_peer_exchange/rpc_codec,
@@ -354,7 +354,7 @@ suite "Waku Peer Exchange":
 
       var buffer: seq[byte]
       await conn.writeLP(rpc.encode().buffer)
-      buffer = await conn.readLp(MaxRpcSize.int)
+      buffer = await conn.readLp(DefaultMaxRpcSize.int)
 
       # Decode the response
       let decodedBuff = PeerExchangeRpc.decode(buffer)

tests/waku_peer_exchange/test_rpc_codec.nim

@@ -12,13 +12,12 @@ import
   eth/p2p/discoveryv5/enr
 
 import
-  ../../../waku/
-    [
-      node/peer_manager,
-      waku_discv5,
-      waku_peer_exchange/rpc,
-      waku_peer_exchange/rpc_codec,
-    ],
+  ../../../waku/[
+    node/peer_manager,
+    discovery/waku_discv5,
+    waku_peer_exchange/rpc,
+    waku_peer_exchange/rpc_codec,
+  ],
   ../testlib/[wakucore]
 
 suite "Peer Exchange RPC":

tests/waku_peer_exchange/utils.nim

@@ -13,7 +13,7 @@ import
 import
   ../../../waku/[
     waku_node,
-    waku_discv5,
+    discovery/waku_discv5,
     waku_peer_exchange,
     waku_peer_exchange/rpc,
     waku_peer_exchange/protocol,