From 34c96b9abfb78b0040b67d59071f1fa17b5fd606 Mon Sep 17 00:00:00 2001 From: Richard Ramos Date: Mon, 9 Dec 2024 18:07:04 -0400 Subject: [PATCH 1/3] fix(libwaku): support string and int64 for timestamps --- library/events/json_message_event.nim | 12 ++++++++++-- .../requests/protocols/store_request.nim | 14 ++++++++++---- 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/library/events/json_message_event.nim b/library/events/json_message_event.nim index 04dc0b8b4a..3910598e8c 100644 --- a/library/events/json_message_event.nim +++ b/library/events/json_message_event.nim @@ -1,4 +1,4 @@ -import system, results, std/json +import system, results, std/json, std/strutils import stew/byteutils import ../../waku/common/base64, @@ -21,7 +21,15 @@ func fromJsonNode*(T: type JsonMessage, jsonContent: JsonNode): JsonMessage = payload: Base64String(jsonContent["payload"].getStr()), contentTopic: jsonContent["contentTopic"].getStr(), version: uint32(jsonContent{"version"}.getInt()), - timestamp: int64(jsonContent{"timestamp"}.getBiggestInt()), + timestamp: + if jsonContent.hasKey("timestamp"): + if jsonContent["timestamp"].kind == JString: + parseInt(jsonContent["timestamp"].getStr()) + else: + jsonContent{"timestamp"}.getBiggestInt() + else: + 0 + , ephemeral: jsonContent{"ephemeral"}.getBool(), meta: Base64String(jsonContent{"meta"}.getStr()), proof: Base64String(jsonContent{"proof"}.getStr()), diff --git a/library/waku_thread/inter_thread_communication/requests/protocols/store_request.nim b/library/waku_thread/inter_thread_communication/requests/protocols/store_request.nim index ee2b608c37..a88ae8c69b 100644 --- a/library/waku_thread/inter_thread_communication/requests/protocols/store_request.nim +++ b/library/waku_thread/inter_thread_communication/requests/protocols/store_request.nim @@ -46,14 +46,20 @@ func fromJsonNode( none(string) let startTime = - if jsonContent.contains("time_start"): - some(Timestamp(jsonContent["time_start"].getInt())) + if jsonContent.hasKey("time_start"): + if jsonContent["time_start"].kind == JString: + some(Timestamp(parseInt(jsonContent["time_start"].getStr()))) + else: + some(Timestamp(jsonContent{"time_start"}.getBiggestInt())) else: none(Timestamp) let endTime = - if jsonContent.contains("time_end"): - some(Timestamp(jsonContent["time_end"].getInt())) + if jsonContent.hasKey("time_end"): + if jsonContent["time_end"].kind == JString: + some(Timestamp(parseInt(jsonContent["time_end"].getStr()))) + else: + some(Timestamp(jsonContent{"time_end"}.getBiggestInt())) else: none(Timestamp) From 9daec3df8864feb65dfca9398e74118b3de66242 Mon Sep 17 00:00:00 2001 From: Richard Ramos Date: Tue, 10 Dec 2024 10:01:50 -0400 Subject: [PATCH 2/3] fix: code review --- library/events/json_message_event.nim | 11 ++------- library/utils.nim | 18 +++++++++++++++ .../requests/protocols/store_request.nim | 23 +++---------------- 3 files changed, 23 insertions(+), 29 deletions(-) create mode 100644 library/utils.nim diff --git a/library/events/json_message_event.nim b/library/events/json_message_event.nim index 3910598e8c..fa8e9c5119 100644 --- a/library/events/json_message_event.nim +++ b/library/events/json_message_event.nim @@ -4,6 +4,7 @@ import ../../waku/common/base64, ../../waku/waku_core/message, ../../waku/waku_core/message/message, + ../utils, ./json_base_event type JsonMessage* = ref object # https://rfc.vac.dev/spec/36/#jsonmessage-type @@ -21,15 +22,7 @@ func fromJsonNode*(T: type JsonMessage, jsonContent: JsonNode): JsonMessage = payload: Base64String(jsonContent["payload"].getStr()), contentTopic: jsonContent["contentTopic"].getStr(), version: uint32(jsonContent{"version"}.getInt()), - timestamp: - if jsonContent.hasKey("timestamp"): - if jsonContent["timestamp"].kind == JString: - parseInt(jsonContent["timestamp"].getStr()) - else: - jsonContent{"timestamp"}.getBiggestInt() - else: - 0 - , + timestamp: jsonContent.getProtoInt64("timestamp").get(0), ephemeral: jsonContent{"ephemeral"}.getBool(), meta: Base64String(jsonContent{"meta"}.getStr()), proof: Base64String(jsonContent{"proof"}.getStr()), diff --git a/library/utils.nim b/library/utils.nim new file mode 100644 index 0000000000..fac0acdcf2 --- /dev/null +++ b/library/utils.nim @@ -0,0 +1,18 @@ +import std/[json, options, strutils] +import results +import ../waku/waku_core/time + +proc getProtoInt64*(node: JsonNode, key: string): Option[int64] = + let (value, ok) = + if node.hasKey(key): + if node[key].kind == JString: + (parseBiggestInt(node[key].getStr()), true) + else: + (node[key].getBiggestInt(), true) + else: + (0, false) + + if ok: + return some(value) + + return none(int64) diff --git a/library/waku_thread/inter_thread_communication/requests/protocols/store_request.nim b/library/waku_thread/inter_thread_communication/requests/protocols/store_request.nim index a88ae8c69b..65bc171592 100644 --- a/library/waku_thread/inter_thread_communication/requests/protocols/store_request.nim +++ b/library/waku_thread/inter_thread_communication/requests/protocols/store_request.nim @@ -3,6 +3,7 @@ import chronos, chronicles, results import ../../../../../waku/factory/waku, ../../../../alloc, + ../../../../utils, ../../../../../waku/waku_core/peers, ../../../../../waku/waku_core/time, ../../../../../waku/waku_core/message/digest, @@ -45,24 +46,6 @@ func fromJsonNode( else: none(string) - let startTime = - if jsonContent.hasKey("time_start"): - if jsonContent["time_start"].kind == JString: - some(Timestamp(parseInt(jsonContent["time_start"].getStr()))) - else: - some(Timestamp(jsonContent{"time_start"}.getBiggestInt())) - else: - none(Timestamp) - - let endTime = - if jsonContent.hasKey("time_end"): - if jsonContent["time_end"].kind == JString: - some(Timestamp(parseInt(jsonContent["time_end"].getStr()))) - else: - some(Timestamp(jsonContent{"time_end"}.getBiggestInt())) - else: - none(Timestamp) - let paginationCursor = if jsonContent.contains("pagination_cursor"): var hash: WakuMessageHash @@ -90,8 +73,8 @@ func fromJsonNode( includeData: jsonContent["include_data"].getBool(), pubsubTopic: pubsubTopic, contentTopics: contentTopics, - startTime: startTime, - endTime: endTime, + startTime: jsonContent.getProtoInt64("time_start"), + endTime: jsonContent.getProtoInt64("time_end"), messageHashes: msgHashes, paginationCursor: paginationCursor, paginationForward: paginationForward, From 5d7f7ea2e07d4ea157597b1d11f882bfcc447204 Mon Sep 17 00:00:00 2001 From: Richard Ramos Date: Tue, 10 Dec 2024 11:01:00 -0400 Subject: [PATCH 3/3] fix: code review --- library/events/json_message_event.nim | 22 +++++++++------ library/libwaku.nim | 6 ++-- library/utils.nim | 26 +++++++++-------- .../requests/protocols/store_request.nim | 28 ++++++++++--------- 4 files changed, 46 insertions(+), 36 deletions(-) diff --git a/library/events/json_message_event.nim b/library/events/json_message_event.nim index fa8e9c5119..ffb6065f56 100644 --- a/library/events/json_message_event.nim +++ b/library/events/json_message_event.nim @@ -16,16 +16,20 @@ type JsonMessage* = ref object # https://rfc.vac.dev/spec/36/#jsonmessage-type meta*: Base64String proof*: Base64String -func fromJsonNode*(T: type JsonMessage, jsonContent: JsonNode): JsonMessage = +func fromJsonNode*( + T: type JsonMessage, jsonContent: JsonNode +): Result[JsonMessage, string] = # Visit https://rfc.vac.dev/spec/14/ for further details - JsonMessage( - payload: Base64String(jsonContent["payload"].getStr()), - contentTopic: jsonContent["contentTopic"].getStr(), - version: uint32(jsonContent{"version"}.getInt()), - timestamp: jsonContent.getProtoInt64("timestamp").get(0), - ephemeral: jsonContent{"ephemeral"}.getBool(), - meta: Base64String(jsonContent{"meta"}.getStr()), - proof: Base64String(jsonContent{"proof"}.getStr()), + ok( + JsonMessage( + payload: Base64String(jsonContent["payload"].getStr()), + contentTopic: jsonContent["contentTopic"].getStr(), + version: uint32(jsonContent{"version"}.getInt()), + timestamp: (?jsonContent.getProtoInt64("timestamp")).get(0), + ephemeral: jsonContent{"ephemeral"}.getBool(), + meta: Base64String(jsonContent{"meta"}.getStr()), + proof: Base64String(jsonContent{"proof"}.getStr()), + ) ) proc toWakuMessage*(self: JsonMessage): Result[WakuMessage, string] = diff --git a/library/libwaku.nim b/library/libwaku.nim index f51ab4aae8..17facdce0d 100644 --- a/library/libwaku.nim +++ b/library/libwaku.nim @@ -267,7 +267,8 @@ proc waku_relay_publish( var jsonMessage: JsonMessage try: let jsonContent = parseJson($jwm) - jsonMessage = JsonMessage.fromJsonNode(jsonContent) + jsonMessage = JsonMessage.fromJsonNode(jsonContent).valueOr: + raise newException(JsonParsingError, $error) except JsonParsingError: deallocShared(jwm) let msg = fmt"Error parsing json message: {getCurrentExceptionMsg()}" @@ -495,7 +496,8 @@ proc waku_lightpush_publish( var jsonMessage: JsonMessage try: let jsonContent = parseJson($jwm) - jsonMessage = JsonMessage.fromJsonNode(jsonContent) + jsonMessage = JsonMessage.fromJsonNode(jsonContent).valueOr: + raise newException(JsonParsingError, $error) except JsonParsingError: let msg = fmt"Error parsing json message: {getCurrentExceptionMsg()}" callback(RET_ERR, unsafeAddr msg[0], cast[csize_t](len(msg)), userData) diff --git a/library/utils.nim b/library/utils.nim index fac0acdcf2..926ec4e882 100644 --- a/library/utils.nim +++ b/library/utils.nim @@ -1,18 +1,20 @@ import std/[json, options, strutils] import results -import ../waku/waku_core/time -proc getProtoInt64*(node: JsonNode, key: string): Option[int64] = - let (value, ok) = - if node.hasKey(key): - if node[key].kind == JString: - (parseBiggestInt(node[key].getStr()), true) +proc getProtoInt64*(node: JsonNode, key: string): Result[Option[int64], string] = + try: + let (value, ok) = + if node.hasKey(key): + if node[key].kind == JString: + (parseBiggestInt(node[key].getStr()), true) + else: + (node[key].getBiggestInt(), true) else: - (node[key].getBiggestInt(), true) - else: - (0, false) + (0, false) - if ok: - return some(value) + if ok: + return ok(some(value)) - return none(int64) + return ok(none(int64)) + except CatchableError: + return err("Invalid int64 value in `" & key & "`") diff --git a/library/waku_thread/inter_thread_communication/requests/protocols/store_request.nim b/library/waku_thread/inter_thread_communication/requests/protocols/store_request.nim index 65bc171592..3e2523fec1 100644 --- a/library/waku_thread/inter_thread_communication/requests/protocols/store_request.nim +++ b/library/waku_thread/inter_thread_communication/requests/protocols/store_request.nim @@ -25,7 +25,7 @@ type StoreRequest* = object func fromJsonNode( T: type JsonStoreQueryRequest, jsonContent: JsonNode -): StoreQueryRequest = +): Result[StoreQueryRequest, string] = let contentTopics = collect(newSeq): for cTopic in jsonContent["content_topics"].getElems(): cTopic.getStr() @@ -68,17 +68,19 @@ func fromJsonNode( else: none(uint64) - return StoreQueryRequest( - requestId: jsonContent["request_id"].getStr(), - includeData: jsonContent["include_data"].getBool(), - pubsubTopic: pubsubTopic, - contentTopics: contentTopics, - startTime: jsonContent.getProtoInt64("time_start"), - endTime: jsonContent.getProtoInt64("time_end"), - messageHashes: msgHashes, - paginationCursor: paginationCursor, - paginationForward: paginationForward, - paginationLimit: paginationLimit, + return ok( + StoreQueryRequest( + requestId: jsonContent["request_id"].getStr(), + includeData: jsonContent["include_data"].getBool(), + pubsubTopic: pubsubTopic, + contentTopics: contentTopics, + startTime: ?jsonContent.getProtoInt64("time_start"), + endTime: ?jsonContent.getProtoInt64("time_end"), + messageHashes: msgHashes, + paginationCursor: paginationCursor, + paginationForward: paginationForward, + paginationLimit: paginationLimit, + ) ) proc createShared*( @@ -117,7 +119,7 @@ proc process( let peer = peers.parsePeerInfo(($self[].peerAddr).split(",")).valueOr: return err("JsonStoreQueryRequest failed to parse peer addr: " & $error) - let queryResponse = (await waku.node.wakuStoreClient.query(storeQueryRequest, peer)).valueOr: + let queryResponse = (await waku.node.wakuStoreClient.query(?storeQueryRequest, peer)).valueOr: return err("JsonStoreQueryRequest failed store query: " & $error) return ok($(%*queryResponse)) ## returning the response in json format