Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat(rad): decode JSON more efficiently #2373

Merged
merged 3 commits into from
Aug 3, 2023
Merged

feat(rad): decode JSON more efficiently #2373

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0a540a4
feat(rad): decode JSON more efficiently
aesedepece May 22, 2023
21a2229
feat(rad): drop deprecated `json` library and use `serde_json` instead
aesedepece May 23, 2023
081dac6
chore: update codebase to latest Rust version (1.71)
aesedepece Jul 27, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 1 addition & 7 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions data_structures/src/chain/tapi.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ impl TapiEngine {
}
}
for n in 0..self.bit_tapi_counter.len() {
if let Some(mut bit_counter) = self.bit_tapi_counter.get_mut(n, &epoch_to_update) {
if let Some(bit_counter) = self.bit_tapi_counter.get_mut(n, &epoch_to_update) {
if !self.wip_activation.contains_key(&bit_counter.wip)
&& !avoid_wip_list.contains(&bit_counter.wip)
{
Expand Down Expand Up @@ -639,7 +639,7 @@ mod tests {
assert_eq!(tapi_counter.current_length, 1);

assert_eq!(tapi_counter.get(0, &100).unwrap().votes, 0);
let mut votes_counter = tapi_counter.get_mut(0, &100).unwrap();
let votes_counter = tapi_counter.get_mut(0, &100).unwrap();
votes_counter.votes += 1;
assert_eq!(tapi_counter.get(0, &100).unwrap().votes, 1);

Expand Down
2 changes: 1 addition & 1 deletion node/src/actors/inventory_manager/handlers.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -512,7 +512,7 @@ mod tests {
// Start relevant actors
config_mngr::start(config);
storage_mngr::start();
let inventory_manager = InventoryManager::default().start();
let inventory_manager = InventoryManager.start();

// Create first block with value transfer transactions
let block = build_block_with_vt_transactions(1);
Expand Down
4 changes: 2 additions & 2 deletions node/src/actors/node.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ pub fn run(config: Arc<Config>, ops: NodeOps, callback: fn()) -> Result<(), fail
SystemRegistry::set(peers_manager_addr);

// Start ConnectionsManager actor
let connections_manager_addr = ConnectionsManager::default().start();
let connections_manager_addr = ConnectionsManager.start();
SystemRegistry::set(connections_manager_addr);

// Start SessionManager actor
Expand All @@ -69,7 +69,7 @@ pub fn run(config: Arc<Config>, ops: NodeOps, callback: fn()) -> Result<(), fail
SystemRegistry::set(chain_manager_addr);

// Start InventoryManager actor
let inventory_manager_addr = InventoryManager::default().start();
let inventory_manager_addr = InventoryManager.start();
SystemRegistry::set(inventory_manager_addr);

// Start RadManager actor
Expand Down
2 changes: 1 addition & 1 deletion partial_struct/tests/partial_struct_derive.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,5 +81,5 @@ fn test_partial_attr_partial() {

let p = PartialObj::default();

assert_eq!(p.f, PartialAnotherObj::default());
assert_eq!(p.f, PartialAnotherObj);
}
4 changes: 2 additions & 2 deletions rad/Cargo.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,14 @@ if_rust_version = "1.0.0"
# the http crate is used to perform additional validations before passing arguments to the surf http client
# the version of http must be kept in sync with the version used by surf
http = "0.2.1"
json = "0.12.1"
log = "0.4.8"
minidom = { git = "https://github.com/witnet/xmpp-rs", rev = "bc8a33ff5da95ee4039ad7ee3376c100d9e35c74" }
num_enum = "0.4.2"
ordered-float = "3.0"
rand = "0.7.3"
serde = "1.0.111"
serde_cbor = "0.11.1"
serde_cbor = "0.11.2"
serde_json = "1.0.96"
# the url crate is used to perform additional validations before passing arguments to the surf http client
# the version of url must be kept in sync with the version used by surf in the `witnet_net` crate
url = "2.1.1"
Expand Down
169 changes: 58 additions & 111 deletions rad/src/operators/string.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ use std::{
};

use serde_cbor::value::{from_value, Value};
use serde_json::Value as JsonValue;

use crate::{
error::RadError,
Expand All @@ -19,16 +20,14 @@ const MAX_DEPTH: u8 = 20;
const DEFAULT_THOUSANDS_SEPARATOR: &str = ",";
const DEFAULT_DECIMAL_SEPARATOR: &str = ".";

/// Parse `RadonTypes` from a JSON-encoded `RadonString`.
pub fn parse_json(input: &RadonString) -> Result<RadonTypes, RadError> {
match json::parse(&input.value()) {
Ok(json_value) => {
let value = json_to_cbor(&json_value);
RadonTypes::try_from(value)
}
Err(json_error) => Err(RadError::JsonParse {
description: json_error.to_string(),
}),
}
let json_value: JsonValue =
serde_json::from_str(&input.value()).map_err(|err| RadError::JsonParse {
description: err.to_string(),
})?;

RadonTypes::try_from(json_value)
}

pub fn parse_json_map(input: &RadonString) -> Result<RadonMap, RadError> {
Expand Down Expand Up @@ -282,39 +281,6 @@ pub fn string_match(input: &RadonString, args: &[Value]) -> Result<RadonTypes, R
.unwrap_or(Ok(temp_def))
}

/// Converts a JSON value (`json::JsonValue`) into a CBOR value (`serde_cbor::value::Value`).
/// Some conversions are totally straightforward, but some others need some more logic (e.g.
/// telling apart integers from floats).
#[allow(clippy::cast_possible_truncation)]
fn json_to_cbor(value: &json::JsonValue) -> Value {
match value {
json::JsonValue::Array(value) => Value::Array(value.iter().map(json_to_cbor).collect()),
json::JsonValue::Object(value) => {
let entries = value
.iter()
.map(|(key, value)| (Value::Text(String::from(key)), json_to_cbor(value)))
.collect();
Value::Map(entries)
}
json::JsonValue::Short(value) => Value::Text(String::from(value.as_str())),
json::JsonValue::Number(value) => {
let (_, _, exponent) = value.as_parts();
let floating = f64::from(*value);
// Cast the float into an integer if it has no fractional part and its value will fit
// into the range of `i128` (38 is the biggest power of 10 that `i128` can safely hold)
if floating.fract() == 0.0 && exponent.unsigned_abs() < 38 {
// This cast is assumed to be safe as per the previous guard
Value::Integer(floating as i128)
} else {
Value::Float(floating)
}
}
json::JsonValue::String(value) => Value::Text(String::from(value.as_str())),
json::JsonValue::Boolean(b) => Value::Bool(*b),
json::JsonValue::Null => Value::Null,
}
}

/// Replace thousands and decimals separators in a `String`.
#[inline]
pub fn replace_separators(
Expand Down Expand Up @@ -568,7 +534,7 @@ mod tests {
let output = parse_json_map(&invalid_json).unwrap_err();

let expected_err = RadError::JsonParse {
description: "Unexpected character: } at (1:13)".to_string(),
description: "expected value at line 1 column 13".to_string(),
};
assert_eq!(output, expected_err);

Expand Down Expand Up @@ -616,7 +582,7 @@ mod tests {
let output = parse_json_array(&invalid_json).unwrap_err();

let expected_err = RadError::JsonParse {
description: "Unexpected character: } at (1:13)".to_string(),
description: "expected value at line 1 column 13".to_string(),
};
assert_eq!(output, expected_err);

Expand Down Expand Up @@ -1168,48 +1134,48 @@ mod tests {
}

#[test]
fn test_json_numbers_to_cbor_numbers() {
use json::{number::Number, JsonValue};

let json = JsonValue::Number(Number::from(2.0));
let resulting_cbor = json_to_cbor(&json);
let expected_cbor = serde_cbor::Value::Integer(2);
assert_eq!(resulting_cbor, expected_cbor);

let json = JsonValue::Number(Number::from(20.0));
let resulting_cbor = json_to_cbor(&json);
let expected_cbor = serde_cbor::Value::Integer(20);
assert_eq!(resulting_cbor, expected_cbor);

let json = JsonValue::Number(Number::from(2_000.0));
let resulting_cbor = json_to_cbor(&json);
let expected_cbor = serde_cbor::Value::Integer(2_000);
assert_eq!(resulting_cbor, expected_cbor);

let json = JsonValue::Number(Number::from(2_000_000.0));
let resulting_cbor = json_to_cbor(&json);
let expected_cbor = serde_cbor::Value::Integer(2_000_000);
assert_eq!(resulting_cbor, expected_cbor);

let json = JsonValue::Number(Number::from(std::f64::consts::PI));
let resulting_cbor = json_to_cbor(&json);
let expected_cbor = serde_cbor::Value::Float(std::f64::consts::PI);
assert_eq!(resulting_cbor, expected_cbor);

let json = JsonValue::Number(Number::from(1e100));
let resulting_cbor = json_to_cbor(&json);
let expected_cbor = serde_cbor::Value::Float(1e100);
assert_eq!(resulting_cbor, expected_cbor);

let json = JsonValue::Number(Number::from(4.0));
let resulting_cbor = json_to_cbor(&json);
let expected_cbor = serde_cbor::Value::Integer(4);
assert_eq!(resulting_cbor, expected_cbor);

let json = JsonValue::Number(Number::from(4.1));
let resulting_cbor = json_to_cbor(&json);
let expected_cbor = serde_cbor::Value::Float(4.1);
assert_eq!(resulting_cbor, expected_cbor);
fn test_json_numbers_to_radon_numbers() {
use serde_json::{value::Number, Value as JsonValue};

let json = JsonValue::Number(Number::from_f64(2.0).unwrap());
let resulting_radon = RadonTypes::try_from(json).unwrap();
let expected_radon = RadonInteger::from(2).into();
assert_eq!(resulting_radon, expected_radon);

let json = JsonValue::Number(Number::from_f64(20.0).unwrap());
let resulting_radon = RadonTypes::try_from(json).unwrap();
let expected_radon = RadonInteger::from(20).into();
assert_eq!(resulting_radon, expected_radon);

let json = JsonValue::Number(Number::from_f64(2_000.0).unwrap());
let resulting_radon = RadonTypes::try_from(json).unwrap();
let expected_radon = RadonInteger::from(2_000).into();
assert_eq!(resulting_radon, expected_radon);

let json = JsonValue::Number(Number::from_f64(2_000_000.0).unwrap());
let resulting_radon = RadonTypes::try_from(json).unwrap();
let expected_radon = RadonInteger::from(2_000_000).into();
assert_eq!(resulting_radon, expected_radon);

let json = JsonValue::Number(Number::from_f64(std::f64::consts::PI).unwrap());
let resulting_radon = RadonTypes::try_from(json).unwrap();
let expected_radon = RadonFloat::from(std::f64::consts::PI).into();
assert_eq!(resulting_radon, expected_radon);

let json = JsonValue::Number(Number::from_f64(1e100).unwrap());
let resulting_radon = RadonTypes::try_from(json).unwrap();
let expected_radon = RadonFloat::from(1e100).into();
assert_eq!(resulting_radon, expected_radon);

let json = JsonValue::Number(Number::from_f64(4.0).unwrap());
let resulting_radon = RadonTypes::try_from(json).unwrap();
let expected_radon = RadonInteger::from(4).into();
assert_eq!(resulting_radon, expected_radon);

let json = JsonValue::Number(Number::from_f64(4.1).unwrap());
let resulting_radon = RadonTypes::try_from(json).unwrap();
let expected_radon = RadonFloat::from(4.1).into();
assert_eq!(resulting_radon, expected_radon);
}

#[test]
Expand All @@ -1219,38 +1185,19 @@ mod tests {
// does not cause any overflows

// Parse the number using json::parse because otherwise it is just converted to 0.0
let json = json::parse("0.1E-99999").unwrap();
// The exponent is too small to fit in a i16, so the json library saturates the value to
// i16::MIN:
let (sign, mantissa, exponent) = json.as_number().unwrap().as_parts();
assert_eq!((sign, mantissa, exponent), (true, 1, i16::MIN));
let number: serde_json::Number = serde_json::from_str("0.1E-99999").unwrap();
// This number is rounded to exactly 0.0 when converted to f64
assert_eq!(json.as_f64().unwrap(), 0.0);
assert_eq!(number.as_f64().unwrap(), 0.0);

// Convert to CBOR
let resulting_cbor = json_to_cbor(&json);
// Convert to RadonTypes
let resulting_radon = RadonTypes::try_from(JsonValue::Number(number)).unwrap();

// This exponent is too small to fit in a f64, so expected_f64 is equal to 0.0
let expected_f64 = 0.1E-99999;
assert_eq!(expected_f64, 0.0);
// And the expected CBOR value is a float, not an integer
let expected_cbor = serde_cbor::Value::Float(expected_f64);
assert_eq!(resulting_cbor, expected_cbor);
}

#[test]
fn test_json_numbers_to_cbor_booleans() {
use json::JsonValue;

let json = JsonValue::Boolean(false);
let resulting_cbor = json_to_cbor(&json);
let expected_cbor = serde_cbor::Value::Bool(false);
assert_eq!(resulting_cbor, expected_cbor);

let json = JsonValue::Boolean(true);
let resulting_cbor = json_to_cbor(&json);
let expected_cbor = serde_cbor::Value::Bool(true);
assert_eq!(resulting_cbor, expected_cbor);
let expected_radon = RadonFloat::from(expected_f64).into();
assert_eq!(resulting_radon, expected_radon);
}

#[test]
Expand Down
Loading