Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Testing harness for compatability with negentropy reference test suite, fix issues found by test suite #2

Merged
merged 2 commits into from
Sep 13, 2023
Merged

Testing harness for compatability with negentropy reference test suite, fix issues found by test suite #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
593a079
use VecDeque to match C++ impl more closely (possibly not necessary, …
hoytech Sep 12, 2023
dbc3e78
testing harness for compatability with negentropy reference test suit…
hoytech Sep 12, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 59 additions & 0 deletions examples/harness.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
// This is a testing harness for compatibility with the negentropy reference
// implementation's test suite: https://github.com/hoytech/negentropy/tree/master/test

use negentropy::Negentropy;
use std::io;
use std::env;

fn main() {
let id_size = 16;

let frame_size_limit_env_var = env::var("FRAMESIZELIMIT");
let frame_size_limit = if frame_size_limit_env_var.is_ok() { frame_size_limit_env_var.unwrap().parse::<usize>().unwrap() } else { 0 };

let mut ne = Negentropy::new(id_size, Some(frame_size_limit as u64)).unwrap();

for line in io::stdin().lines() {
let line_unwrapped = line.unwrap();
let items: Vec<&str> = line_unwrapped.split(",").collect();

if items[0] == "item" {
let created = items[1].parse::<u64>().unwrap();
let id = items[2];
ne.add_item(created, id).unwrap();
} else if items[0] == "seal" {
ne.seal().unwrap();
} else if items[0] == "initiate" {
let q = ne.initiate().unwrap();
if frame_size_limit > 0 && q.len()/2 > frame_size_limit { panic!("frameSizeLimit exceeded"); }
println!("msg,{}", q);
} else if items[0] == "msg" {
let mut q = String::new();

if items.len() >= 2 {
q = items[1].to_string();
}

if ne.is_initiator() {
let mut have_ids = Vec::new();
let mut need_ids = Vec::new();
q = ne.reconcile_with_ids(&q, &mut have_ids, &mut need_ids).unwrap();

for id in &have_ids { println!("have,{}", id); }
for id in &need_ids { println!("need,{}", id); }

if q.len() == 0 {
println!("done");
continue;
}
} else {
q = ne.reconcile(&q).unwrap();
}

if frame_size_limit > 0 && q.len()/2 > frame_size_limit { panic!("frameSizeLimit exceeded"); }
println!("msg,{}", q);
} else {
panic!("unknwown cmd");
}
}
}
70 changes: 33 additions & 37 deletions src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,10 +23,10 @@ use alloc::vec::Vec;
use core::cmp::Ordering;
use core::convert::TryFrom;
use core::fmt;
use core::iter;
use core::ops::BitXorAssign;
#[cfg(feature = "std")]
use std::collections::HashSet;
use std::collections::VecDeque;

mod hex;

Expand Down Expand Up @@ -211,7 +211,7 @@ pub struct Negentropy {
sealed: bool,
is_initiator: bool,
continuation_needed: bool,
pending_outputs: Vec<BoundOutput>,
pending_outputs: VecDeque<BoundOutput>,
}

impl Negentropy {
Expand All @@ -234,7 +234,7 @@ impl Negentropy {
sealed: false,
is_initiator: false,
continuation_needed: false,
pending_outputs: Vec::new(),
pending_outputs: VecDeque::new(),
})
}

Expand Down Expand Up @@ -287,7 +287,7 @@ impl Negentropy {

self.is_initiator = true;

let mut outputs: Vec<BoundOutput> = Vec::new();
let mut outputs: VecDeque<BoundOutput> = VecDeque::new();

self.split_range(
&self.items,
Expand Down Expand Up @@ -350,7 +350,7 @@ impl Negentropy {
let mut prev_bound: XorElem = XorElem::new();
let mut prev_index: usize = 0;
let mut last_timestamp_in: u64 = 0;
let mut outputs: Vec<BoundOutput> = Vec::new();
let mut outputs: VecDeque<BoundOutput> = VecDeque::new();
let mut query: &[u8] = query.as_ref();

while !query.is_empty() {
Expand Down Expand Up @@ -457,15 +457,17 @@ impl Negentropy {
prev_bound = curr_bound;
}

self.pending_outputs.extend(outputs.into_iter().rev());
while outputs.len() > 0 {
self.pending_outputs.push_front(outputs.pop_back().unwrap());
}

Ok(())
}

#[allow(clippy::too_many_arguments)]
fn flush_id_list_output(
&self,
outputs: &mut Vec<BoundOutput>,
outputs: &mut VecDeque<BoundOutput>,
upper: usize,
prev_bound: XorElem,
did_split: &mut bool,
Expand All @@ -489,7 +491,7 @@ impl Negentropy {
self.get_minimal_bound(&self.items[*it], &self.items[*it + 1])?
};

outputs.push(BoundOutput {
outputs.push_back(BoundOutput {
start: if *did_split { *split_bound } else { prev_bound },
end: next_split_bound,
payload,
Expand All @@ -508,7 +510,7 @@ impl Negentropy {
items: &[XorElem],
lower_bound: XorElem,
upper_bound: XorElem,
outputs: &mut Vec<BoundOutput>,
outputs: &mut VecDeque<BoundOutput>,
) -> Result<(), Error> {
let num_elems: usize = items.len();

Expand All @@ -521,52 +523,40 @@ impl Negentropy {
payload.extend_from_slice(elem.get_id_subsize(self.id_size));
}

outputs.push(BoundOutput {
outputs.push_back(BoundOutput {
start: lower_bound,
end: upper_bound,
payload,
});
} else {
let items_per_bucket: usize = num_elems / BUCKETS;
let buckets_with_extra: usize = num_elems % BUCKETS;
let lower: XorElem = items.first().copied().unwrap_or_default();
let mut prev_bound: XorElem = lower;
let bucket_end = items.iter().copied().take(items_per_bucket);
let mut curr : usize = 0;
let mut prev_bound = items.iter().nth(0).unwrap();

for i in 0..BUCKETS {
let mut our_xor_set: XorElem = XorElem::new();
let bucket_end : usize = curr + items_per_bucket + (if i < buckets_with_extra { 1 } else { 0 });

let bucket_end = bucket_end.clone();
if i < buckets_with_extra {
for elem in bucket_end.chain(iter::once(lower)) {
our_xor_set ^= elem;
}
} else {
for elem in bucket_end {
our_xor_set ^= elem;
}
};
while curr != bucket_end {
our_xor_set ^= items[curr];
curr += 1;
}

let mut payload: Vec<u8> = Vec::with_capacity(10 + self.id_size as usize);
payload.extend(self.encode_mode(Mode::Fingerprint));
payload.extend(our_xor_set.get_id_subsize(self.id_size));

let next_bound: XorElem = if i == 0 {
lower_bound
} else {
self.get_minimal_bound(&prev_bound, &lower)?
};

outputs.push(BoundOutput {
start: if i == 0 { lower_bound } else { prev_bound },
end: upper_bound,
outputs.push_back(BoundOutput {
start: if i == 0 { lower_bound } else { *prev_bound },
end: if bucket_end == items.len() { upper_bound } else { self.get_minimal_bound(&items[curr-1], &items[curr])? },
payload,
});

prev_bound = next_bound;
prev_bound = &outputs.back().unwrap().end;
}

if let Some(output) = outputs.last_mut() {
if let Some(output) = outputs.back_mut() {
output.end = upper_bound;
}
}
Expand All @@ -579,9 +569,9 @@ impl Negentropy {
let mut curr_bound: XorElem = XorElem::new();
let mut last_timestamp_out: u64 = 0;

self.pending_outputs.sort_by(|a, b| b.start.cmp(&a.start));
self.pending_outputs.make_contiguous().sort_by(|a, b| a.start.cmp(&b.start));

while let Some(p) = self.pending_outputs.last() {
while let Some(p) = self.pending_outputs.front() {
let mut o: Vec<u8> = Vec::new();

if p.start < curr_bound {
Expand All @@ -606,7 +596,7 @@ impl Negentropy {

output.extend(o);
curr_bound = p.end;
self.pending_outputs.pop();
self.pending_outputs.pop_front();
}

if (!self.is_initiator && !self.pending_outputs.is_empty())
Expand Down Expand Up @@ -693,6 +683,12 @@ impl Negentropy {
n >>= 7;
}

o.reverse();

for i in 0..(o.len() - 1) {
o[i] |= 0x80;
}

o
}

Expand Down