Skip to content

Commit

Permalink
debugging - make runnable on fullnode
Browse files Browse the repository at this point in the history
  • Loading branch information
@williampsmith
williampsmith committed Sep 22, 2023
1 parent 05bc794 commit 84cc493
Show file tree
Hide file tree
Showing 2 changed files with 93 additions and 80 deletions.
14 changes: 9 additions & 5 deletions crates/sui-core/src/authority/authority_store.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -892,11 +892,15 @@ impl AuthorityStore {
}
}
}
// TODO(william) uncomment!!!!!!
// let sha3_digest = hasher.finalize().digest;
// if *expected_sha3_digest != sha3_digest {
// return Err(SuiError::from("Sha does not match"));
// }
let sha3_digest = hasher.finalize().digest;
if *expected_sha3_digest != sha3_digest {
info!(
"TESTING -- sha does not match!!!!! expected: {:?}, actual: {:?}",
expected_sha3_digest, sha3_digest
);
// TODO(william) UNCOMMENT!!!!!!
// return Err(SuiError::from("Sha does not match"));
}
info!("TESTING -- finished inserting live objects for object file");
batch.write()?;
Ok(())
Expand Down
159 changes: 84 additions & 75 deletions crates/sui-snapshot/src/reader.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,24 +129,24 @@ impl StateSnapshotReaderV1 {
})
.collect();

// let progress_bar = m.add(
// ProgressBar::new(files.len() as u64).with_style(
// ProgressStyle::with_template(
// "[{elapsed_precise}] {wide_bar} {pos} out of {len} .ref files done\n({msg})",
// )
// .unwrap(),
// ),
// );
// copy_files(
// &files,
// &files,
// remote_object_store.clone(),
// local_object_store.clone(),
// download_concurrency,
// Some(progress_bar.clone()),
// )
// .await?;
// progress_bar.finish_with_message("ref files download complete");
let progress_bar = m.add(
ProgressBar::new(files.len() as u64).with_style(
ProgressStyle::with_template(
"[{elapsed_precise}] {wide_bar} {pos} out of {len} .ref files done\n({msg})",
)
.unwrap(),
),
);
copy_files(
&files,
&files,
remote_object_store.clone(),
local_object_store.clone(),
download_concurrency,
Some(progress_bar.clone()),
)
.await?;
progress_bar.finish_with_message("ref files download complete");
Ok(StateSnapshotReaderV1 {
epoch,
local_staging_dir_root,
Expand Down Expand Up @@ -176,60 +176,60 @@ impl StateSnapshotReaderV1 {
let mut accumulator = Accumulator::default();

info!("Computing sha3 digests and accumulator");
// let mut instant = Instant::now();
// let num_part_files = self
// .ref_files
// .iter()
// .map(|(_bucket, part_files)| part_files.len())
// .sum::<usize>();
// let accum_progress_bar = self.m.add(
// ProgressBar::new(num_part_files as u64).with_style(
// ProgressStyle::with_template(
// "[{elapsed_precise}] {wide_bar} {pos} out of {len} ref files accumulated ({msg})",
// )
// .unwrap(),
// ),
// );
let mut instant = Instant::now();
let num_part_files = self
.ref_files
.iter()
.map(|(_bucket, part_files)| part_files.len())
.sum::<usize>();
let accum_progress_bar = self.m.add(
ProgressBar::new(num_part_files as u64).with_style(
ProgressStyle::with_template(
"[{elapsed_precise}] {wide_bar} {pos} out of {len} ref files accumulated ({msg})",
)
.unwrap(),
),
);

// info!("TESTING -- before accumulation");
// for (bucket, part_files) in self.ref_files.clone().iter() {
// for (part, part_file) in part_files.iter() {
// info!("TESTING -- (1)");
// let mut sha3_digests = sha3_digests.lock().await;
// info!("TESTING -- (1.2)");
// let ref_iter = self.ref_iter(*bucket, *part)?;
// info!("TESTING -- (1.3)");
// let mut hasher = Sha3_256::default();
// info!("TESTING -- (1.4)");
// let mut empty = true;
// info!("TESTING -- (1.5)");
// self.object_files
// .get(bucket)
// .context(format!("No bucket exists for: {bucket}"))?
// .get(part)
// .context(format!("No part exists for bucket: {bucket}, part: {part}"))?;
// info!("TESTING -- (2)");
// for object_ref in ref_iter {
// hasher.update(object_ref.2.inner());
// accumulator.insert(object_ref.2);
// }
// info!("TESTING -- (3)");
// if !empty {
// sha3_digests
// .entry(*bucket)
// .or_insert(BTreeMap::new())
// .entry(*part)
// .or_insert(hasher.finalize().digest);
// }
// info!("TESTING -- (4)");
// accum_progress_bar.inc(1);
// accum_progress_bar.set_message(format!("Bucket: {}, Part: {}", bucket, part));
// instant = Instant::now();
// empty = false;
// }
// }
// info!("TESTING -- after accumulation");
// accum_progress_bar.finish_with_message("Accumulation complete");
info!("TESTING -- before accumulation");
for (bucket, part_files) in self.ref_files.clone().iter() {
for (part, part_file) in part_files.iter() {
info!("TESTING -- (1)");
let mut sha3_digests = sha3_digests.lock().await;
info!("TESTING -- (1.2)");
let ref_iter = self.ref_iter(*bucket, *part)?;
info!("TESTING -- (1.3)");
let mut hasher = Sha3_256::default();
info!("TESTING -- (1.4)");
let mut empty = true;
info!("TESTING -- (1.5)");
self.object_files
.get(bucket)
.context(format!("No bucket exists for: {bucket}"))?
.get(part)
.context(format!("No part exists for bucket: {bucket}, part: {part}"))?;
info!("TESTING -- (2)");
for object_ref in ref_iter {
hasher.update(object_ref.2.inner());
// accumulator.insert(object_ref.2);
}
info!("TESTING -- (3)");
if !empty {
sha3_digests
.entry(*bucket)
.or_insert(BTreeMap::new())
.entry(*part)
.or_insert(hasher.finalize().digest);
}
info!("TESTING -- (4)");
accum_progress_bar.inc(1);
accum_progress_bar.set_message(format!("Bucket: {}, Part: {}", bucket, part));
instant = Instant::now();
empty = false;
}
}
info!("TESTING -- after accumulation");
accum_progress_bar.finish_with_message("Accumulation complete");

if let Some(sender) = sender {
if let Err(_) = sender.send(accumulator) {
Expand Down Expand Up @@ -296,14 +296,23 @@ impl StateSnapshotReaderV1 {
);
let sha3_digest = sha3_digests_cloned.lock().await;
info!("TESTING -- aquired the sha3_digest lock");
// TODO(william) uncomment

// TODO(william) UNCOMMENT!!
// let bucket_map = sha3_digest.get(bucket).context("Missing bucket")?;
// let sha3_digest = bucket_map.get(part_num).context("Missing part")?;

// TODO(william) REMOVE!!
let bucket_map = sha3_digest.get(bucket).unwrap_or_else(|| {
info!("TESTING -- missing bucket!");
panic!("Missing bucket");
});

info!("TESTING -- got bucket_map. Size: {}", bucket_map.len());
let sha3_digest = bucket_map.get(part_num).context("Missing part")?;
info!("TESTING -- got sha3_digest");
Ok::<(Bytes, FileMetadata, [u8; 32]), anyhow::Error>((
bytes,
(*file_metadata).clone(),
// *sha3_digest,
[0; 32],
*sha3_digest,
))
}
})
Expand Down

0 comments on commit 84cc493

Please sign in to comment.