Skip to content

Commit

Permalink
Add multithread to similar image finder
Browse files Browse the repository at this point in the history
  • Loading branch information
@qarmin
qarmin committed Nov 7, 2020
1 parent 110d601 commit 6ee5566
Show file tree
Hide file tree
Showing 7 changed files with 190 additions and 46 deletions.
1 change: 1 addition & 0 deletions Cargo.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion czkawka_core/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ repository = "https://github.com/qarmin/czkawka"
[dependencies]
humansize = "1"
blake3 = "0.3"
#rayon = "1"
rayon = "1"
crossbeam-channel = "0.4.4"


Expand Down
2 changes: 1 addition & 1 deletion czkawka_core/src/common.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ impl Common {
/// Printing time which took between start and stop point and prints also function name
#[allow(unused_variables)]
pub fn print_time(start_time: SystemTime, end_time: SystemTime, function_name: String) {
#[cfg(debug_assertions)]
// #[cfg(debug_assertions)]
println!("Execution of function \"{}\" took {:?}", function_name, end_time.duration_since(start_time).expect("Time cannot go reverse."));
}

Expand Down
61 changes: 44 additions & 17 deletions czkawka_core/src/similar_images.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ use crossbeam_channel::Receiver;
use humansize::{file_size_opts as options, FileSize};
use image::GenericImageView;
use img_hash::HasherConfig;
use rayon::prelude::*;
use std::collections::HashMap;
use std::fs;
use std::fs::{File, Metadata};
Expand Down Expand Up @@ -58,6 +59,7 @@ pub struct SimilarImages {
image_hashes: HashMap<Node, Vec<FileEntry>>, // Hashmap with image hashes and Vector with names of files
stopped_search: bool,
similarity: Similarity,
images_to_check: Vec<FileEntry>,
}

/// Info struck with helpful information's about results
Expand Down Expand Up @@ -95,6 +97,7 @@ impl SimilarImages {
image_hashes: Default::default(),
stopped_search: false,
similarity: Similarity::High,
images_to_check: vec![],
}
}

Expand Down Expand Up @@ -231,16 +234,10 @@ impl SimilarImages {
continue 'dir;
}

let image = match image::open(&current_file_name) {
Ok(t) => t,
Err(_) => continue 'dir, // Something is wrong with image
};
let dimensions = image.dimensions();
// Creating new file entry
let fe: FileEntry = FileEntry {
path: current_file_name.clone(),
size: metadata.len(),
dimensions: format!("{}x{}", dimensions.0, dimensions.1),
dimensions: "".to_string(),
modified_date: match metadata.modified() {
Ok(t) => match t.duration_since(UNIX_EPOCH) {
Ok(d) => d.as_secs(),
Expand All @@ -257,15 +254,8 @@ impl SimilarImages {

similarity: Similarity::None,
};
let hasher = HasherConfig::with_bytes_type::<[u8; 8]>().to_hasher();

let hash = hasher.hash_image(&image);
let mut buf = [0u8; 8];
buf.copy_from_slice(&hash.as_bytes());

self.bktree.add(buf);
self.image_hashes.entry(buf).or_insert_with(Vec::<FileEntry>::new);
self.image_hashes.get_mut(&buf).unwrap().push(fe);
self.images_to_check.push(fe);

self.information.size_of_checked_images += metadata.len();
self.information.number_of_checked_files += 1;
Expand All @@ -286,6 +276,40 @@ impl SimilarImages {
fn sort_images(&mut self, stop_receiver: Option<&Receiver<()>>) -> bool {
let hash_map_modification = SystemTime::now();

let vec_file_entry = self
.images_to_check
.par_iter()
.filter_map(|file_entry| {
if stop_receiver.is_some() && stop_receiver.unwrap().try_recv().is_ok() {
return None;
}
let mut file_entry = file_entry.clone();

let image = match image::open(file_entry.path.clone()) {
Ok(t) => t,
Err(_) => return None, // Something is wrong with image
};
let dimensions = image.dimensions();

file_entry.dimensions = format!("{}x{}", dimensions.0, dimensions.1);
let hasher = HasherConfig::with_bytes_type::<[u8; 8]>().to_hasher();

let hash = hasher.hash_image(&image);
let mut buf = [0u8; 8];
buf.copy_from_slice(&hash.as_bytes());

Option::from((file_entry, buf))
})
// .filter(|e| e.is_some())
// .map
.collect::<Vec<_>>();

for (file_entry, buf) in vec_file_entry {
self.bktree.add(buf);
self.image_hashes.entry(buf).or_insert_with(Vec::<FileEntry>::new);
self.image_hashes.get_mut(&buf).unwrap().push(file_entry.clone());
}

//let hash_map_modification = SystemTime::now();
let similarity: u64 = match self.similarity {
Similarity::VeryHigh => 0,
Expand Down Expand Up @@ -327,7 +351,7 @@ impl SimilarImages {

for (similarity, similar_hash) in vector_with_found_similar_hashes.iter() {
if *similarity == 0 && hash == *similar_hash {
// This was already readed before
// This was already read before
continue;
} else if hash == *similar_hash {
panic!("I'm not sure if same hash can have distance > 0");
Expand Down Expand Up @@ -355,7 +379,10 @@ impl SimilarImages {
hashes_to_check.remove(*similar_hash);
}
}
new_vector.push((*vector_of_similar_images).to_owned());
if vector_of_similar_images.len() > 1 {
// Not sure why it may happens
new_vector.push((*vector_of_similar_images).to_owned());
}
}

self.similar_vectors = new_vector;
Expand Down
98 changes: 91 additions & 7 deletions czkawka_gui/czkawka.glade
View file
Original file line number Diff line number Diff line change
Expand Up @@ -745,13 +745,13 @@ Author: Rafał Mikrut
<property name="visible">True</property>
<property name="can_focus">False</property>
<child>
<object class="GtkRadioButton" id="radio_button_name">
<object class="GtkRadioButton" id="radio_button_duplicates_name">
<property name="label" translatable="yes">Name(very fast)</property>
<property name="visible">True</property>
<property name="can_focus">True</property>
<property name="receives_default">False</property>
<property name="draw_indicator">True</property>
<property name="group">radio_button_hash</property>
<property name="group">radio_button_duplicates_hash</property>
</object>
<packing>
<property name="expand">False</property>
Expand All @@ -760,13 +760,13 @@ Author: Rafał Mikrut
</packing>
</child>
<child>
<object class="GtkRadioButton" id="radio_button_size">
<object class="GtkRadioButton" id="radio_button_duplicates_size">
<property name="label" translatable="yes">Size(very fast)</property>
<property name="visible">True</property>
<property name="can_focus">True</property>
<property name="receives_default">False</property>
<property name="draw_indicator">True</property>
<property name="group">radio_button_hash</property>
<property name="group">radio_button_duplicates_hash</property>
</object>
<packing>
<property name="expand">False</property>
Expand All @@ -775,13 +775,13 @@ Author: Rafał Mikrut
</packing>
</child>
<child>
<object class="GtkRadioButton" id="radio_button_hashmb">
<object class="GtkRadioButton" id="radio_button_duplicates_hashmb">
<property name="label" translatable="yes">HashMb(fast)</property>
<property name="visible">True</property>
<property name="can_focus">True</property>
<property name="receives_default">False</property>
<property name="draw_indicator">True</property>
<property name="group">radio_button_hash</property>
<property name="group">radio_button_duplicates_hash</property>
</object>
<packing>
<property name="expand">False</property>
Expand All @@ -790,7 +790,7 @@ Author: Rafał Mikrut
</packing>
</child>
<child>
<object class="GtkRadioButton" id="radio_button_hash">
<object class="GtkRadioButton" id="radio_button_duplicates_hash">
<property name="label" translatable="yes">Hash(slow but accurate)</property>
<property name="visible">True</property>
<property name="can_focus">True</property>
Expand Down Expand Up @@ -1066,6 +1066,90 @@ Author: Rafał Mikrut
<property name="position">0</property>
</packing>
</child>
<child>
<object class="GtkBox">
<property name="visible">True</property>
<property name="can_focus">False</property>
<child>
<object class="GtkLabel">
<property name="visible">True</property>
<property name="can_focus">False</property>
<property name="label" translatable="yes">Similarity level </property>
</object>
<packing>
<property name="expand">False</property>
<property name="fill">True</property>
<property name="position">0</property>
</packing>
</child>
<child>
<object class="GtkRadioButton" id="radio_button_similar_images_small">
<property name="label" translatable="yes">Small</property>
<property name="visible">True</property>
<property name="can_focus">True</property>
<property name="receives_default">False</property>
<property name="draw_indicator">True</property>
<property name="group">radio_button_similar_images_very_high</property>
</object>
<packing>
<property name="expand">False</property>
<property name="fill">True</property>
<property name="position">1</property>
</packing>
</child>
<child>
<object class="GtkRadioButton" id="radio_button_similar_images_medium">
<property name="label" translatable="yes">Medium</property>
<property name="visible">True</property>
<property name="can_focus">True</property>
<property name="receives_default">False</property>
<property name="draw_indicator">True</property>
<property name="group">radio_button_similar_images_very_high</property>
</object>
<packing>
<property name="expand">False</property>
<property name="fill">True</property>
<property name="position">2</property>
</packing>
</child>
<child>
<object class="GtkRadioButton" id="radio_button_similar_images_high">
<property name="label" translatable="yes">High</property>
<property name="visible">True</property>
<property name="can_focus">True</property>
<property name="receives_default">False</property>
<property name="active">True</property>
<property name="draw_indicator">True</property>
<property name="group">radio_button_similar_images_very_high</property>
</object>
<packing>
<property name="expand">False</property>
<property name="fill">True</property>
<property name="position">3</property>
</packing>
</child>
<child>
<object class="GtkRadioButton" id="radio_button_similar_images_very_high">
<property name="label" translatable="yes">Very High</property>
<property name="visible">True</property>
<property name="can_focus">True</property>
<property name="receives_default">False</property>
<property name="active">True</property>
<property name="draw_indicator">True</property>
</object>
<packing>
<property name="expand">False</property>
<property name="fill">True</property>
<property name="position">4</property>
</packing>
</child>
</object>
<packing>
<property name="expand">False</property>
<property name="fill">True</property>
<property name="position">1</property>
</packing>
</child>
<child>
<object class="GtkScrolledWindow" id="scrolled_window_similar_images_finder">
<property name="visible">True</property>
Expand Down
34 changes: 26 additions & 8 deletions czkawka_gui/src/connect_button_search.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,14 @@ pub fn connect_button_search(gui_data: &GuiData, sender: Sender<Message>) {
let entry_same_music_minimal_size = gui_data.entry_same_music_minimal_size.clone();
let entry_allowed_extensions = gui_data.entry_allowed_extensions.clone();
let buttons_names = gui_data.buttons_names.clone();
let radio_button_name = gui_data.radio_button_name.clone();
let radio_button_size = gui_data.radio_button_size.clone();
let radio_button_hashmb = gui_data.radio_button_hashmb.clone();
let radio_button_hash = gui_data.radio_button_hash.clone();
let radio_button_duplicates_name = gui_data.radio_button_duplicates_name.clone();
let radio_button_duplicates_size = gui_data.radio_button_duplicates_size.clone();
let radio_button_duplicates_hashmb = gui_data.radio_button_duplicates_hashmb.clone();
let radio_button_duplicates_hash = gui_data.radio_button_duplicates_hash.clone();
let radio_button_similar_images_small = gui_data.radio_button_similar_images_small.clone();
let radio_button_similar_images_medium = gui_data.radio_button_similar_images_medium.clone();
let radio_button_similar_images_high = gui_data.radio_button_similar_images_high.clone();
let radio_button_similar_images_very_high = gui_data.radio_button_similar_images_very_high.clone();
let entry_duplicate_minimal_size = gui_data.entry_duplicate_minimal_size.clone();
let stop_receiver = gui_data.stop_receiver.clone();
let entry_big_files_number = gui_data.entry_big_files_number.clone();
Expand Down Expand Up @@ -70,13 +74,13 @@ pub fn connect_button_search(gui_data: &GuiData, sender: Sender<Message>) {
get_list_store(&scrolled_window_duplicate_finder).clear();

let check_method;
if radio_button_name.get_active() {
if radio_button_duplicates_name.get_active() {
check_method = duplicate::CheckingMethod::Name;
} else if radio_button_size.get_active() {
} else if radio_button_duplicates_size.get_active() {
check_method = duplicate::CheckingMethod::Size;
} else if radio_button_hashmb.get_active() {
} else if radio_button_duplicates_hashmb.get_active() {
check_method = duplicate::CheckingMethod::HashMB;
} else if radio_button_hash.get_active() {
} else if radio_button_duplicates_hash.get_active() {
check_method = duplicate::CheckingMethod::Hash;
} else {
panic!("No radio button is pressed");
Expand Down Expand Up @@ -186,6 +190,19 @@ pub fn connect_button_search(gui_data: &GuiData, sender: Sender<Message>) {
Err(_) => 1024 * 16, // By default
};

let similarity;
if radio_button_similar_images_small.get_active() {
similarity = similar_images::Similarity::Small;
} else if radio_button_similar_images_medium.get_active() {
similarity = similar_images::Similarity::Medium;
} else if radio_button_similar_images_high.get_active() {
similarity = similar_images::Similarity::High;
} else if radio_button_similar_images_very_high.get_active() {
similarity = similar_images::Similarity::VeryHigh;
} else {
panic!("No radio button is pressed");
}

// Find similar images
thread::spawn(move || {
let mut sf = SimilarImages::new();
Expand All @@ -195,6 +212,7 @@ pub fn connect_button_search(gui_data: &GuiData, sender: Sender<Message>) {
sf.set_recursive_search(recursive_search);
sf.set_excluded_items(excluded_items);
sf.set_minimal_file_size(minimal_file_size);
sf.set_similarity(similarity);
sf.find_similar_images(Option::from(&receiver_stop));
let _ = sender.send(Message::SimilarImages(sf));
});
Expand Down
Loading

0 comments on commit 6ee5566

Please sign in to comment.