Resolve #289 Find similar students #316
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -314,15 +314,29 @@ private File createAvatarFileFromStudentImage(StudentImage studentImage, Student | |
|
|
||
| /** | ||
| * Find similar students | ||
| * | ||
| * | ||
| */ | ||
| public synchronized void findAndMergeSimilarStudents(){ | ||
| Log.i(getClass().getName(), "findAndMergeSimilarStudents"); | ||
| PreProcessorFactory ppF = new PreProcessorFactory(context); | ||
| TensorFlow tensorFlow = getInitializedTensorFlow(); | ||
| findSimilarStudentsUsingAvatarImages(ppF, tensorFlow); | ||
| findSimilarStudentsUsingMeanFeatureVector(ppF, tensorFlow); | ||
| } | ||
|
|
||
| /** | ||
| * Find similar students | ||
| * Case 1: Student was added during fallback but in the meantime the same person has an existing StudentImageCollectionEvent and a new Student entry | ||
| * ---> Use the avatar image as input for the recognition | ||
| * @param ppF | ||
| * @param tensorFlow | ||
| */ | ||
| private synchronized void findSimilarStudentsUsingAvatarImages(PreProcessorFactory ppF, TensorFlow tensorFlow){ | ||
| // Iterate through all Students | ||
| List<Student> students = studentDao.loadAll(); | ||
|
There was a problem hiding this comment. Add logging here so that we can see if the method was triggered
|
||
| for (Student student : students){ | ||
| // Take the avatar image of the Student | ||
| Mat avatarImage = Imgcodecs.imread(student.getAvatar()); | ||
| // Search for faces in the avatar image | ||
| List<Mat> faceImages = ppF.getCroppedImage(avatarImage); | ||
|
|
@@ -334,26 +348,60 @@ public synchronized void findAndMergeSimilarStudents(){ | |
| Rect[] faces = ppF.getFacesForRecognition(); | ||
| if (faces != null && faces.length == 1) { | ||
| // Proceed if exactly one face rectangle exists | ||
| RecognitionThread recognitionThread = new RecognitionThread(tensorFlow, studentImageCollectionEventDao); | ||
| recognitionThread.setImg(faceImage); | ||
| Log.i(getClass().getName(), "findSimilarStudentsUsingAvatarImages: recognitionThread will be started to recognize student: " + student.getUniqueId()); | ||
| recognitionThread.start(); | ||
| try { | ||
| recognitionThread.join(); | ||
| Student recognizedStudent = recognitionThread.getStudent(); | ||
| if (recognizedStudent != null){ | ||
| Log.i(getClass().getName(), "findSimilarStudentsUsingAvatarImages: The student " + student.getUniqueId() + " has been recognized as " + recognizedStudent.getUniqueId()); | ||
| mergeSimilarStudents(student, recognizedStudent); | ||
| } else { | ||
| Log.i(getClass().getName(), "findSimilarStudentsUsingAvatarImages: The student " + student.getUniqueId() + " was not recognized"); | ||
| } | ||
| } catch (InterruptedException e) { | ||
| e.printStackTrace(); | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Find similar students | ||
| * Case 2: Student was added regularly but maybe on another tablet or due to some reason the authentication didn't recognize the student correctly in the numberOfTries | ||
| * ---> Use the meanFeatureVector as input for the cosineSimilarityScore calculation | ||
| * @param ppF | ||
| * @param tensorFlow | ||
| */ | ||
| private synchronized void findSimilarStudentsUsingMeanFeatureVector(PreProcessorFactory ppF, TensorFlow tensorFlow){ | ||
| // Iterate through all StudentImageCollectionEvents | ||
|
There was a problem hiding this comment. Add |
||
| List<StudentImageCollectionEvent> studentImageCollectionEvents = studentImageCollectionEventDao.loadAll(); | ||
| for (StudentImageCollectionEvent studentImageCollectionEvent : studentImageCollectionEvents){ | ||
| // Take the meanFeatureVector of the StudentImageCollectionEvent | ||
| List<Float> meanFeatureVectorList = gson.fromJson(studentImageCollectionEvent.getMeanFeatureVector(), new TypeToken<List<Float>>(){}.getType()); | ||
| Mat meanFeatureVector = Converters.vector_float_to_Mat(meanFeatureVectorList); | ||
| RecognitionThread recognitionThread = new RecognitionThread(tensorFlow, studentImageCollectionEventDao); | ||
| recognitionThread.setImg(meanFeatureVector); | ||
| // To indicate, that this Mat object contains the already extracted features and therefore this step can be skipped in the RecognitionThread | ||
| recognitionThread.setFeaturesAlreadyExtracted(true); | ||
| Student student = studentImageCollectionEvent.getStudent(); | ||
| Log.i(getClass().getName(), "findSimilarStudentsUsingMeanFeatureVector: recognitionThread will be started to recognize student: " + student.getUniqueId()); | ||
| recognitionThread.start(); | ||
| try { | ||
| recognitionThread.join(); | ||
| Student recognizedStudent = recognitionThread.getStudent(); | ||
| if (recognizedStudent != null){ | ||
| Log.i(getClass().getName(), "findSimilarStudentsUsingMeanFeatureVector: The student " + student.getUniqueId() + " has been recognized as " + recognizedStudent.getUniqueId()); | ||
| mergeSimilarStudents(student, recognizedStudent); | ||
| } else { | ||
| Log.i(getClass().getName(), "findSimilarStudentsUsingMeanFeatureVector: The student " + student.getUniqueId() + " was not recognized"); | ||
| } | ||
| } catch (InterruptedException e) { | ||
| e.printStackTrace(); | ||
| } | ||
| } | ||
| } | ||
|
|
@@ -365,5 +413,6 @@ public synchronized void findAndMergeSimilarStudents(){ | |
| */ | ||
| private synchronized void mergeSimilarStudents(Student student1, Student student2){ | ||
| Log.i(getClass().getName(), "mergeSimilarStudents: student1: " + student1.getUniqueId() + " student2: " + student2.getUniqueId()); | ||
| // TODO Implement merging of students (maybe in another class like StudentMergeHelper) | ||
|
There was a problem hiding this comment. This will be implemented in #314 |
||
| } | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,30 @@ | ||
| package org.literacyapp.service.synchronization; | ||
|
|
||
| import android.app.Service; | ||
| import android.app.job.JobParameters; | ||
| import android.app.job.JobService; | ||
| import android.content.Intent; | ||
| import android.os.IBinder; | ||
| import android.util.Log; | ||
|
|
||
| import org.literacyapp.authentication.TrainingHelper; | ||
|
|
||
| public class MergeSimilarStudentsJobService extends JobService { | ||
| public static boolean isRunning = false; | ||
|
|
||
| @Override | ||
| public boolean onStartJob(JobParameters jobParameters) { | ||
| Log.i(getClass().getName(), "onStartJob"); | ||
| isRunning = true; | ||
| TrainingHelper trainingHelper = new TrainingHelper(getApplicationContext()); | ||
| trainingHelper.findAndMergeSimilarStudents(); | ||
| isRunning = false; | ||
| return false; | ||
| } | ||
|
|
||
| @Override | ||
| public boolean onStopJob(JobParameters jobParameters) { | ||
| Log.i(getClass().getName(), "onStopJob"); | ||
| return false; | ||
| } | ||
| } |
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Does an avatar image also include those taken manually in the fall-back registration activity? If so, you currently get a mix between 180px and 224px images. Will this inconsistency in image size/format cause problems?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
No, because our library uses the default value 224 for resizing the image to 224 x 224 if no sharedPreference with the key "key_faceSize" exists (this key doesn't exist in our app because we don't have the Settings from the library included)