diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index c17eef7..fa17da7 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -52,7 +52,7 @@ android:name=".content.task.StarActivity" android:screenOrientation="fullSensor" /> - + - + - + + + \ No newline at end of file diff --git a/app/src/main/java/org/literacyapp/LiteracyApplication.java b/app/src/main/java/org/literacyapp/LiteracyApplication.java index c12021e..8c7d143 100644 --- a/app/src/main/java/org/literacyapp/LiteracyApplication.java +++ b/app/src/main/java/org/literacyapp/LiteracyApplication.java @@ -15,6 +15,7 @@ public class LiteracyApplication extends Application { public static final int FACE_RECOGNITION_TRAINING_JOB_ID = 0; public static final int CONTENT_SYNCRHONIZATION_JOB_ID = 1; public static final int AUTHENTICATION_JOB_ID = 2; + public static final int MERGE_SIMILAR_STUDENTS_JOB_ID = 3; public static final String PREF_APP_VERSION_CODE = "pref_app_version_code"; diff --git a/app/src/main/java/org/literacyapp/authentication/RecognitionThread.java b/app/src/main/java/org/literacyapp/authentication/RecognitionThread.java index c04fb47..36ef9be 100644 --- a/app/src/main/java/org/literacyapp/authentication/RecognitionThread.java +++ b/app/src/main/java/org/literacyapp/authentication/RecognitionThread.java @@ -30,16 +30,23 @@ public class RecognitionThread extends Thread { private Mat img; private Student student; private Gson gson; + private boolean featuresAlreadyExtracted; public RecognitionThread(TensorFlow tensorFlow, StudentImageCollectionEventDao studentImageCollectionEventDao) { this.tensorFlow = tensorFlow; this.studentImageCollectionEventDao = studentImageCollectionEventDao; gson = new Gson(); + featuresAlreadyExtracted = false; } @Override public void run() { - Mat featureVectorToRecognize = getFeatureVector(img); + Mat featureVectorToRecognize; + if (!featuresAlreadyExtracted){ + featureVectorToRecognize = getFeatureVector(img); + } else { + featureVectorToRecognize = img; + } student = getMostSimilarStudentIfInThreshold(featureVectorToRecognize); } @@ -60,6 +67,10 @@ public Student getStudent() { return student; } + public void setFeaturesAlreadyExtracted(boolean featuresAlreadyExtracted) { + this.featuresAlreadyExtracted = featuresAlreadyExtracted; + } + /** * Returns the recognized Student if the cosineSimilarity was above the threshold * @param featureVectorToRecognize diff --git a/app/src/main/java/org/literacyapp/authentication/TrainingHelper.java b/app/src/main/java/org/literacyapp/authentication/TrainingHelper.java index d38031f..f1d27a0 100644 --- a/app/src/main/java/org/literacyapp/authentication/TrainingHelper.java +++ b/app/src/main/java/org/literacyapp/authentication/TrainingHelper.java @@ -314,15 +314,29 @@ private File createAvatarFileFromStudentImage(StudentImage studentImage, Student /** * Find similar students - * Case 1: Student was added during fallback but in the meantime the same person has an existing StudentImageCollectionEvent and a new Student entry + * + * */ public synchronized void findAndMergeSimilarStudents(){ Log.i(getClass().getName(), "findAndMergeSimilarStudents"); PreProcessorFactory ppF = new PreProcessorFactory(context); + TensorFlow tensorFlow = getInitializedTensorFlow(); + findSimilarStudentsUsingAvatarImages(ppF, tensorFlow); + findSimilarStudentsUsingMeanFeatureVector(ppF, tensorFlow); + } + + /** + * Find similar students + * Case 1: Student was added during fallback but in the meantime the same person has an existing StudentImageCollectionEvent and a new Student entry + * ---> Use the avatar image as input for the recognition + * @param ppF + * @param tensorFlow + */ + private synchronized void findSimilarStudentsUsingAvatarImages(PreProcessorFactory ppF, TensorFlow tensorFlow){ + // Iterate through all Students List students = studentDao.loadAll(); - // Iterate through all students for (Student student : students){ - // Take the avatar image of the student + // Take the avatar image of the Student Mat avatarImage = Imgcodecs.imread(student.getAvatar()); // Search for faces in the avatar image List faceImages = ppF.getCroppedImage(avatarImage); @@ -334,26 +348,60 @@ public synchronized void findAndMergeSimilarStudents(){ Rect[] faces = ppF.getFacesForRecognition(); if (faces != null && faces.length == 1) { // Proceed if exactly one face rectangle exists - RecognitionThread recognitionThread = new RecognitionThread(getInitializedTensorFlow(), studentImageCollectionEventDao); + RecognitionThread recognitionThread = new RecognitionThread(tensorFlow, studentImageCollectionEventDao); recognitionThread.setImg(faceImage); - Log.i(getClass().getName(), "findAndMergeSimilarStudents: recognitionThread will be started to recognize student: " + student.getUniqueId()); + Log.i(getClass().getName(), "findSimilarStudentsUsingAvatarImages: recognitionThread will be started to recognize student: " + student.getUniqueId()); recognitionThread.start(); try { recognitionThread.join(); Student recognizedStudent = recognitionThread.getStudent(); if (recognizedStudent != null){ - Log.i(getClass().getName(), "findAndMergeSimilarStudents: The student " + student.getUniqueId() + " has been recognized as " + recognizedStudent.getUniqueId()); + Log.i(getClass().getName(), "findSimilarStudentsUsingAvatarImages: The student " + student.getUniqueId() + " has been recognized as " + recognizedStudent.getUniqueId()); mergeSimilarStudents(student, recognizedStudent); } else { - Log.i(getClass().getName(), "findAndMergeSimilarStudents: The student " + student.getUniqueId() + " was not recognized"); + Log.i(getClass().getName(), "findSimilarStudentsUsingAvatarImages: The student " + student.getUniqueId() + " was not recognized"); } } catch (InterruptedException e) { e.printStackTrace(); } - } } + } + } + } + /** + * Find similar students + * Case 2: Student was added regularly but maybe on another tablet or due to some reason the authentication didn't recognize the student correctly in the numberOfTries + * ---> Use the meanFeatureVector as input for the cosineSimilarityScore calculation + * @param ppF + * @param tensorFlow + */ + private synchronized void findSimilarStudentsUsingMeanFeatureVector(PreProcessorFactory ppF, TensorFlow tensorFlow){ + // Iterate through all StudentImageCollectionEvents + List studentImageCollectionEvents = studentImageCollectionEventDao.loadAll(); + for (StudentImageCollectionEvent studentImageCollectionEvent : studentImageCollectionEvents){ + // Take the meanFeatureVector of the StudentImageCollectionEvent + List meanFeatureVectorList = gson.fromJson(studentImageCollectionEvent.getMeanFeatureVector(), new TypeToken>(){}.getType()); + Mat meanFeatureVector = Converters.vector_float_to_Mat(meanFeatureVectorList); + RecognitionThread recognitionThread = new RecognitionThread(tensorFlow, studentImageCollectionEventDao); + recognitionThread.setImg(meanFeatureVector); + // To indicate, that this Mat object contains the already extracted features and therefore this step can be skipped in the RecognitionThread + recognitionThread.setFeaturesAlreadyExtracted(true); + Student student = studentImageCollectionEvent.getStudent(); + Log.i(getClass().getName(), "findSimilarStudentsUsingMeanFeatureVector: recognitionThread will be started to recognize student: " + student.getUniqueId()); + recognitionThread.start(); + try { + recognitionThread.join(); + Student recognizedStudent = recognitionThread.getStudent(); + if (recognizedStudent != null){ + Log.i(getClass().getName(), "findSimilarStudentsUsingMeanFeatureVector: The student " + student.getUniqueId() + " has been recognized as " + recognizedStudent.getUniqueId()); + mergeSimilarStudents(student, recognizedStudent); + } else { + Log.i(getClass().getName(), "findSimilarStudentsUsingMeanFeatureVector: The student " + student.getUniqueId() + " was not recognized"); + } + } catch (InterruptedException e) { + e.printStackTrace(); } } } @@ -365,5 +413,6 @@ public synchronized void findAndMergeSimilarStudents(){ */ private synchronized void mergeSimilarStudents(Student student1, Student student2){ Log.i(getClass().getName(), "mergeSimilarStudents: student1: " + student1.getUniqueId() + " student2: " + student2.getUniqueId()); + // TODO Implement merging of students (maybe in another class like StudentMergeHelper) } } diff --git a/app/src/main/java/org/literacyapp/receiver/BootReceiver.java b/app/src/main/java/org/literacyapp/receiver/BootReceiver.java index 23a28ee..6a2eafd 100644 --- a/app/src/main/java/org/literacyapp/receiver/BootReceiver.java +++ b/app/src/main/java/org/literacyapp/receiver/BootReceiver.java @@ -13,9 +13,11 @@ import org.literacyapp.service.FaceRecognitionTrainingJobService; import org.literacyapp.service.ScreenOnService; import org.literacyapp.service.synchronization.AuthenticationJobService; +import org.literacyapp.service.synchronization.MergeSimilarStudentsJobService; public class BootReceiver extends BroadcastReceiver { public static final int MINUTES_BETWEEN_AUTHENTICATIONS = 30; + private static final int MINUTES_BETWEEN_FACE_RECOGNITION_TRAININGS = 15; @Override public void onReceive(Context context, Intent intent) { @@ -37,9 +39,12 @@ public void onReceive(Context context, Intent intent) { // Initiate background job for face recognition training scheduleFaceRecognitionTranining(context); - // Initiate authentication job for face recognition authentication + // Initiate background job for face recognition authentication scheduleAuthentication(context, MINUTES_BETWEEN_AUTHENTICATIONS); + // Initiate background job for merging similar students + scheduleMergeSimilarStudents(context); + // Start service for detecting when the screen is turned on Intent screenOnServiceIntent = new Intent(context, ScreenOnService.class); context.startService(screenOnServiceIntent); @@ -50,7 +55,7 @@ public void onReceive(Context context, Intent intent) { public static void scheduleFaceRecognitionTranining(Context context){ ComponentName componentNameFaceRecognitionTranining = new ComponentName(context, FaceRecognitionTrainingJobService.class); JobInfo.Builder builderFaceRecognitionTranining = new JobInfo.Builder(LiteracyApplication.FACE_RECOGNITION_TRAINING_JOB_ID, componentNameFaceRecognitionTranining); - int faceRecognitionTrainingPeriodic = 15 * 60 * 1000; + int faceRecognitionTrainingPeriodic = MINUTES_BETWEEN_FACE_RECOGNITION_TRAININGS * 60 * 1000; builderFaceRecognitionTranining.setPeriodic(faceRecognitionTrainingPeriodic); // Every 15 minutes JobInfo faceRecognitionTrainingJobInfo = builderFaceRecognitionTranining.build(); JobScheduler jobSchedulerFaceRecognitionTranining = (JobScheduler) context.getSystemService(Context.JOB_SCHEDULER_SERVICE); @@ -68,4 +73,15 @@ public static void scheduleAuthentication(Context context, int minutesBetweenAut jobSchedulerAuthentication.schedule(authenticationJobInfo); Log.i(context.getClass().getName(), "AUTHENTICATION_JOB with ID " + LiteracyApplication.AUTHENTICATION_JOB_ID + " has been scheduled with periodic time = " + authenticationPeriodic); } + + public static void scheduleMergeSimilarStudents(Context context){ + ComponentName componentNameMergeSimilarStudents = new ComponentName(context, MergeSimilarStudentsJobService.class); + JobInfo.Builder builderMergeSimilarStudents = new JobInfo.Builder(LiteracyApplication.MERGE_SIMILAR_STUDENTS_JOB_ID, componentNameMergeSimilarStudents); + boolean requiresCharging = true; + builderMergeSimilarStudents.setRequiresCharging(requiresCharging); + JobInfo mergeSimilarStudentsJobInfo = builderMergeSimilarStudents.build(); + JobScheduler jobSchedulerMergeSimilarStudents = (JobScheduler) context.getSystemService(Context.JOB_SCHEDULER_SERVICE); + jobSchedulerMergeSimilarStudents.schedule(mergeSimilarStudentsJobInfo); + Log.i(context.getClass().getName(), "MERGE_SIMILAR_STUDENTS_JOB with ID " + LiteracyApplication.MERGE_SIMILAR_STUDENTS_JOB_ID + " has been scheduled with requiresCharging = " + requiresCharging); + } } diff --git a/app/src/main/java/org/literacyapp/service/FaceRecognitionTrainingJobService.java b/app/src/main/java/org/literacyapp/service/FaceRecognitionTrainingJobService.java index 9013d1e..f2970ce 100644 --- a/app/src/main/java/org/literacyapp/service/FaceRecognitionTrainingJobService.java +++ b/app/src/main/java/org/literacyapp/service/FaceRecognitionTrainingJobService.java @@ -20,7 +20,6 @@ public boolean onStartJob(JobParameters jobParameters) { TrainingHelper trainingHelper = new TrainingHelper(getApplicationContext()); trainingHelper.extractFeatures(); trainingHelper.trainClassifier(); - isRunning = false; return false; } diff --git a/app/src/main/java/org/literacyapp/service/synchronization/AuthenticationJobService.java b/app/src/main/java/org/literacyapp/service/synchronization/AuthenticationJobService.java index 2f1434e..92c5b08 100644 --- a/app/src/main/java/org/literacyapp/service/synchronization/AuthenticationJobService.java +++ b/app/src/main/java/org/literacyapp/service/synchronization/AuthenticationJobService.java @@ -31,7 +31,7 @@ public boolean onStartJob(JobParameters jobParameters) { Log.i(getClass().getName(), "onStartJob"); if (!isScreenTurnedOff()){ if (didTheMinimumTimePassSinceLastAuthentication()){ - if (!rescheduleIfTrainingJobServiceIsRunning()){ + if (!rescheduleIfCpuIntensiveServicesAreRunning()){ Intent authenticationIntent = new Intent(getApplicationContext(), AuthenticationActivity.class); authenticationIntent.setFlags(Intent.FLAG_ACTIVITY_NEW_TASK); startActivity(authenticationIntent); @@ -53,14 +53,21 @@ public boolean onStopJob(JobParameters jobParameters) { return false; } - private boolean rescheduleIfTrainingJobServiceIsRunning(){ + private boolean rescheduleIfCpuIntensiveServicesAreRunning(){ boolean isFaceRecognitionTrainingJobServiceRunning = FaceRecognitionTrainingJobService.isRunning; Log.i(getClass().getName(), "isFaceRecognitionTrainingJobServiceRunning: " + isFaceRecognitionTrainingJobServiceRunning); - if (isFaceRecognitionTrainingJobServiceRunning){ + + boolean isMergeSimilarStudentsJobServiceRunning = MergeSimilarStudentsJobService.isRunning; + Log.i(getClass().getName(), "isMergeSimilarStudentsJobServiceRunning: " + isMergeSimilarStudentsJobServiceRunning); + + boolean isCpuIntensiveServiceRunning = (isFaceRecognitionTrainingJobServiceRunning || isMergeSimilarStudentsJobServiceRunning); + + if (isCpuIntensiveServiceRunning){ BootReceiver.scheduleAuthentication(getApplicationContext(), BootReceiver.MINUTES_BETWEEN_AUTHENTICATIONS); - Log.i(getClass().getName(), "The AuthenticationJobService has been rescheduled, because the CPU hungry FaceRecognitionTrainingJobService is running at the moment."); + Log.i(getClass().getName(), "The AuthenticationJobService has been rescheduled, because a Cpu intensive service is running at the moment."); } - return isFaceRecognitionTrainingJobServiceRunning; + + return isCpuIntensiveServiceRunning; } private boolean isScreenTurnedOff(){ diff --git a/app/src/main/java/org/literacyapp/service/synchronization/MergeSimilarStudentsJobService.java b/app/src/main/java/org/literacyapp/service/synchronization/MergeSimilarStudentsJobService.java new file mode 100644 index 0000000..27e7fdb --- /dev/null +++ b/app/src/main/java/org/literacyapp/service/synchronization/MergeSimilarStudentsJobService.java @@ -0,0 +1,30 @@ +package org.literacyapp.service.synchronization; + +import android.app.Service; +import android.app.job.JobParameters; +import android.app.job.JobService; +import android.content.Intent; +import android.os.IBinder; +import android.util.Log; + +import org.literacyapp.authentication.TrainingHelper; + +public class MergeSimilarStudentsJobService extends JobService { + public static boolean isRunning = false; + + @Override + public boolean onStartJob(JobParameters jobParameters) { + Log.i(getClass().getName(), "onStartJob"); + isRunning = true; + TrainingHelper trainingHelper = new TrainingHelper(getApplicationContext()); + trainingHelper.findAndMergeSimilarStudents(); + isRunning = false; + return false; + } + + @Override + public boolean onStopJob(JobParameters jobParameters) { + Log.i(getClass().getName(), "onStopJob"); + return false; + } +}