Merge pull request #52 from mikaelGusse/master

Add filters to Dolos integration in Radar

data/models.py

@@ -232,6 +232,38 @@ def valid_submissions(self):
     def invalid_submissions(self):
         return self.submissions.filter(invalid=True)
 
+    @property
+    def best_submissions(self):
+        students = self.submissions.values_list('student', flat=True).distinct()
+
+        best_submission_ids = []
+
+        for student in students:
+            best_submission = self.submissions.filter(student=student).order_by('-grade').first()
+            if best_submission:
+                best_submission_ids.append(best_submission.id)
+
+        best_submissions_queryset = self.submissions.filter(id__in=best_submission_ids)
+        print("Best submissions queryset:", best_submissions_queryset)
+        return best_submissions_queryset
+
+    @property
+    def flagged_submissions(self):
+        submission_ids_a = (Comparison.objects.filter(submission_a__exercise=self)
+                                            .filter(review__gte=5)
+                                            .select_related('submission_a', 'submission_b')
+                                            .values_list('submission_a', flat=True))
+        submission_ids_b = (Comparison.objects.filter(submission_b__exercise=self)
+                                            .filter(review__gte=5)
+                                            .select_related('submission_b')
+                                            .values_list('submission_b', flat=True))
+
+        submission_ids = list(set(submission_ids_a) | set(submission_ids_b))
+        print(submission_ids)
+        submissions = Submission.objects.filter(id__in=submission_ids)
+        return submissions
+
+
     @property
     def valid_matched_submissions(self):
         return self.valid_submissions.filter(matched=True)

review/templates/review/exercise.html

@@ -8,21 +8,37 @@ <h4>
     </a>
 </h4>
 
-<div class="dolos-button">
-	<a href="{% url 'dolos' course_key=exercise.course.key exercise_key=exercise.key %}" class="btn btn-default btn-xl"}">
-		View in Dolos
-	</a>
-
-	<div class="dolos-progress-parent">
-		<div class="dolos-progress"></div>
-		<div class="dolos-progress-text">Preparing files...</div>
+
+<div class="dolos-div">
+	<div class="dolos-filter-container">
+		<h4>View in Dolos</h4>
+		<p>Include only...</p>
+		<div class="filter-select" style="display: flex; flex-direction: row;">
+			<select id="dolos-filter" name="dolos-filter">
+				<option id="all" value="all" title="Include all submissions for all students in this exercise. WARNING: In very large exercises the analysis might fail">All</option>
+				<option id="best_submissions" value="best_submissions" selected="selected" title="Include only one submission per student, that being the one with the most points in A+">Best</option>
+				<option id="flagged" value="flagged" title="Include only submissions that have been tagged with the 'Plagiate' flag in Radar">Flagged</option>
+			</select>
+		</div>
+		<div class="dolos-button">
+			<a id="dolos-gen-button" href="{% url 'dolos' course_key=exercise.course.key exercise_key=exercise.key best_submissions='false' flagged='false' %}" class="btn btn-default btn-xl"}">
+				<p class="dolos-button-text">
+					Run analysis
+				</p>
+			</a>
+		</div>
 	</div>
 </div>
 
 <p class="dolos-description">
 	Dolos is a modern similarity detection tool for source code. Opening this exercise in Dolos will redirect you to an Aalto hosted version of Dolos. This feature is still in its early stages, so some issues might occur.
 </p>
 
+<div class="dolos-progress-parent">
+	<div class="dolos-progress"></div>
+	<div class="dolos-progress-text">Preparing files...</div>
+</div>
+
 <h4>Similarity distribution of matches</h4>
 <div class="histogram"></div>
 
@@ -49,10 +65,17 @@ <h4>Comparison pairs with highest similarity</h4>
 document.addEventListener('DOMContentLoaded', function() {
 	const dolosButton = document.querySelector('.dolos-button');
 	const viewInDolosButton = dolosButton.querySelector('a');
-	const dolosProgressParent = dolosButton.querySelector('.dolos-progress-parent');
+	const dolosProgressParent = document.querySelector('.dolos-progress-parent');
 
 	viewInDolosButton.addEventListener('click', function() {
-	  dolosProgressParent.style.display = 'block';
+		dolosProgressParent.style.display = 'block';
+
+		let bestSubmissions = document.getElementById('best_submissions').selected.toString();
+		let flagged = document.getElementById('flagged').selected.toString();
+
+		console.log(bestSubmissions, flagged);
+		const url = `${bestSubmissions}/${flagged}/gen_dolos`;
+		document.getElementById('dolos-gen-button').href = url;
 	});
 
 	// Hide dolos-progress-parent initially

review/urls.py

@@ -100,7 +100,7 @@
         name='comparison',
     ),
     re_path(
-        r'^(?P<course_key>\w+)/(?P<exercise_key>\w+)/gen_dolos$',
+        r'^(?P<course_key>\w+)/(?P<exercise_key>\w+)/(?P<best_submissions>\w+)/(?P<flagged>\w+)/gen_dolos$',
         generate_dolos_view,
         name='dolos',
     ),

review/views.py

@@ -478,15 +478,15 @@ def download_file(output_dir, submission, local_course):
     p_config = provider_config(local_course.provider)
     get_submission_text = configured_function(p_config, "get_submission_text")
 
-    with open(os.path.join(output_dir, filename + "|" + str(submission.id)), 'w') as f:
+    with open(os.path.join(output_dir, filename + "|" + str( "Points: " + str(submission.grade))), 'w') as f:
         submission_text = get_submission_text(submission, p_config)
         print("Writing something with length: ", len(submission_text))
         f.write(submission_text)
 
-def download_files(output_dir, local_exercise, local_course):
+def download_files(output_dir, local_exercise, local_course, submissions):
     with concurrent.futures.ThreadPoolExecutor() as executor:
         futures = [executor.submit(download_file, output_dir, sub, local_course) for
-                   sub in local_exercise.valid_submissions | local_exercise.invalid_submissions]
+                   sub in submissions]
         concurrent.futures.wait(futures)
 
 def zip_files(directory, output_dir):
@@ -505,17 +505,15 @@ def zip_files(directory, output_dir):
                 zip_handle.write(file_path, arcname=filename)
 
 
-def write_metadata_for_dolos(exercise_directory, local_exercise) -> None:
-    submissions = local_exercise.valid_submissions | local_exercise.invalid_submissions
-
+def write_metadata_for_dolos(exercise_directory, local_exercise, submissions) -> None:
     # Write metadata to CSV file
     with open(exercise_directory + '/info.csv', 'w', newline='') as csvfile:
         writer = csv.writer(csvfile)
 
         writer.writerow(['filename', 'label', 'created_at'])
 
         for submission in submissions:
-            filename = "student" + submission.student.key + "|" + str(submission.id)
+            filename = "student" + submission.student.key + "|" + "Points: " + str(submission.grade)
             created_at = submission.provider_submission_time
 
             if isinstance(created_at, datetime.datetime):
@@ -536,7 +534,9 @@ def go_to_dolos_view(request, course_key=None, exercise_key=None) -> HttpRespons
 
 
 @access_resource
-def generate_dolos_view(request, course_key=None, exercise_key=None, course=None, exercise=None, ) -> HttpResponse:
+def generate_dolos_view(
+    request, course_key=None, exercise_key=None, course=None, exercise=None, best_submissions=False, flagged=False
+                        ) -> HttpResponse:
     """
     Create a Dolos report of this exercise and redirect to the report visualization
     """
@@ -549,8 +549,15 @@ def generate_dolos_view(request, course_key=None, exercise_key=None, course=None
     if not os.path.exists(new_submissions_dir):
         os.mkdir(new_submissions_dir)
 
-    download_files(new_submissions_dir, exercise, course)
-    write_metadata_for_dolos(new_submissions_dir, exercise)
+    submissions = (exercise.valid_submissions | exercise.invalid_submissions)
+    if best_submissions == 'true':
+        submissions = exercise.best_submissions
+    elif flagged == 'true':
+        submissions = exercise.flagged_submissions
+    print("Submissions", submissions)
+
+    download_files(new_submissions_dir, exercise, course, submissions.distinct())
+    write_metadata_for_dolos(new_submissions_dir, exercise, submissions.distinct())
     zip_files(new_submissions_dir, temp_submissions_dir)
 
     timestamp = time.time()

static/main.css

@@ -126,8 +126,24 @@ pre.sample-highlight { display:inline-block; }
   animation: bounce 1s ease-in-out infinite;
 }
 
-.dolos-button {
+.dolos-div{
   display: flex;
+  align-items: center;
+}
+
+.dolos-filter-container{
+  font-size: 14px;
+  line-height: 1.5;
+  color: #333;
+  overflow-wrap: break-word;
+  max-width: 500px;
+  padding: 10px;
+  border: 1px solid #e0e0e0;
+  background-color: #f4f4f4;
+}
+
+.dolos-button {
+  margin-top: 10px;
 }
 
 .dolos-progress-parent {
@@ -138,7 +154,6 @@ pre.sample-highlight { display:inline-block; }
   font-size: 14px;
   line-height: 1.5;
   color: #333;
-  margin-bottom: 20px;
   overflow-wrap: break-word;
   max-width: 500px;
   padding: 10px;