Remove duplicates from playlist feature

app/src/main/java/org/schabi/newpipe/database/playlist/dao/PlaylistStreamDAO.java

@@ -83,4 +83,23 @@ default Flowable<List<PlaylistStreamEntity>> listByService(final int serviceId)
             + " GROUP BY " + JOIN_PLAYLIST_ID
             + " ORDER BY " + PLAYLIST_NAME + " COLLATE NOCASE ASC")
     Flowable<List<PlaylistMetadataEntry>> getPlaylistMetadata();
+
+    @RewriteQueriesToDropUnusedColumns
+    @Transaction
+    @Query("SELECT *, MIN(" + JOIN_INDEX + ") FROM " + STREAM_TABLE + " INNER JOIN "
+            + "(SELECT " + JOIN_STREAM_ID + "," + JOIN_INDEX
+            + " FROM " + PLAYLIST_STREAM_JOIN_TABLE
+            + " WHERE " + JOIN_PLAYLIST_ID + " = :playlistId)"
+            + " ON " + STREAM_ID + " = " + JOIN_STREAM_ID
+            + " LEFT JOIN "
+            + "(SELECT " + JOIN_STREAM_ID + " AS " + JOIN_STREAM_ID_ALIAS + ", "
+            + STREAM_PROGRESS_MILLIS
+            + " FROM " + STREAM_STATE_TABLE + " )"
+            + " ON " + STREAM_ID + " = " + JOIN_STREAM_ID_ALIAS
+            + " GROUP BY " + STREAM_ID
+            + " ORDER BY " + JOIN_INDEX + " ASC")
+    Flowable<List<PlaylistStreamEntry>> getStreamsWithoutDuplicates(long playlistId);
+
+
+
 }

app/src/main/java/org/schabi/newpipe/local/playlist/LocalPlaylistFragment.java

@@ -367,6 +367,8 @@ public boolean onOptionsItemSelected(final MenuItem item) {
                         .create()
                         .show();
             }
+        } else if (item.getItemId() == R.id.menu_item_remove_duplicates) {
+            openRemoveDuplicatesDialog();
         } else {
             return super.onOptionsItemSelected(item);
         }
@@ -621,6 +623,31 @@ private void updateThumbnailUrl() {
         changeThumbnailUrl(newThumbnailUrl);
     }
 
+
+    private void openRemoveDuplicatesDialog() {
+        final AlertDialog.Builder builder = new AlertDialog.Builder(this.getActivity());
+
+        builder.setTitle(R.string.remove_duplicates_title)
+                .setMessage(R.string.remove_duplicates_message)
+                .setPositiveButton(android.R.string.yes, (dialog, i) -> {
+                    removeDuplicatesInPlaylist();
+                })
+                .setNeutralButton(R.string.cancel, null);
+
+        builder.create().show();
+    }
+
+    private void removeDuplicatesInPlaylist() {
+        final List<PlaylistStreamEntry> itemsToKeep = playlistManager
+                .getDistinctPlaylistStreams(playlistId).blockingFirst();
+
+        itemListAdapter.clearStreamItemList();
+        itemListAdapter.addItems(itemsToKeep);
+        setVideoCount(itemListAdapter.getItemsList().size());
+
+        saveChanges();
+    }
+
     private void deleteItem(final PlaylistStreamEntry item) {
         if (itemListAdapter == null) {
             return;

app/src/main/java/org/schabi/newpipe/local/playlist/LocalPlaylistManager.java

@@ -86,6 +86,11 @@ public Flowable<List<PlaylistMetadataEntry>> getPlaylists() {
         return playlistStreamTable.getPlaylistMetadata().subscribeOn(Schedulers.io());
     }
 
+    public Flowable<List<PlaylistStreamEntry>> getDistinctPlaylistStreams(final long playlistId) {
+        return playlistStreamTable
+                .getStreamsWithoutDuplicates(playlistId).subscribeOn(Schedulers.io());
+    }
+
     public Flowable<List<PlaylistStreamEntry>> getPlaylistStreams(final long playlistId) {
         return playlistStreamTable.getOrderedStreamsOf(playlistId).subscribeOn(Schedulers.io());
     }

app/src/main/res/menu/menu_local_playlist.xml

@@ -12,8 +12,14 @@
         android:id="@+id/menu_item_rename_playlist"
         android:title="@string/rename_playlist"
         app:showAsAction="never" />
+
     <item
         android:id="@+id/menu_item_remove_watched"
         android:title="@string/remove_watched"
         app:showAsAction="never" />
+
+    <item
+        android:id="@+id/menu_item_remove_duplicates"
+        android:title="@string/remove_duplicates"
+        app:showAsAction="never" />
 </menu>

app/src/main/res/values/strings.xml

@@ -626,6 +626,9 @@
     <string name="systems_language">System default</string>
     <string name="remove_watched">Remove watched</string>
     <string name="remove_watched_popup_title">Remove watched videos?</string>
+    <string name="remove_duplicates">Remove duplicates</string>
+    <string name="remove_duplicates_title">Remove duplicates?</string>
+    <string name="remove_duplicates_message">Do you want to remove all duplicate streams in this playlist?</string>
     <string name="remove_watched_popup_warning">Videos that have been watched before and after being added to the playlist will be removed.
 \nAre you sure\? This cannot be undone!</string>
     <string name="remove_watched_popup_yes_and_partially_watched_videos">Yes, and partially watched videos</string>