Remove duplicates from playlist feature

app/src/main/java/org/schabi/newpipe/database/playlist/dao/PlaylistStreamDAO.java

@@ -97,6 +97,23 @@ default Flowable<List<PlaylistStreamEntity>> listByService(final int serviceId)
             + " ORDER BY " + PLAYLIST_NAME + " COLLATE NOCASE ASC")
     Flowable<List<PlaylistMetadataEntry>> getPlaylistMetadata();
 
+    @RewriteQueriesToDropUnusedColumns
+    @Transaction
+    @Query("SELECT *, MIN(" + JOIN_INDEX + ")"
+            + " FROM " + STREAM_TABLE + " INNER JOIN"
+            + " (SELECT " + JOIN_STREAM_ID + "," + JOIN_INDEX
+            + " FROM " + PLAYLIST_STREAM_JOIN_TABLE
+            + " WHERE " + JOIN_PLAYLIST_ID + " = :playlistId)"
+            + " ON " + STREAM_ID + " = " + JOIN_STREAM_ID
+            + " LEFT JOIN "
+            + "(SELECT " + JOIN_STREAM_ID + " AS " + JOIN_STREAM_ID_ALIAS + ", "
+            + STREAM_PROGRESS_MILLIS
+            + " FROM " + STREAM_STATE_TABLE + " )"
+            + " ON " + STREAM_ID + " = " + JOIN_STREAM_ID_ALIAS
+            + " GROUP BY " + STREAM_ID
+            + " ORDER BY MIN(" + JOIN_INDEX + ") ASC")
+    Flowable<List<PlaylistStreamEntry>> getStreamsWithoutDuplicates(long playlistId);
+
     @Transaction
     @Query("SELECT " + PLAYLIST_TABLE + "." + PLAYLIST_ID + ", "
             + PLAYLIST_NAME + ", "

app/src/main/java/org/schabi/newpipe/local/playlist/LocalPlaylistFragment.java

@@ -95,8 +95,8 @@ public class LocalPlaylistFragment extends BaseLocalListFragment<List<PlaylistSt
     private AtomicBoolean isLoadingComplete;
     /* Has the playlist been modified (e.g. items reordered or deleted) */
     private AtomicBoolean isModified;
-    /* Is the playlist currently being processed to remove watched videos */
-    private boolean isRemovingWatched = false;
+    /* Flag to prevent simultaneous rewrites of the playlist */
+    private boolean isRewritingPlaylist = false;
 
     public static LocalPlaylistFragment getInstance(final long playlistId, final String name) {
         final LocalPlaylistFragment instance = new LocalPlaylistFragment();
@@ -353,7 +353,7 @@ public boolean onOptionsItemSelected(final MenuItem item) {
         } else if (item.getItemId() == R.id.menu_item_rename_playlist) {
             createRenameDialog();
         } else if (item.getItemId() == R.id.menu_item_remove_watched) {
-            if (!isRemovingWatched) {
+            if (!isRewritingPlaylist) {
                 new AlertDialog.Builder(requireContext())
                         .setMessage(R.string.remove_watched_popup_warning)
                         .setTitle(R.string.remove_watched_popup_title)
@@ -367,6 +367,10 @@ public boolean onOptionsItemSelected(final MenuItem item) {
                         .create()
                         .show();
             }
+        } else if (item.getItemId() == R.id.menu_item_remove_duplicates) {
+            if (!isRewritingPlaylist) {
+                openRemoveDuplicatesDialog();
+            }
         } else {
             return super.onOptionsItemSelected(item);
         }
@@ -388,10 +392,10 @@ public void sharePlaylist() {
     }
 
     public void removeWatchedStreams(final boolean removePartiallyWatched) {
-        if (isRemovingWatched) {
+        if (isRewritingPlaylist) {
             return;
         }
-        isRemovingWatched = true;
+        isRewritingPlaylist = true;
         showLoading();
 
         final var recordManager = new HistoryRecordManager(getContext());
@@ -469,7 +473,7 @@ public void removeWatchedStreams(final boolean removePartiallyWatched) {
                     }
 
                     hideLoading();
-                    isRemovingWatched = false;
+                    isRewritingPlaylist = false;
                 }, throwable -> showError(new ErrorInfo(throwable, UserAction.REQUESTED_BOOKMARK,
                         "Removing watched videos, partially watched=" + removePartiallyWatched))));
     }
@@ -628,6 +632,43 @@ private void updateThumbnailUrl() {
         changeThumbnailUrl(newThumbnailUrl, false);
     }
 
+    private void openRemoveDuplicatesDialog() {
+        final AlertDialog.Builder builder = new AlertDialog.Builder(this.getActivity());
+
+        builder.setTitle(R.string.remove_duplicates_title)
+                .setMessage(R.string.remove_duplicates_message)
+                .setPositiveButton(R.string.ok,
+                        (dialog, i) -> removeDuplicatesInPlaylist())
+                .setNeutralButton(R.string.cancel, null);
+
+        builder.create().show();
+    }
+
+    private void removeDuplicatesInPlaylist() {
+        if (isRewritingPlaylist) {
+            return;
+        }
+        isRewritingPlaylist = true;
+        showLoading();
+
+        final var streamsMaybe = playlistManager
+                .getDistinctPlaylistStreams(playlistId).firstElement();
+
+
+        disposables.add(streamsMaybe.subscribeOn(Schedulers.io())
+                .observeOn(AndroidSchedulers.mainThread())
+                .subscribe(itemsToKeep -> {
+                    itemListAdapter.clearStreamItemList();
+                    itemListAdapter.addItems(itemsToKeep);
+                    setVideoCount(itemListAdapter.getItemsList().size());
+                    saveChanges();
+
+                    hideLoading();
+                    isRewritingPlaylist = false;
+                }, throwable -> showError(new ErrorInfo(throwable, UserAction.REQUESTED_BOOKMARK,
+                        "Removing duplicated streams"))));
+    }
+
     private void deleteItem(final PlaylistStreamEntry item) {
         if (itemListAdapter == null) {
             return;

app/src/main/java/org/schabi/newpipe/local/playlist/LocalPlaylistManager.java

@@ -88,6 +88,11 @@ public Flowable<List<PlaylistMetadataEntry>> getPlaylists() {
         return playlistStreamTable.getPlaylistMetadata().subscribeOn(Schedulers.io());
     }
 
+    public Flowable<List<PlaylistStreamEntry>> getDistinctPlaylistStreams(final long playlistId) {
+        return playlistStreamTable
+                .getStreamsWithoutDuplicates(playlistId).subscribeOn(Schedulers.io());
+    }
+
     /**
      * Get playlists with attached information about how many times the provided stream is already
      * contained in each playlist.

app/src/main/res/menu/menu_local_playlist.xml

@@ -12,8 +12,14 @@
         android:id="@+id/menu_item_rename_playlist"
         android:title="@string/rename_playlist"
         app:showAsAction="never" />
+
     <item
         android:id="@+id/menu_item_remove_watched"
         android:title="@string/remove_watched"
         app:showAsAction="never" />
+
+    <item
+        android:id="@+id/menu_item_remove_duplicates"
+        android:title="@string/remove_duplicates"
+        app:showAsAction="never" />
 </menu>

app/src/main/res/values/strings.xml

@@ -631,6 +631,9 @@
     <string name="systems_language">System default</string>
     <string name="remove_watched">Remove watched</string>
     <string name="remove_watched_popup_title">Remove watched videos?</string>
+    <string name="remove_duplicates">Remove duplicates</string>
+    <string name="remove_duplicates_title">Remove duplicates?</string>
+    <string name="remove_duplicates_message">Do you want to remove all duplicate streams in this playlist?</string>
     <string name="remove_watched_popup_warning">Videos that have been watched before and after being added to the playlist will be removed.
 \nAre you sure\? This cannot be undone!</string>
     <string name="remove_watched_popup_yes_and_partially_watched_videos">Yes, and partially watched videos</string>