[SoundCloud] Detect whether there are any more search results

Add test for this edge case.
TeamNewPipe · Jul 20, 2023 · 428a1ec · 428a1ec
1 parent 986a3f7
commit 428a1ec
Show file tree

Hide file tree

Showing 2 changed files with 60 additions and 12 deletions.
diff --git a/...rg/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java b/...rg/schabi/newpipe/extractor/services/soundcloud/extractors/SoundcloudSearchExtractor.java
@@ -33,7 +33,9 @@
 import javax.annotation.Nonnull;
 
 public class SoundcloudSearchExtractor extends SearchExtractor {
-    private JsonArray initialSearchCollection;
+    private JsonObject initialSearchObject;
+    private static final String COLLECTION = "collection";
+    private static final String TOTAL_RESULTS = "total_results";
 
     public SoundcloudSearchExtractor(final StreamingService service,
                                      final SearchQueryHandler linkHandler) {
@@ -60,9 +62,15 @@ public List<MetaInfo> getMetaInfo() {
     @Nonnull
     @Override
     public InfoItemsPage<InfoItem> getInitialPage() throws IOException, ExtractionException {
-        return new InfoItemsPage<>(
-                collectItems(initialSearchCollection),
-                getNextPageFromCurrentUrl(getUrl(), currentOffset -> ITEMS_PER_PAGE));
+        if (initialSearchObject.getInt(TOTAL_RESULTS) > ITEMS_PER_PAGE) {
+            return new InfoItemsPage<>(
+                    collectItems(initialSearchObject.getArray(COLLECTION)),
+                    getNextPageFromCurrentUrl(getUrl(), currentOffset -> ITEMS_PER_PAGE));
+        } else {
+            return new InfoItemsPage<>(
+                    collectItems(initialSearchObject.getArray(COLLECTION)), null);
+        }
+
     }
 
     @Override
@@ -74,17 +82,29 @@ public InfoItemsPage<InfoItem> getPage(final Page page) throws IOException,
 
         final Downloader dl = getDownloader();
         final JsonArray searchCollection;
+        final int totalResults;
         try {
             final String response = dl.get(page.getUrl(), getExtractorLocalization())
                     .responseBody();
-            searchCollection = JsonParser.object().from(response).getArray("collection");
+            final JsonObject result = JsonParser.object().from(response);
+            searchCollection = result.getArray(COLLECTION);
+            totalResults = result.getInt(TOTAL_RESULTS);
         } catch (final JsonParserException e) {
             throw new ParsingException("Could not parse json response", e);
         }
+        final boolean hasNextPage;
+        try {
+            hasNextPage = getOffsetFromUrl(page.getUrl()) + ITEMS_PER_PAGE <= totalResults;
+        } catch (MalformedURLException | UnsupportedEncodingException e) {
+            throw new ParsingException("Could not get offset from page URL", e);
+        }
+        if (hasNextPage) {
+            return new InfoItemsPage<>(collectItems(searchCollection),
+                    getNextPageFromCurrentUrl(page.getUrl(),
+                            currentOffset -> currentOffset + ITEMS_PER_PAGE));
+        }
+        return new InfoItemsPage<>(collectItems(searchCollection), null);
 
-        return new InfoItemsPage<>(collectItems(searchCollection),
-                getNextPageFromCurrentUrl(page.getUrl(),
-                        currentOffset -> currentOffset + ITEMS_PER_PAGE));
     }
 
     @Override
@@ -94,12 +114,12 @@ public void onFetchPage(@Nonnull final Downloader downloader) throws IOException
         final String url = getUrl();
         try {
             final String response = dl.get(url, getExtractorLocalization()).responseBody();
-            initialSearchCollection = JsonParser.object().from(response).getArray("collection");
+            initialSearchObject = JsonParser.object().from(response);
         } catch (final JsonParserException e) {
             throw new ParsingException("Could not parse json response", e);
         }
 
-        if (initialSearchCollection.isEmpty()) {
+        if (initialSearchObject.getArray(COLLECTION).isEmpty()) {
             throw new SearchExtractor.NothingFoundException("Nothing found");
         }
     }
@@ -134,12 +154,16 @@ private InfoItemsCollector<InfoItem, InfoItemExtractor> collectItems(
     private Page getNextPageFromCurrentUrl(final String currentUrl,
                                            final IntUnaryOperator newPageOffsetCalculator)
             throws MalformedURLException, UnsupportedEncodingException {
-        final int currentPageOffset = Integer.parseInt(
-                    Parser.compatParseMap(new URL(currentUrl).getQuery()).get("offset"));
+        final int currentPageOffset = getOffsetFromUrl(currentUrl);
 
         return new Page(
                 currentUrl.replace(
                         "&offset=" + currentPageOffset,
                         "&offset=" + newPageOffsetCalculator.applyAsInt(currentPageOffset)));
     }
+
+    private int getOffsetFromUrl(final String url)
+            throws MalformedURLException, UnsupportedEncodingException {
+        return Integer.parseInt(Parser.compatParseMap(new URL(url).getQuery()).get("offset"));
+    }
 }
diff --git a/...rg/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java b/...rg/schabi/newpipe/extractor/services/soundcloud/search/SoundcloudSearchExtractorTest.java
@@ -1,5 +1,6 @@
 package org.schabi.newpipe.extractor.services.soundcloud.search;
 
+import static org.junit.jupiter.api.Assertions.assertFalse;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
 import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoDuplicatedItems;
@@ -181,4 +182,27 @@ void testIsVerified() throws IOException, ExtractionException {
             assertTrue(verified);
         }
     }
+
+    public static class NoNextPage extends DefaultSearchExtractorTest {
+
+        private static SearchExtractor extractor;
+        private static final String QUERY = "Dan at hor#berlgbd";
+
+        @BeforeAll
+        public static void setUp() throws Exception {
+            NewPipe.init(DownloaderTestImpl.getInstance());
+            extractor = SoundCloud.getSearchExtractor(QUERY);
+            extractor.fetchPage();
+        }
+
+        @Override public boolean expectedHasMoreItems() { return false; }
+        @Override public SearchExtractor extractor() throws Exception { return extractor; }
+        @Override public StreamingService expectedService() throws Exception { return SoundCloud; }
+        @Override public String expectedName() throws Exception { return QUERY; }
+        @Override public String expectedId() throws Exception { return QUERY; }
+        @Override public String expectedUrlContains() { return "soundcloud.com/search?q=" + urlEncode(QUERY); }
+        @Override public String expectedOriginalUrlContains() { return "soundcloud.com/search?q=" + urlEncode(QUERY); }
+        @Override public String expectedSearchString() { return QUERY; }
+        @Nullable @Override public String expectedSearchSuggestion() { return null; }
+    }
 }