From 26f9d5a7d49f80e4187bd8f11c65f5f4bbf02ead Mon Sep 17 00:00:00 2001 From: Marco Sirabella Date: Sun, 19 May 2024 04:06:09 -0400 Subject: [PATCH 1/6] Add support for on.soundcloud.com urls Fixes #1178 --- .../soundcloud/SoundcloudParsingHelper.java | 18 +++++++++++++++++- .../SoundcloudStreamLinkHandlerFactory.java | 4 +++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index 930d79f512..fadc0dc77d 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -45,6 +45,7 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import java.util.regex.Pattern; public final class SoundcloudParsingHelper { // CHECKSTYLE:OFF @@ -178,6 +179,8 @@ public static String resolveUrlWithEmbedPlayer(final String apiUrl) throws IOExc .attr("abs:href"); } + private static final Pattern ON_PATTERN = Pattern.compile("(https?)?://on\\."); + /** * Fetch the widget API with the url and return the id (like the id from the json API). * @@ -185,8 +188,21 @@ public static String resolveUrlWithEmbedPlayer(final String apiUrl) throws IOExc */ public static String resolveIdWithWidgetApi(final String urlString) throws IOException, ParsingException { - // Remove the tailing slash from URLs due to issues with the SoundCloud API String fixedUrl = urlString; + + // if URL is an on.soundcloud link, to a request to resolve the redirect + + if (ON_PATTERN.matcher(fixedUrl).find()) { + // fixedUrl = Utils.followOnFromUrl(fixedUrl); + try { + fixedUrl = NewPipe.getDownloader().head(fixedUrl).latestUrl(); + } catch (final ExtractionException e) { + throw new ParsingException( + "Could not follow redirect", e); + } + } + + // Remove the tailing slash from URLs due to issues with the SoundCloud API if (fixedUrl.charAt(fixedUrl.length() - 1) == '/') { fixedUrl = fixedUrl.substring(0, fixedUrl.length() - 1); } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java index 64b34ef7a3..7b1e1bab97 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java @@ -9,10 +9,12 @@ public final class SoundcloudStreamLinkHandlerFactory extends LinkHandlerFactory { private static final SoundcloudStreamLinkHandlerFactory INSTANCE = new SoundcloudStreamLinkHandlerFactory(); - private static final String URL_PATTERN = "^https?://(www\\.|m\\.)?soundcloud.com/[0-9a-z_-]+" + private static final String URL_PATTERN = "^https?://(www\\.|m\\.|on\\.)?" + + "soundcloud.com/[0-9a-z_-]+" + "/(?!(tracks|albums|sets|reposts|followers|following)/?$)[0-9a-z_-]+/?([#?].*)?$"; private static final String API_URL_PATTERN = "^https?://api-v2\\.soundcloud.com" + "/(tracks|albums|sets|reposts|followers|following)/([0-9a-z_-]+)/"; + private static final String ON_URL_PATTERN = "^https?://on.soundcloud.com/[0-9a-zA-Z]+$"; private SoundcloudStreamLinkHandlerFactory() { } From deb39e0f00b06be1c8b5e79b4ed5e4994f9ac6a8 Mon Sep 17 00:00:00 2001 From: Marco Sirabella Date: Sun, 19 May 2024 13:20:18 -0400 Subject: [PATCH 2/6] fixup! Add support for on.soundcloud.com urls --- .../soundcloud/SoundcloudParsingHelper.java | 13 +++++++------ .../SoundcloudStreamLinkHandlerFactory.java | 1 - 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index fadc0dc77d..8ad6b49c57 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -89,6 +89,10 @@ public final class SoundcloudParsingHelper { private static String clientId; public static final String SOUNDCLOUD_API_V2_URL = "https://api-v2.soundcloud.com/"; + private static final Pattern ON_URL_PATTERN = Pattern.compile( + "^https?://on.soundcloud.com/[0-9a-zA-Z]+$" + ); + private SoundcloudParsingHelper() { } @@ -179,8 +183,6 @@ public static String resolveUrlWithEmbedPlayer(final String apiUrl) throws IOExc .attr("abs:href"); } - private static final Pattern ON_PATTERN = Pattern.compile("(https?)?://on\\."); - /** * Fetch the widget API with the url and return the id (like the id from the json API). * @@ -190,15 +192,14 @@ public static String resolveIdWithWidgetApi(final String urlString) throws IOExc ParsingException { String fixedUrl = urlString; - // if URL is an on.soundcloud link, to a request to resolve the redirect + // if URL is an on.soundcloud link, do a request to resolve the redirect - if (ON_PATTERN.matcher(fixedUrl).find()) { - // fixedUrl = Utils.followOnFromUrl(fixedUrl); + if (ON_URL_PATTERN.matcher(fixedUrl).find()) { try { fixedUrl = NewPipe.getDownloader().head(fixedUrl).latestUrl(); } catch (final ExtractionException e) { throw new ParsingException( - "Could not follow redirect", e); + "Could not follow on.soundcloud.com redirect", e); } } diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java index 7b1e1bab97..e7809c52a1 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/linkHandler/SoundcloudStreamLinkHandlerFactory.java @@ -14,7 +14,6 @@ public final class SoundcloudStreamLinkHandlerFactory extends LinkHandlerFactory + "/(?!(tracks|albums|sets|reposts|followers|following)/?$)[0-9a-z_-]+/?([#?].*)?$"; private static final String API_URL_PATTERN = "^https?://api-v2\\.soundcloud.com" + "/(tracks|albums|sets|reposts|followers|following)/([0-9a-z_-]+)/"; - private static final String ON_URL_PATTERN = "^https?://on.soundcloud.com/[0-9a-zA-Z]+$"; private SoundcloudStreamLinkHandlerFactory() { } From cba405213f251a2416afe49822355e422e07dd5b Mon Sep 17 00:00:00 2001 From: Marco Sirabella Date: Sun, 19 May 2024 13:31:34 -0400 Subject: [PATCH 3/6] fixup! Add support for on.soundcloud.com urls --- .../services/soundcloud/SoundcloudParsingHelperTest.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java index e8fc629925..63481f0b1b 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java @@ -25,6 +25,9 @@ void resolveUrlWithEmbedPlayerTest() throws Exception { void resolveIdWithWidgetApiTest() throws Exception { assertEquals("26057743", SoundcloudParsingHelper.resolveIdWithWidgetApi("https://soundcloud.com/trapcity")); assertEquals("16069159", SoundcloudParsingHelper.resolveIdWithWidgetApi("https://soundcloud.com/nocopyrightsounds")); + + assertEquals("1818813498", SoundcloudParsingHelper.resolveIdWithWidgetApi("https://on.soundcloud.com/a8QmYdMnmxnsSTEp9")); + assertEquals("1468401502", SoundcloudParsingHelper.resolveIdWithWidgetApi("https://on.soundcloud.com/rdt7e")); } } \ No newline at end of file From ae18d416b0516e389ff7ccdb1d73392b77cd1feb Mon Sep 17 00:00:00 2001 From: Marco Sirabella Date: Sun, 19 May 2024 13:36:01 -0400 Subject: [PATCH 4/6] fixup! Add support for on.soundcloud.com urls --- .../services/soundcloud/SoundcloudParsingHelperTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java index 63481f0b1b..fe3b923ad2 100644 --- a/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java +++ b/extractor/src/test/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelperTest.java @@ -26,6 +26,7 @@ void resolveIdWithWidgetApiTest() throws Exception { assertEquals("26057743", SoundcloudParsingHelper.resolveIdWithWidgetApi("https://soundcloud.com/trapcity")); assertEquals("16069159", SoundcloudParsingHelper.resolveIdWithWidgetApi("https://soundcloud.com/nocopyrightsounds")); + assertEquals("26057743", SoundcloudParsingHelper.resolveIdWithWidgetApi("https://on.soundcloud.com/Rr2JyfFcYwbawpw49")); assertEquals("1818813498", SoundcloudParsingHelper.resolveIdWithWidgetApi("https://on.soundcloud.com/a8QmYdMnmxnsSTEp9")); assertEquals("1468401502", SoundcloudParsingHelper.resolveIdWithWidgetApi("https://on.soundcloud.com/rdt7e")); } From af63ec4111b6e3978e0720a0a707bac3098fd024 Mon Sep 17 00:00:00 2001 From: Marco Sirabella Date: Mon, 20 May 2024 14:18:18 -0400 Subject: [PATCH 5/6] fixup! fixup! Add support for on.soundcloud.com urls --- .../extractor/services/soundcloud/SoundcloudParsingHelper.java | 1 + 1 file changed, 1 insertion(+) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index 8ad6b49c57..201ef34ba6 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -197,6 +197,7 @@ public static String resolveIdWithWidgetApi(final String urlString) throws IOExc if (ON_URL_PATTERN.matcher(fixedUrl).find()) { try { fixedUrl = NewPipe.getDownloader().head(fixedUrl).latestUrl(); + fixedUrl = fixedUrl.split("\\?")[0]; // remove the query string } catch (final ExtractionException e) { throw new ParsingException( "Could not follow on.soundcloud.com redirect", e); From 17bbf4efba627dcf0e3d94081383fbcd820bb52e Mon Sep 17 00:00:00 2001 From: TobiGr Date: Mon, 22 Jul 2024 09:02:37 +0200 Subject: [PATCH 6/6] Modify comment --- .../services/soundcloud/SoundcloudParsingHelper.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java index 201ef34ba6..621bc360d3 100644 --- a/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java +++ b/extractor/src/main/java/org/schabi/newpipe/extractor/services/soundcloud/SoundcloudParsingHelper.java @@ -197,10 +197,10 @@ public static String resolveIdWithWidgetApi(final String urlString) throws IOExc if (ON_URL_PATTERN.matcher(fixedUrl).find()) { try { fixedUrl = NewPipe.getDownloader().head(fixedUrl).latestUrl(); - fixedUrl = fixedUrl.split("\\?")[0]; // remove the query string + // remove tracking params which are in the query string + fixedUrl = fixedUrl.split("\\?")[0]; } catch (final ExtractionException e) { - throw new ParsingException( - "Could not follow on.soundcloud.com redirect", e); + throw new ParsingException("Could not follow on.soundcloud.com redirect", e); } }