Skip to content

Commit

Permalink
Merge branch 'mixPL' into dev
Browse files Browse the repository at this point in the history
  • Loading branch information
@XiangRongLin
XiangRongLin committed Dec 12, 2020
2 parents 8ade913 + f9cf340 commit a2a1b88
Show file tree
Hide file tree
Showing 10 changed files with 817 additions and 29 deletions.
4 changes: 3 additions & 1 deletion ...xtractor/services/media_ccc/extractors/infoItems/MediaCCCConferenceInfoItemExtractor.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package org.schabi.newpipe.extractor.services.media_ccc.extractors.infoItems;

import com.grack.nanojson.JsonObject;

import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;

Expand All @@ -23,7 +25,7 @@ public long getSubscriberCount() {

@Override
public long getStreamCount() {
return -1;
return ListExtractor.ITEM_COUNT_UNKNOWN;
}

@Override
Expand Down
124 changes: 115 additions & 9 deletions ...tor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,10 @@
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import com.grack.nanojson.JsonWriter;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.schabi.newpipe.extractor.Page;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ContentNotAvailableException;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
Expand Down Expand Up @@ -35,6 +37,7 @@
import static org.schabi.newpipe.extractor.utils.Utils.HTTP;
import static org.schabi.newpipe.extractor.utils.Utils.HTTPS;
import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import static org.schabi.newpipe.extractor.utils.Utils.join;

/*
* Created by Christian Schabesberger on 02.03.16.
Expand All @@ -61,6 +64,12 @@ public class YoutubeParsingHelper {
private YoutubeParsingHelper() {
}

/**
* The official youtube app supports intents in this format, where after the ':' is the videoId.
* Accordingly there are other apps sharing streams in this format.
*/
public final static String BASE_YOUTUBE_INTENT_URL = "vnd.youtube";

private static final String HARDCODED_CLIENT_VERSION = "2.20200214.04.00";
private static String clientVersion;

Expand Down Expand Up @@ -192,6 +201,57 @@ public static OffsetDateTime parseDateFrom(String textualUploadDate) throws Pars
}
}

/**
* Checks if the given playlist id is a YouTube Mix (auto-generated playlist)
* Ids from a YouTube Mix start with "RD"
* @param playlistId
* @return Whether given id belongs to a YouTube Mix
*/
public static boolean isYoutubeMixId(final String playlistId) {
return playlistId.startsWith("RD") && !isYoutubeMusicMixId(playlistId);
}

/**
* Checks if the given playlist id is a YouTube Music Mix (auto-generated playlist)
* Ids from a YouTube Music Mix start with "RDAMVM"
* @param playlistId
* @return Whether given id belongs to a YouTube Music Mix
*/
public static boolean isYoutubeMusicMixId(final String playlistId) {
return playlistId.startsWith("RDAMVM");
}
/**
* Checks if the given playlist id is a YouTube Channel Mix (auto-generated playlist)
* Ids from a YouTube channel Mix start with "RDCM"
* @return Whether given id belongs to a YouTube Channel Mix
*/
public static boolean isYoutubeChannelMixId(final String playlistId) {
return playlistId.startsWith("RDCM");
}

/**
* Extracts the video id from the playlist id for Mixes.
* @throws ParsingException If the playlistId is a Channel Mix or not a mix.
*/
public static String extractVideoIdFromMixId(final String playlistId) throws ParsingException {
if (playlistId.startsWith("RDMM")) { //My Mix
return playlistId.substring(4);

} else if (playlistId.startsWith("RDAMVM")) { //Music mix
return playlistId.substring(6);

} else if (playlistId.startsWith("RMCM")) { //Channel mix
//Channel mix are build with RMCM{channelId}, so videoId can't be determined
throw new ParsingException("Video id could not be determined from mix id: " + playlistId);

} else if (playlistId.startsWith("RD")) { // Normal mix
return playlistId.substring(2);

} else { //not a mix
throw new ParsingException("Video id could not be determined from mix id: " + playlistId);
}
}

public static JsonObject getInitialData(String html) throws ParsingException {
try {
try {
Expand Down Expand Up @@ -416,10 +476,14 @@ public static String getUrlFromNavigationEndpoint(JsonObject navigationEndpoint)
} else if (navigationEndpoint.has("watchEndpoint")) {
StringBuilder url = new StringBuilder();
url.append("https://www.youtube.com/watch?v=").append(navigationEndpoint.getObject("watchEndpoint").getString("videoId"));
if (navigationEndpoint.getObject("watchEndpoint").has("playlistId"))
url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint").getString("playlistId"));
if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds"))
url.append("&t=").append(navigationEndpoint.getObject("watchEndpoint").getInt("startTimeSeconds"));
if (navigationEndpoint.getObject("watchEndpoint").has("playlistId")) {
url.append("&list=").append(navigationEndpoint.getObject("watchEndpoint")
.getString("playlistId"));
}
if (navigationEndpoint.getObject("watchEndpoint").has("startTimeSeconds")) {
url.append("&t=").append(navigationEndpoint.getObject("watchEndpoint")
.getInt("startTimeSeconds"));
}
return url.toString();
} else if (navigationEndpoint.has("watchPlaylistEndpoint")) {
return "https://www.youtube.com/playlist?list=" +
Expand Down Expand Up @@ -485,8 +549,8 @@ public static String fixThumbnailUrl(String thumbnailUrl) {
public static String getValidJsonResponseBody(final Response response)
throws ParsingException, MalformedURLException {
if (response.responseCode() == 404) {
throw new ContentNotAvailableException("Not found" +
" (\"" + response.responseCode() + " " + response.responseMessage() + "\")");
throw new ContentNotAvailableException("Not found"
+ " (\"" + response.responseCode() + " " + response.responseMessage() + "\")");
}

final String responseBody = response.responseBody();
Expand All @@ -506,22 +570,64 @@ public static String getValidJsonResponseBody(final Response response)
final String responseContentType = response.getHeader("Content-Type");
if (responseContentType != null
&& responseContentType.toLowerCase().contains("text/html")) {
throw new ParsingException("Got HTML document, expected JSON response" +
" (latest url was: \"" + response.latestUrl() + "\")");
throw new ParsingException("Got HTML document, expected JSON response"
+ " (latest url was: \"" + response.latestUrl() + "\")");
}

return responseBody;
}

public static Response getResponse(final String url, final Localization localization)
throws IOException, ExtractionException {
final Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));

final Response response = getDownloader().get(url, headers, localization);
getValidJsonResponseBody(response);

return response;
}

public static String extractCookieValue(final String cookieName, final Response response) {
final List<String> cookies = response.responseHeaders().get("Set-Cookie");
int startIndex;
String result = "";
for (final String cookie : cookies) {
startIndex = cookie.indexOf(cookieName);
if (startIndex != -1) {
result = cookie.substring(startIndex + cookieName.length() + "=".length(),
cookie.indexOf(";", startIndex));
}
}
return result;
}

public static JsonArray getJsonResponse(final String url, final Localization localization)
throws IOException, ExtractionException {
Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));
final Response response = getDownloader().get(url, headers, localization);

final String responseBody = getValidJsonResponseBody(response);
return toJsonArray(getValidJsonResponseBody(response));
}

public static JsonArray getJsonResponse(final Page page, final Localization localization)
throws IOException, ExtractionException {
final Map<String, List<String>> headers = new HashMap<>();
if (!isNullOrEmpty(page.getCookies())) {
headers.put("Cookie", Collections.singletonList(join(";", "=", page.getCookies())));
}
headers.put("X-YouTube-Client-Name", Collections.singletonList("1"));
headers.put("X-YouTube-Client-Version", Collections.singletonList(getClientVersion()));

final Response response = getDownloader().get(page.getUrl(), headers, localization);

return toJsonArray(getValidJsonResponseBody(response));
}

public static JsonArray toJsonArray(final String responseBody) throws ParsingException {
try {
return JsonParser.array().from(responseBody);
} catch (JsonParserException e) {
Expand Down
9 changes: 7 additions & 2 deletions extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeChannelExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeCommentsExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeFeedExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMixPlaylistExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeMusicSearchExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubePlaylistExtractor;
import org.schabi.newpipe.extractor.services.youtube.extractors.YoutubeSearchExtractor;
Expand Down Expand Up @@ -109,8 +110,12 @@ public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler) {
}

@Override
public PlaylistExtractor getPlaylistExtractor(ListLinkHandler linkHandler) {
return new YoutubePlaylistExtractor(this, linkHandler);
public PlaylistExtractor getPlaylistExtractor(final ListLinkHandler linkHandler) {
if (YoutubeParsingHelper.isYoutubeMixId(linkHandler.getId())) {
return new YoutubeMixPlaylistExtractor(this, linkHandler);
} else {
return new YoutubePlaylistExtractor(this, linkHandler);
}
}

@Override
Expand Down
3 changes: 2 additions & 1 deletion ...schabi/newpipe/extractor/services/youtube/extractors/YoutubeChannelInfoItemExtractor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

import com.grack.nanojson.JsonObject;

import org.schabi.newpipe.extractor.ListExtractor;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.services.youtube.linkHandler.YoutubeChannelLinkHandlerFactory;
Expand Down Expand Up @@ -86,7 +87,7 @@ public long getStreamCount() throws ParsingException {
try {
if (!channelInfoItem.has("videoCountText")) {
// Video count is not available, channel probably has no public uploads.
return -1;
return ListExtractor.ITEM_COUNT_UNKNOWN;
}

return Long.parseLong(Utils.removeNonDigitCharacters(getTextFromObject(
Expand Down
Loading

0 comments on commit a2a1b88

Please sign in to comment.