Skip to content

Commit

Permalink
Showing 22 changed files with 244 additions and 349 deletions.
1 change: 1 addition & 0 deletions build.gradle
Original file line number Diff line number Diff line change
@@ -10,6 +10,7 @@ allprojects {

repositories {
jcenter()
maven { url "https://jitpack.io" }
}
}

4 changes: 2 additions & 2 deletions extractor/build.gradle
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
dependencies {
implementation project(':timeago-parser')

implementation 'com.grack:nanojson:1.1'
implementation 'com.github.TeamNewPipe:nanojson:1d9e1aea9049fc9f85e68b43ba39fe7be1c1f751'
implementation 'org.jsoup:jsoup:1.9.2'
implementation 'org.mozilla:rhino:1.7.7.1'
implementation 'com.github.spotbugs:spotbugs-annotations:3.1.0'
implementation 'org.nibor.autolink:autolink:0.8.0'

testImplementation 'junit:junit:4.12'
}
}
Original file line number Diff line number Diff line change
@@ -109,7 +109,7 @@ public String getNextPageUrl() throws IOException, ExtractionException {
public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
Response response = getDownloader().get(pageUrl);
JsonObject json = null;
if (null != response && !StringUtil.isBlank(response.responseBody())) {
if (response != null && !StringUtil.isBlank(response.responseBody())) {
try {
json = JsonParser.object().from(response.responseBody());
} catch (Exception e) {
@@ -169,7 +169,7 @@ private void setInitialData(String responseBody) throws ExtractionException {
try {
json = JsonParser.object().from(responseBody);
} catch (JsonParserException e) {
throw new ExtractionException("Unable to extract peertube channel data", e);
throw new ExtractionException("Unable to extract PeerTube channel data", e);
}
if (json == null) throw new ExtractionException("Unable to extract PeerTube channel data");
}
Original file line number Diff line number Diff line change
@@ -167,7 +167,7 @@ public List<VideoStream> getVideoStreams() throws IOException, ExtractionExcepti
assertPageFetched();
List<VideoStream> videoStreams = new ArrayList<>();
try {
JsonArray streams = json.getArray("files", new JsonArray());
JsonArray streams = json.getArray("files");
for (Object s : streams) {
if (!(s instanceof JsonObject)) continue;
JsonObject stream = (JsonObject) s;
Original file line number Diff line number Diff line change
@@ -16,6 +16,8 @@
import javax.annotation.Nonnull;
import java.io.IOException;

import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;

@SuppressWarnings("WeakerAccess")
public class SoundcloudChannelExtractor extends ChannelExtractor {
private String userId;
@@ -62,10 +64,7 @@ public String getAvatarUrl() {

@Override
public String getBannerUrl() {
return user.getObject("visuals", new JsonObject())
.getArray("visuals", new JsonArray())
.getObject(0, new JsonObject())
.getString("visual_url");
return user.getObject("visuals").getArray("visuals").getObject(0).getString("visual_url");
}

@Override
@@ -80,7 +79,7 @@ public long getSubscriberCount() {

@Override
public String getDescription() {
return user.getString("description", "");
return user.getString("description", EMPTY_STRING);
}

@Nonnull
Original file line number Diff line number Diff line change
@@ -3,6 +3,7 @@
import com.grack.nanojson.JsonObject;
import org.schabi.newpipe.extractor.channel.ChannelInfoItemExtractor;

import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;

public class SoundcloudChannelInfoItemExtractor implements ChannelInfoItemExtractor {
@@ -24,7 +25,7 @@ public String getUrl() {

@Override
public String getThumbnailUrl() {
String avatarUrl = itemObject.getString("avatar_url", "");
String avatarUrl = itemObject.getString("avatar_url", EMPTY_STRING);
String avatarUrlBetterResolution = avatarUrl.replace("large.jpg", "crop.jpg");
return avatarUrlBetterResolution;
}
@@ -41,6 +42,6 @@ public long getStreamCount() {

@Override
public String getDescription() {
return itemObject.getString("description", "");
return itemObject.getString("description", EMPTY_STRING);
}
}
Original file line number Diff line number Diff line change
@@ -28,6 +28,7 @@

import static java.util.Collections.singletonList;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;

public class SoundcloudParsingHelper {
@@ -256,17 +257,17 @@ public static String getStreamsFromApi(StreamInfoItemsCollector collector, Strin

@Nonnull
static String getUploaderUrl(JsonObject object) {
String url = object.getObject("user").getString("permalink_url", "");
String url = object.getObject("user").getString("permalink_url", EMPTY_STRING);
return replaceHttpWithHttps(url);
}

@Nonnull
static String getAvatarUrl(JsonObject object) {
String url = object.getObject("user", new JsonObject()).getString("avatar_url", "");
String url = object.getObject("user").getString("avatar_url", EMPTY_STRING);
return replaceHttpWithHttps(url);
}

public static String getUploaderName(JsonObject object) {
return object.getObject("user").getString("username", "");
return object.getObject("user").getString("username", EMPTY_STRING);
}
}
Original file line number Diff line number Diff line change
@@ -4,6 +4,7 @@
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.playlist.PlaylistInfoItemExtractor;

import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;

public class SoundcloudPlaylistInfoItemExtractor implements PlaylistInfoItemExtractor {
@@ -31,7 +32,7 @@ public String getUrl() {
public String getThumbnailUrl() throws ParsingException {
// Over-engineering at its finest
if (itemObject.isString(ARTWORK_URL_KEY)) {
final String artworkUrl = itemObject.getString(ARTWORK_URL_KEY, "");
final String artworkUrl = itemObject.getString(ARTWORK_URL_KEY, EMPTY_STRING);
if (!artworkUrl.isEmpty()) {
String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg");
return artworkUrlBetterResolution;
@@ -45,16 +46,16 @@ public String getThumbnailUrl() throws ParsingException {

// First look for track artwork url
if (trackObject.isString(ARTWORK_URL_KEY)) {
String artworkUrl = trackObject.getString(ARTWORK_URL_KEY, "");
String artworkUrl = trackObject.getString(ARTWORK_URL_KEY, EMPTY_STRING);
if (!artworkUrl.isEmpty()) {
String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg");
return artworkUrlBetterResolution;
}
}

// Then look for track creator avatar url
final JsonObject creator = trackObject.getObject(USER_KEY, new JsonObject());
final String creatorAvatar = creator.getString(AVATAR_URL_KEY, "");
final JsonObject creator = trackObject.getObject(USER_KEY);
final String creatorAvatar = creator.getString(AVATAR_URL_KEY, EMPTY_STRING);
if (!creatorAvatar.isEmpty()) return creatorAvatar;
}
} catch (Exception ignored) {
@@ -63,7 +64,7 @@ public String getThumbnailUrl() throws ParsingException {

try {
// Last resort, use user avatar url. If still not found, then throw exception.
return itemObject.getObject(USER_KEY).getString(AVATAR_URL_KEY, "");
return itemObject.getObject(USER_KEY).getString(AVATAR_URL_KEY, EMPTY_STRING);
} catch (Exception e) {
throw new ParsingException("Failed to extract playlist thumbnail url", e);
}
Original file line number Diff line number Diff line change
@@ -23,6 +23,7 @@
import java.net.URL;

import static org.schabi.newpipe.extractor.services.soundcloud.SoundcloudSearchQueryHandlerFactory.ITEMS_PER_PAGE;
import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;

public class SoundcloudSearchExtractor extends SearchExtractor {

@@ -84,7 +85,7 @@ private InfoItemsCollector<InfoItem, InfoItemExtractor> collectItems(JsonArray s
if (!(result instanceof JsonObject)) continue;
//noinspection ConstantConditions
JsonObject searchResult = (JsonObject) result;
String kind = searchResult.getString("kind", "");
String kind = searchResult.getString("kind", EMPTY_STRING);
switch (kind) {
case "user":
collector.commit(new SoundcloudChannelInfoItemExtractor(searchResult));
Original file line number Diff line number Diff line change
@@ -34,6 +34,8 @@

import javax.annotation.Nonnull;

import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;

public class SoundcloudStreamExtractor extends StreamExtractor {
private JsonObject track;

@@ -45,7 +47,7 @@ public SoundcloudStreamExtractor(StreamingService service, LinkHandler linkHandl
public void onFetchPage(@Nonnull Downloader downloader) throws IOException, ExtractionException {
track = SoundcloudParsingHelper.resolveFor(downloader, getOriginalUrl());

String policy = track.getString("policy", "");
String policy = track.getString("policy", EMPTY_STRING);
if (!policy.equals("ALLOW") && !policy.equals("MONETIZE")) {
throw new ContentNotAvailableException("Content not available: policy " + policy);
}
@@ -78,9 +80,9 @@ public DateWrapper getUploadDate() throws ParsingException {
@Nonnull
@Override
public String getThumbnailUrl() {
String artworkUrl = track.getString("artwork_url", "");
String artworkUrl = track.getString("artwork_url", EMPTY_STRING);
if (artworkUrl.isEmpty()) {
artworkUrl = track.getObject("user").getString("avatar_url", "");
artworkUrl = track.getObject("user").getString("avatar_url", EMPTY_STRING);
}
String artworkUrlBetterResolution = artworkUrl.replace("large.jpg", "crop.jpg");
return artworkUrlBetterResolution;
Original file line number Diff line number Diff line change
@@ -6,6 +6,7 @@
import org.schabi.newpipe.extractor.stream.StreamInfoItemExtractor;
import org.schabi.newpipe.extractor.stream.StreamType;

import static org.schabi.newpipe.extractor.utils.JsonUtils.EMPTY_STRING;
import static org.schabi.newpipe.extractor.utils.Utils.replaceHttpWithHttps;

public class SoundcloudStreamInfoItemExtractor implements StreamInfoItemExtractor {
@@ -62,7 +63,7 @@ public long getViewCount() {

@Override
public String getThumbnailUrl() {
String artworkUrl = itemObject.getString("artwork_url", "");
String artworkUrl = itemObject.getString("artwork_url", EMPTY_STRING);
if (artworkUrl.isEmpty()) {
artworkUrl = itemObject.getObject("user").getString("avatar_url");
}
Original file line number Diff line number Diff line change
@@ -72,22 +72,16 @@ public void onFetchPage(@Nonnull Downloader downloader) throws IOException, Extr
while (level < 3) {
final JsonArray jsonResponse = getJsonResponse(url, getExtractorLocalization());

final JsonObject endpoint = jsonResponse.getObject(1, EMPTY_OBJECT)
.getObject("response", EMPTY_OBJECT).getArray("onResponseReceivedActions", EMPTY_ARRAY)
.getObject(0, EMPTY_OBJECT).getObject("navigateAction", EMPTY_OBJECT)
.getObject("endpoint", EMPTY_OBJECT);

final String webPageType = endpoint
.getObject("commandMetadata", EMPTY_OBJECT)
.getObject("webCommandMetadata", EMPTY_OBJECT)
final JsonObject endpoint = jsonResponse.getObject(1).getObject("response")
.getArray("onResponseReceivedActions").getObject(0).getObject("navigateAction")
.getObject("endpoint");

final String webPageType = endpoint.getObject("commandMetadata").getObject("webCommandMetadata")
.getString("webPageType", EMPTY_STRING);

final String browseId = endpoint
.getObject("browseEndpoint", EMPTY_OBJECT)
.getString("browseId", EMPTY_STRING);
final String browseId = endpoint.getObject("browseEndpoint").getString("browseId", EMPTY_STRING);

if (webPageType.equalsIgnoreCase("WEB_PAGE_TYPE_BROWSE") && !browseId.isEmpty()) {

if (!browseId.startsWith("UC")) {
throw new ExtractionException("Redirected id is not pointing to a channel");
}
@@ -131,10 +125,8 @@ public String getUrl() throws ParsingException {
@Nonnull
@Override
public String getId() throws ParsingException {
final String channelId = initialData
.getObject("header", EMPTY_OBJECT)
.getObject("c4TabbedHeaderRenderer", EMPTY_OBJECT)
.getString("channelId", EMPTY_STRING);
final String channelId = initialData.getObject("header").getObject("c4TabbedHeaderRenderer")
.getString("channelId", EMPTY_STRING);

if (!channelId.isEmpty()) {
return channelId;
@@ -170,11 +162,9 @@ public String getAvatarUrl() throws ParsingException {
@Override
public String getBannerUrl() throws ParsingException {
try {
String url = null;
try {
url = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("banner")
String url = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("banner")
.getArray("thumbnails").getObject(0).getString("url");
} catch (Exception ignored) {}

if (url == null || url.contains("s.ytimg.com") || url.contains("default_banner")) {
return null;
}
@@ -196,19 +186,19 @@ public String getFeedUrl() throws ParsingException {

@Override
public long getSubscriberCount() throws ParsingException {
final JsonObject subscriberInfo = initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("subscriberCountText");
if (subscriberInfo != null) {
final JsonObject c4TabbedHeaderRenderer = initialData.getObject("header").getObject("c4TabbedHeaderRenderer");
if (c4TabbedHeaderRenderer.has("subscriberCountText")) {
try {
return Utils.mixedNumberWordToLong(getTextFromObject(subscriberInfo));
return Utils.mixedNumberWordToLong(getTextFromObject(c4TabbedHeaderRenderer.getObject("subscriberCountText")));
} catch (NumberFormatException e) {
throw new ParsingException("Could not get subscriber count", e);
}
} else {
// If there's no subscribe button, the channel has the subscriber count disabled
if (initialData.getObject("header").getObject("c4TabbedHeaderRenderer").getObject("subscribeButton") == null) {
return -1;
} else {
if (c4TabbedHeaderRenderer.has("subscribeButton")) {
return 0;
} else {
return -1;
}
}
}
@@ -260,7 +250,9 @@ public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) throws IOException,


private String getNextPageUrlFrom(JsonArray continuations) {
if (continuations == null) return "";
if (continuations == null || continuations.isEmpty()) {
return "";
}

JsonObject nextContinuationData = continuations.getObject(0).getObject("nextContinuationData");
String continuation = nextContinuationData.getString("continuation");
@@ -277,7 +269,7 @@ private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonArray vi
final TimeAgoParser timeAgoParser = getTimeAgoParser();

for (Object video : videos) {
if (((JsonObject) video).getObject("gridVideoRenderer") != null) {
if (((JsonObject) video).has("gridVideoRenderer")) {
collector.commit(new YoutubeStreamInfoItemExtractor(
((JsonObject) video).getObject("gridVideoRenderer"), timeAgoParser) {
@Override
@@ -302,8 +294,8 @@ private JsonObject getVideoTab() throws ParsingException {
JsonObject videoTab = null;

for (Object tab : tabs) {
if (((JsonObject) tab).getObject("tabRenderer") != null) {
if (((JsonObject) tab).getObject("tabRenderer").getString("title").equals("Videos")) {
if (((JsonObject) tab).has("tabRenderer")) {
if (((JsonObject) tab).getObject("tabRenderer").getString("title", EMPTY_STRING).equals("Videos")) {
videoTab = ((JsonObject) tab).getObject("tabRenderer");
break;
}
@@ -314,13 +306,12 @@ private JsonObject getVideoTab() throws ParsingException {
throw new ContentNotSupportedException("This channel has no Videos tab");
}

try {
if (getTextFromObject(videoTab.getObject("content").getObject("sectionListRenderer")
.getArray("contents").getObject(0).getObject("itemSectionRenderer")
.getArray("contents").getObject(0).getObject("messageRenderer")
.getObject("text")).equals("This channel has no videos."))
return null;
} catch (Exception ignored) {}
if (getTextFromObject(videoTab.getObject("content").getObject("sectionListRenderer")
.getArray("contents").getObject(0).getObject("itemSectionRenderer")
.getArray("contents").getObject(0).getObject("messageRenderer")
.getObject("text")).equals("This channel has no videos.")) {
return null;
}

this.videoTab = videoTab;
return videoTab;
Loading

0 comments on commit 49157fc

Please sign in to comment.