Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[PeerTube] channels support #308

Merged
merged 3 commits into from
Apr 10, 2020
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,10 @@ public SubscriptionExtractor getSubscriptionExtractor() {
@Override
public ChannelExtractor getChannelExtractor(ListLinkHandler linkHandler)
throws ExtractionException {
return new PeertubeChannelExtractor(this, linkHandler);

return linkHandler.getUrl().matches("^.*\\/accounts\\/[^\\/]*$") ?
new PeertubeUserExtractor(this, linkHandler) :
new PeertubeChannelExtractor(this, linkHandler);
Royosef marked this conversation as resolved.
Show resolved Hide resolved
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ public String getBannerUrl() throws ParsingException {

@Override
public String getFeedUrl() throws ParsingException {
return getBaseUrl() + "/feeds/videos.xml?accountId=" + json.get("id");
return getBaseUrl() + "/feeds/videos.xml?videoChannelId=" + json.get("id");
}

@Override
Expand Down Expand Up @@ -181,7 +181,7 @@ public String getName() throws ParsingException {

@Override
public String getOriginalUrl() throws ParsingException {
return baseUrl + "/accounts/" + getId();
return baseUrl + "/" + getId();
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,7 @@ public String getAuthorName() throws ParsingException {
public String getAuthorEndpoint() throws ParsingException {
String name = JsonUtils.getString(item, "account.name");
String host = JsonUtils.getString(item, "account.host");
return ServiceList.PeerTube.getChannelLHFactory().fromId(name + "@" + host, baseUrl).getUrl();
return ServiceList.PeerTube.getChannelLHFactory().fromId("accounts/" + name + "@" + host, baseUrl).getUrl();
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@ public long getDislikeCount() throws ParsingException {
public String getUploaderUrl() throws ParsingException {
String name = JsonUtils.getString(json, "account.name");
String host = JsonUtils.getString(json, "account.host");
return getService().getChannelLHFactory().fromId(name + "@" + host, baseUrl).getUrl();
return getService().getChannelLHFactory().fromId("accounts/" + name + "@" + host, baseUrl).getUrl();
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,8 @@ public long getViewCount() throws ParsingException {
public String getUploaderUrl() throws ParsingException {
String name = JsonUtils.getString(item, "account.name");
String host = JsonUtils.getString(item, "account.host");
return ServiceList.PeerTube.getChannelLHFactory().fromId(name + "@" + host, baseUrl).getUrl();

return ServiceList.PeerTube.getChannelLHFactory().fromId("accounts/" + name + "@" + host, baseUrl).getUrl();
}

@Override
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,187 @@
package org.schabi.newpipe.extractor.services.peertube.extractors;

import com.grack.nanojson.JsonArray;
import com.grack.nanojson.JsonObject;
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import org.jsoup.helper.StringUtil;
import org.schabi.newpipe.extractor.StreamingService;
import org.schabi.newpipe.extractor.channel.ChannelExtractor;
import org.schabi.newpipe.extractor.downloader.Downloader;
import org.schabi.newpipe.extractor.downloader.Response;
import org.schabi.newpipe.extractor.exceptions.ExtractionException;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.stream.StreamInfoItem;
import org.schabi.newpipe.extractor.stream.StreamInfoItemsCollector;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Parser.RegexException;

import java.io.IOException;

public class PeertubeUserExtractor extends ChannelExtractor {
Royosef marked this conversation as resolved.
Show resolved Hide resolved

private static final String START_KEY = "start";
private static final String COUNT_KEY = "count";
private static final int ITEMS_PER_PAGE = 12;
private static final String START_PATTERN = "start=(\\d*)";

private InfoItemsPage<StreamInfoItem> initPage;
private long total;

private JsonObject json;
private final String baseUrl;

public PeertubeUserExtractor(StreamingService service, ListLinkHandler linkHandler) throws ParsingException {
super(service, linkHandler);
this.baseUrl = getBaseUrl();
}

@Override
public String getAvatarUrl() throws ParsingException {
String value;
try {
value = JsonUtils.getString(json, "avatar.path");
} catch (Exception e) {
value = "/client/assets/images/default-avatar.png";
}
return baseUrl + value;
}

@Override
public String getBannerUrl() throws ParsingException {
return null;
}

@Override
public String getFeedUrl() throws ParsingException {
return getBaseUrl() + "/feeds/videos.xml?accountId=" + json.get("id");
}

@Override
public long getSubscriberCount() throws ParsingException {
Number number = JsonUtils.getNumber(json, "followersCount");
return number.longValue();
}

@Override
public String getDescription() throws ParsingException {
try {
return JsonUtils.getString(json, "description");
} catch (ParsingException e) {
return "No description";
}
}

@Override
public InfoItemsPage<StreamInfoItem> getInitialPage() throws IOException, ExtractionException {
super.fetchPage();
return initPage;
}

private void collectStreamsFrom(StreamInfoItemsCollector collector, JsonObject json, String pageUrl) throws ParsingException {
JsonArray contents;
try {
contents = (JsonArray) JsonUtils.getValue(json, "data");
} catch (Exception e) {
throw new ParsingException("unable to extract channel streams", e);
}

for (Object c : contents) {
if (c instanceof JsonObject) {
final JsonObject item = (JsonObject) c;
PeertubeStreamInfoItemExtractor extractor = new PeertubeStreamInfoItemExtractor(item, baseUrl);
collector.commit(extractor);
}
}

}

@Override
public String getNextPageUrl() throws IOException, ExtractionException {
super.fetchPage();
return initPage.getNextPageUrl();
}

@Override
public InfoItemsPage<StreamInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
Response response = getDownloader().get(pageUrl);
JsonObject json = null;
if (null != response && !StringUtil.isBlank(response.responseBody())) {
try {
json = JsonParser.object().from(response.responseBody());
} catch (Exception e) {
throw new ParsingException("Could not parse json data for kiosk info", e);
}
}

StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
if (json != null) {
PeertubeParsingHelper.validate(json);
Number number = JsonUtils.getNumber(json, "total");
if (number != null) this.total = number.longValue();
collectStreamsFrom(collector, json, pageUrl);
} else {
throw new ExtractionException("Unable to get PeerTube kiosk info");
}
return new InfoItemsPage<>(collector, getNextPageUrl(pageUrl));
}


private String getNextPageUrl(String prevPageUrl) {
String prevStart;
try {
prevStart = Parser.matchGroup1(START_PATTERN, prevPageUrl);
} catch (RegexException e) {
return "";
}
if (StringUtil.isBlank(prevStart)) return "";
long nextStart = 0;
try {
nextStart = Long.valueOf(prevStart) + ITEMS_PER_PAGE;
} catch (NumberFormatException e) {
return "";
}

if (nextStart >= total) {
return "";
} else {
return prevPageUrl.replace(START_KEY + "=" + prevStart, START_KEY + "=" + String.valueOf(nextStart));
}
}

@Override
public void onFetchPage(Downloader downloader) throws IOException, ExtractionException {
Response response = downloader.get(getUrl());
if (null != response && null != response.responseBody()) {
setInitialData(response.responseBody());
} else {
throw new ExtractionException("Unable to extract PeerTube channel data");
}

String pageUrl = getUrl() + "/videos?" + START_KEY + "=0&" + COUNT_KEY + "=" + ITEMS_PER_PAGE;
this.initPage = getPage(pageUrl);
}

private void setInitialData(String responseBody) throws ExtractionException {
try {
json = JsonParser.object().from(responseBody);
} catch (JsonParserException e) {
throw new ExtractionException("Unable to extract peertube channel data", e);
}
if (json == null) throw new ExtractionException("Unable to extract PeerTube channel data");
}

@Override
public String getName() throws ParsingException {
return JsonUtils.getString(json, "displayName");
}

@Override
public String getOriginalUrl() throws ParsingException {
return baseUrl + "/" + getId();
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,16 @@
public class PeertubeChannelLinkHandlerFactory extends ListLinkHandlerFactory {

private static final PeertubeChannelLinkHandlerFactory instance = new PeertubeChannelLinkHandlerFactory();
private static final String ID_PATTERN = "/accounts/([^/?&#]*)";
private static final String ACCOUNTS_ENDPOINT = "/api/v1/accounts/";
private static final String ID_PATTERN = "(accounts|video-channels)/([^/?&#]*)";
private static final String API_ENDPOINT = "/api/v1/";

public static PeertubeChannelLinkHandlerFactory getInstance() {
return instance;
}

@Override
public String getId(String url) throws ParsingException {
return Parser.matchGroup1(ID_PATTERN, url);
return Parser.matchGroup(ID_PATTERN, url, 0);
}

@Override
Expand All @@ -31,11 +31,11 @@ public String getUrl(String id, List<String> contentFilters, String searchFilter
@Override
public String getUrl(String id, List<String> contentFilter, String sortFilter, String baseUrl)
throws ParsingException {
return baseUrl + ACCOUNTS_ENDPOINT + id;
return baseUrl + API_ENDPOINT + id;
Royosef marked this conversation as resolved.
Show resolved Hide resolved
}

@Override
public boolean onAcceptUrl(String url) {
return url.contains("/accounts/");
return url.contains("/accounts/") || url.contains("/video-channels/");
}
}
Loading