Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Handle isDeleted for PeerTube comments #340

Merged
merged 1 commit into from
Jun 28, 2020
Merged

Handle isDeleted for PeerTube comments #340

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 16 additions & 21 deletions .../org/schabi/newpipe/extractor/services/peertube/extractors/PeertubeCommentsExtractor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,20 +15,19 @@
import org.schabi.newpipe.extractor.linkhandler.ListLinkHandler;
import org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper;
import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Parser;
import org.schabi.newpipe.extractor.utils.Parser.RegexException;
import org.schabi.newpipe.extractor.utils.Utils;

import java.io.IOException;

import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.*;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.COUNT_KEY;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.ITEMS_PER_PAGE;
import static org.schabi.newpipe.extractor.services.peertube.PeertubeParsingHelper.START_KEY;

public class PeertubeCommentsExtractor extends CommentsExtractor {

private InfoItemsPage<CommentsInfoItem> initPage;
private long total;

public PeertubeCommentsExtractor(StreamingService service, ListLinkHandler uiHandler) {
public PeertubeCommentsExtractor(final StreamingService service, final ListLinkHandler uiHandler) {
super(service, uiHandler);
}

Expand All @@ -38,22 +37,18 @@ public InfoItemsPage<CommentsInfoItem> getInitialPage() throws IOException, Extr
return initPage;
}

private void collectStreamsFrom(CommentsInfoItemsCollector collector, JsonObject json, String pageUrl) throws ParsingException {
JsonArray contents;
try {
contents = (JsonArray) JsonUtils.getValue(json, "data");
} catch (Exception e) {
throw new ParsingException("unable to extract comments info", e);
}
private void collectCommentsFrom(final CommentsInfoItemsCollector collector, final JsonObject json) throws ParsingException {
final JsonArray contents = json.getArray("data");

for (Object c : contents) {
for (final Object c : contents) {
if (c instanceof JsonObject) {
final JsonObject item = (JsonObject) c;
PeertubeCommentsInfoItemExtractor extractor = new PeertubeCommentsInfoItemExtractor(item, this);
collector.commit(extractor);
if (!item.getBoolean("isDeleted")) {
final PeertubeCommentsInfoItemExtractor extractor = new PeertubeCommentsInfoItemExtractor(item, this);
collector.commit(extractor);
}
}
}

}

@Override
Expand All @@ -63,8 +58,8 @@ public String getNextPageUrl() throws IOException, ExtractionException {
}

@Override
public InfoItemsPage<CommentsInfoItem> getPage(String pageUrl) throws IOException, ExtractionException {
Response response = getDownloader().get(pageUrl);
public InfoItemsPage<CommentsInfoItem> getPage(final String pageUrl) throws IOException, ExtractionException {
final Response response = getDownloader().get(pageUrl);
JsonObject json = null;
if (response != null && !Utils.isBlank(response.responseBody())) {
try {
Expand All @@ -74,11 +69,11 @@ public InfoItemsPage<CommentsInfoItem> getPage(String pageUrl) throws IOExceptio
}
}

CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId());
final CommentsInfoItemsCollector collector = new CommentsInfoItemsCollector(getServiceId());
if (json != null) {
Number number = JsonUtils.getNumber(json, "total");
final Number number = JsonUtils.getNumber(json, "total");
if (number != null) this.total = number.longValue();
collectStreamsFrom(collector, json, pageUrl);
collectCommentsFrom(collector, json);
} else {
throw new ExtractionException("Unable to get peertube comments info");
}
Expand Down
131 changes: 77 additions & 54 deletions ...st/java/org/schabi/newpipe/extractor/services/peertube/PeertubeCommentsExtractorTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,74 +18,97 @@
import static org.schabi.newpipe.extractor.ServiceList.PeerTube;

public class PeertubeCommentsExtractorTest {
public static class Default {
private static PeertubeCommentsExtractor extractor;

private static PeertubeCommentsExtractor extractor;
@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (PeertubeCommentsExtractor) PeerTube
.getCommentsExtractor("https://framatube.org/videos/watch/04af977f-4201-4697-be67-a8d8cae6fa7a");
}

@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (PeertubeCommentsExtractor) PeerTube
.getCommentsExtractor("https://framatube.org/videos/watch/04af977f-4201-4697-be67-a8d8cae6fa7a");
}
@Test
public void testGetComments() throws IOException, ExtractionException {
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
assertTrue(comments.getErrors().isEmpty());

@Test
public void testGetComments() throws IOException, ExtractionException {
boolean result = false;
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
result = findInComments(comments, "@root A great documentary on a great guy.");
boolean result = findInComments(comments, "@root A great documentary on a great guy.");
while (comments.hasNextPage() && !result) {
comments = extractor.getPage(comments.getNextPageUrl());
result = findInComments(comments, "@root A great documentary on a great guy.");
}
B0pol marked this conversation as resolved.
Show resolved Hide resolved

while (comments.hasNextPage() && !result) {
comments = extractor.getPage(comments.getNextPageUrl());
result = findInComments(comments, "@root A great documentary on a great guy.");
assertTrue(result);
}

assertTrue(result);
}
@Test
public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
final CommentsInfo commentsInfo = CommentsInfo.getInfo("https://framatube.org/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828");
assertTrue(commentsInfo.getErrors().isEmpty());
assertEquals("Comments", commentsInfo.getName());

@Test
public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
boolean result = false;
CommentsInfo commentsInfo = CommentsInfo.getInfo("https://framatube.org/videos/watch/a8ea95b8-0396-49a6-8f30-e25e25fb2828");
assertEquals("Comments", commentsInfo.getName());
result = findInComments(commentsInfo.getRelatedItems(), "Loved it!!!");

String nextPage = commentsInfo.getNextPageUrl();
while (!Utils.isBlank(nextPage) && !result) {
InfoItemsPage<CommentsInfoItem> moreItems = CommentsInfo.getMoreItems(PeerTube, commentsInfo, nextPage);
result = findInComments(moreItems.getItems(), "Loved it!!!");
nextPage = moreItems.getNextPageUrl();
boolean result = findInComments(commentsInfo.getRelatedItems(), "Loved it!!!");
String nextPage = commentsInfo.getNextPageUrl();
while (!Utils.isBlank(nextPage) && !result) {
final InfoItemsPage<CommentsInfoItem> moreItems = CommentsInfo.getMoreItems(PeerTube, commentsInfo, nextPage);
result = findInComments(moreItems.getItems(), "Loved it!!!");
nextPage = moreItems.getNextPageUrl();
}
assertTrue(result);
}

assertTrue(result);
}
@Test
public void testGetCommentsAllData() throws IOException, ExtractionException {
final InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
for (final CommentsInfoItem c : comments.getItems()) {
assertFalse(Utils.isBlank(c.getUploaderUrl()));
assertFalse(Utils.isBlank(c.getUploaderName()));
assertFalse(Utils.isBlank(c.getUploaderAvatarUrl()));
assertFalse(Utils.isBlank(c.getCommentId()));
assertFalse(Utils.isBlank(c.getCommentText()));
assertFalse(Utils.isBlank(c.getName()));
assertFalse(Utils.isBlank(c.getTextualUploadDate()));
assertFalse(Utils.isBlank(c.getThumbnailUrl()));
assertFalse(Utils.isBlank(c.getUrl()));
assertEquals(-1, c.getLikeCount());
}
}

@Test
public void testGetCommentsAllData() throws IOException, ExtractionException {
InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
for (CommentsInfoItem c : comments.getItems()) {
assertFalse(Utils.isBlank(c.getUploaderUrl()));
assertFalse(Utils.isBlank(c.getUploaderName()));
assertFalse(Utils.isBlank(c.getUploaderAvatarUrl()));
assertFalse(Utils.isBlank(c.getCommentId()));
assertFalse(Utils.isBlank(c.getCommentText()));
assertFalse(Utils.isBlank(c.getName()));
assertFalse(Utils.isBlank(c.getTextualUploadDate()));
assertFalse(Utils.isBlank(c.getThumbnailUrl()));
assertFalse(Utils.isBlank(c.getUrl()));
assertFalse(c.getLikeCount() != -1);
private boolean findInComments(final InfoItemsPage<CommentsInfoItem> comments, final String comment) {
return findInComments(comments.getItems(), comment);
}
}

private boolean findInComments(InfoItemsPage<CommentsInfoItem> comments, String comment) {
return findInComments(comments.getItems(), comment);
private boolean findInComments(final List<CommentsInfoItem> comments, final String comment) {
for (final CommentsInfoItem c : comments) {
if (c.getCommentText().contains(comment)) {
return true;
}
}
return false;
}
}

private boolean findInComments(List<CommentsInfoItem> comments, String comment) {
for (CommentsInfoItem c : comments) {
if (c.getCommentText().contains(comment)) {
return true;
}
public static class DeletedComments {
private static PeertubeCommentsExtractor extractor;

@BeforeClass
public static void setUp() throws Exception {
NewPipe.init(DownloaderTestImpl.getInstance());
extractor = (PeertubeCommentsExtractor) PeerTube
.getCommentsExtractor("https://framatube.org/videos/watch/217eefeb-883d-45be-b7fc-a788ad8507d3");
}

wb9688 marked this conversation as resolved.
Show resolved Hide resolved
@Test
public void testGetComments() throws IOException, ExtractionException {
final InfoItemsPage<CommentsInfoItem> comments = extractor.getInitialPage();
assertTrue(comments.getErrors().isEmpty());
B0pol marked this conversation as resolved.
Show resolved Hide resolved
}

@Test
public void testGetCommentsFromCommentsInfo() throws IOException, ExtractionException {
final CommentsInfo commentsInfo = CommentsInfo.getInfo("https://framatube.org/videos/watch/217eefeb-883d-45be-b7fc-a788ad8507d3");
assertTrue(commentsInfo.getErrors().isEmpty());
}
return false;
}
}