Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added same imgur and stuff album downloaders characteristics to ChanRipper #185

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/main/java/com/rarchives/ripme/App.java
Original file line number Diff line number Diff line change
Expand Up @@ -216,7 +216,7 @@ public static CommandLine getArgs(String[] args) {
CommandLine cl = parser.parse(getOptions(), args, false);
return cl;
} catch (ParseException e) {
logger.error("[!] Error while parsing command-line arguments: " + args, e);
logger.error("[!] Error while parsing command-line arguments: " + Arrays.toString(args), e);
System.exit(-1);
return null;
}
Expand Down
37 changes: 34 additions & 3 deletions src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import com.rarchives.ripme.ui.RipStatusMessage;
import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
import com.rarchives.ripme.utils.Utils;
import java.lang.reflect.InvocationTargetException;

public abstract class AbstractRipper
extends Observable
Expand Down Expand Up @@ -94,8 +95,22 @@ public void setObserver(RipStatusHandler obs) {
* URL of the file
* @param saveAs
* Path of the local file to save the content to.
* @return True on success, flase on failure.
*/
public abstract boolean addURLToDownload(URL url, File saveAs);

/**
* Queues image to be downloaded and saved.
* @param url
* URL of the file
* @param saveAs
* Path of the local file to save the content to.
* @param referrer
* The HTTP referrer to use while downloading this file.
* @param cookies
* The cookies to send to the server while downloading this file.
* @return
*/
public abstract boolean addURLToDownload(URL url, File saveAs, String referrer, Map<String,String> cookies);

public boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map<String,String> cookies) {
Expand Down Expand Up @@ -144,6 +159,7 @@ public boolean addURLToDownload(URL url, String prefix, String subdirectory, Str
* Prefix to prepend to the saved filename.
* @param subdirectory
* Sub-directory of the working directory to save the images to.
* @return True on success, flase on failure.
*/
public boolean addURLToDownload(URL url, String prefix, String subdirectory) {
return addURLToDownload(url, prefix, subdirectory, null, null);
Expand All @@ -156,6 +172,7 @@ public boolean addURLToDownload(URL url, String prefix, String subdirectory) {
* URL to download
* @param prefix
* Text to append to saved filename.
* @return True on success, flase on failure.
*/
public boolean addURLToDownload(URL url, String prefix) {
// Use empty subdirectory
Expand Down Expand Up @@ -201,7 +218,7 @@ public void retrievingSource(String url) {
* Notify observers that a download could not be completed,
* but was not technically an "error".
* @param url
* @param message
* @param file
*/
public abstract void downloadExists(URL url, File file);

Expand Down Expand Up @@ -281,7 +298,13 @@ public static AbstractRipper getRipper(URL url) throws Exception {
AlbumRipper ripper = (AlbumRipper) constructor.newInstance(url);
logger.debug("Found album ripper: " + ripper.getClass().getName());
return ripper;
} catch (Exception e) {
} catch (InstantiationException e) {
// Incompatible rippers *will* throw exceptions during instantiation.
} catch (IllegalAccessException e) {
// Incompatible rippers *will* throw exceptions during instantiation.
} catch (IllegalArgumentException e) {
// Incompatible rippers *will* throw exceptions during instantiation.
} catch (InvocationTargetException e) {
// Incompatible rippers *will* throw exceptions during instantiation.
}
}
Expand All @@ -290,14 +313,22 @@ public static AbstractRipper getRipper(URL url) throws Exception {
VideoRipper ripper = (VideoRipper) constructor.newInstance(url);
logger.debug("Found video ripper: " + ripper.getClass().getName());
return ripper;
} catch (Exception e) {
} catch (InstantiationException e) {
// Incompatible rippers *will* throw exceptions during instantiation.
} catch (IllegalAccessException e) {
// Incompatible rippers *will* throw exceptions during instantiation.
} catch (IllegalArgumentException e) {
// Incompatible rippers *will* throw exceptions during instantiation.
} catch (InvocationTargetException e) {
// Incompatible rippers *will* throw exceptions during instantiation.
}
}
throw new Exception("No compatible ripper found");
}

/**
* @param pkg
* The package name.
* @return
* List of constructors for all eligible Rippers.
* @throws Exception
Expand Down
25 changes: 21 additions & 4 deletions src/main/java/com/rarchives/ripme/ripper/rippers/ChanRipper.java
Original file line number Diff line number Diff line change
Expand Up @@ -11,11 +11,12 @@

import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.rarchives.ripme.ripper.AbstractHTMLRipper;
import com.rarchives.ripme.ripper.rippers.ripperhelpers.ChanSite;
import com.rarchives.ripme.utils.Http;
import com.rarchives.ripme.utils.RipUtils;


public class ChanRipper extends AbstractHTMLRipper {
public static List<ChanSite> explicit_domains = Arrays.asList(
Expand Down Expand Up @@ -90,7 +91,11 @@ public boolean canRip(URL url) {
* For example the archives are all known. (Check 4chan-x)
* Should be based on the software the specific chan uses.
* FoolFuuka uses the same (url) layout as 4chan
* */
*
* @param url
* @return
* The thread id in string form
* @throws java.net.MalformedURLException */
@Override
public String getGID(URL url) throws MalformedURLException {
Pattern p;
Expand Down Expand Up @@ -181,8 +186,20 @@ public List<String> getURLsFromPage(Document page) {
}
}
} else {
//TODO also grab imgur/flickr albums (And all other supported rippers) Maybe add a setting?
}
//Copied code from RedditRipper, getFilesFromURL should also implement stuff like flickr albums
URL originalURL;
try {
originalURL = new URL(href);
} catch (MalformedURLException e) {
continue;
}

List<URL> urls = RipUtils.getFilesFromURL(originalURL);
//for (int i = 0; i < urls.size(); i++) {
for(URL imageurl : urls){
imageURLs.add(imageurl.toString());
}
}

if (isStopped()) {
break;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -171,11 +171,12 @@ public void handleBody(String body, String id) {
Pattern p = RipUtils.getURLRegex();
Matcher m = p.matcher(body);
while (m.find()) {
String url = m.group(1);
while (url.endsWith(")")) {
url = url.substring(0, url.length() - 1);
String foundurl;
foundurl = m.group(1);
while (foundurl.endsWith(")")) {
foundurl = foundurl.substring(0, foundurl.length() - 1);
}
handleURL(url, id);
handleURL(foundurl, id);
}
}

Expand Down
12 changes: 10 additions & 2 deletions src/main/java/com/rarchives/ripme/ui/History.java
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@

public class History {

private List<HistoryEntry> list = new ArrayList<HistoryEntry>();
private final List<HistoryEntry> list;
private static final String[] COLUMNS = new String[] {
"URL",
"created",
Expand All @@ -26,6 +26,10 @@ public class History {
""
};

public History() {
this.list = new ArrayList<HistoryEntry>();
}

public void add(HistoryEntry entry) {
list.add(entry);
}
Expand Down Expand Up @@ -124,10 +128,14 @@ public JSONArray toJSON() {
return jsonArray;
}

public List<HistoryEntry> toList() {
public List<HistoryEntry> toList() {
return list;
}

public boolean isEmpty(){
return list.isEmpty();
}

public void toFile(String filename) throws IOException {
OutputStream os = new FileOutputStream(filename);
try {
Expand Down
1 change: 1 addition & 0 deletions src/main/java/com/rarchives/ripme/ui/HistoryEntry.java
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ public JSONObject toJSON() {
return json;
}

@Override
public String toString() {
return this.url;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ public HistoryMenuMouseListener() {
@Override
public void actionPerformed(ActionEvent ae) {
for (int row = 0; row < tableComponent.getRowCount(); row++) {
tableComponent.setValueAt(new Boolean(true), row, 4);
tableComponent.setValueAt(true, row, 4);
}
}
};
Expand All @@ -34,7 +34,7 @@ public void actionPerformed(ActionEvent ae) {
@Override
public void actionPerformed(ActionEvent ae) {
for (int row = 0; row < tableComponent.getRowCount(); row++) {
tableComponent.setValueAt(new Boolean(false), row, 4);
tableComponent.setValueAt(false, row, 4);
}
}
};
Expand All @@ -46,7 +46,7 @@ public void actionPerformed(ActionEvent ae) {
@Override
public void actionPerformed(ActionEvent ae) {
for (int row : tableComponent.getSelectedRows()) {
tableComponent.setValueAt(new Boolean(true), row, 4);
tableComponent.setValueAt(true, row, 4);
}
}
};
Expand All @@ -56,7 +56,7 @@ public void actionPerformed(ActionEvent ae) {
@Override
public void actionPerformed(ActionEvent ae) {
for (int row : tableComponent.getSelectedRows()) {
tableComponent.setValueAt(new Boolean(false), row, 4);
tableComponent.setValueAt(false, row, 4);
}
}
};
Expand Down
42 changes: 29 additions & 13 deletions src/main/java/com/rarchives/ripme/ui/MainWindow.java
Original file line number Diff line number Diff line change
Expand Up @@ -75,11 +75,13 @@
import com.rarchives.ripme.ripper.AbstractRipper;
import com.rarchives.ripme.utils.RipUtils;
import com.rarchives.ripme.utils.Utils;
import java.awt.AWTException;
import javax.swing.UnsupportedLookAndFeelException;

/**
* Everything UI-related starts and ends here.
*/
public class MainWindow implements Runnable, RipStatusHandler {
public final class MainWindow implements Runnable, RipStatusHandler {

private static final Logger logger = Logger.getLogger(MainWindow.class);

Expand Down Expand Up @@ -255,7 +257,13 @@ private void createUI(Container pane) {

try {
UIManager.setLookAndFeel(UIManager.getSystemLookAndFeelClassName());
} catch (Exception e) {
} catch (ClassNotFoundException e) {
logger.error("[!] Exception setting system theme:", e);
} catch (InstantiationException e) {
logger.error("[!] Exception setting system theme:", e);
} catch (IllegalAccessException e) {
logger.error("[!] Exception setting system theme:", e);
} catch (UnsupportedLookAndFeelException e) {
logger.error("[!] Exception setting system theme:", e);
}

Expand All @@ -271,12 +279,13 @@ private void createUI(Container pane) {
} catch (Exception e) { }
JPanel ripPanel = new JPanel(new GridBagLayout());
ripPanel.setBorder(emptyBorder);

gbc.gridx = 0; ripPanel.add(new JLabel("URL:", JLabel.RIGHT), gbc);
gbc.gridx = 1; ripPanel.add(ripTextfield, gbc);
gbc.gridx = 2; ripPanel.add(ripButton, gbc);
gbc.gridx = 3; ripPanel.add(stopButton, gbc);


gbc.gridx = 0; gbc.fill = GridBagConstraints.HORIZONTAL; ripPanel.add(new JLabel("URL:", JLabel.RIGHT), gbc);
gbc.gridx = 1; gbc.fill = GridBagConstraints.HORIZONTAL; ripPanel.add(ripTextfield, gbc);
gbc.gridx = 2; gbc.fill = GridBagConstraints.HORIZONTAL; ripPanel.add(ripButton, gbc);
gbc.gridx = 3; gbc.fill = GridBagConstraints.HORIZONTAL; ripPanel.add(stopButton, gbc);
gbc.fill = GridBagConstraints.BOTH;

statusLabel = new JLabel("Inactive");
statusLabel.setHorizontalAlignment(JLabel.CENTER);
openButton = new JButton();
Expand Down Expand Up @@ -333,6 +342,7 @@ private void createUI(Container pane) {
public String getColumnName(int col) {
return HISTORY.getColumnName(col);
}
@Override
public Class<? extends Object> getColumnClass(int c) {
return getValueAt(0, c).getClass();
}
Expand Down Expand Up @@ -653,7 +663,7 @@ public void actionPerformed(ActionEvent event) {
historyButtonRerip.addActionListener(new ActionListener() {
@Override
public void actionPerformed(ActionEvent event) {
if (HISTORY.toList().size() == 0) {
if (HISTORY.isEmpty()) {
JOptionPane.showMessageDialog(null,
"There are no history entries to re-rip. Rip some albums first",
"RipMe Error",
Expand All @@ -673,7 +683,6 @@ public void actionPerformed(ActionEvent event) {
"Check an entry by clicking the checkbox to the right of the URL or Right-click a URL to check/uncheck all items",
"RipMe Error",
JOptionPane.ERROR_MESSAGE);
return;
}
}
});
Expand Down Expand Up @@ -838,9 +847,13 @@ else if (level.equals("Error")) {

private void setupTrayIcon() {
mainFrame.addWindowListener(new WindowAdapter() {
@Override
public void windowActivated(WindowEvent e) { trayMenuMain.setLabel("Hide"); }
@Override
public void windowDeactivated(WindowEvent e) { trayMenuMain.setLabel("Show"); }
@Override
public void windowDeiconified(WindowEvent e) { trayMenuMain.setLabel("Hide"); }
@Override
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe put these on the same line as the definition to keep the one-line method consistent?

Copy link
Contributor Author

@EraYaN EraYaN Apr 25, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is not my code. But in the same style as C style pragma this should probably be above the method.

EDIT: In other spots in the code it is also done as a method decorator (above the method declaration, with the same indent)

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have no strong opinion on this. Function over style, IMO. It's fiine as-is.

public void windowIconified(WindowEvent e) { trayMenuMain.setLabel("Show"); }
});
PopupMenu trayMenu = new PopupMenu();
Expand Down Expand Up @@ -946,7 +959,11 @@ public void mouseClicked(MouseEvent e) {
mainFrame.setAlwaysOnTop(false);
}
});
} catch (Exception e) {
} catch (IOException e) {
//TODO implement proper stack trace handling this is really just intented as a placeholder until you implement proper error handling
e.printStackTrace();
} catch (AWTException e) {
//TODO implement proper stack trace handling this is really just intented as a placeholder until you implement proper error handling
Copy link
Collaborator

@metaprime metaprime Apr 25, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah, that's true, but I think this code pattern is fair though. The point is to catch the exception, print it out so that someone can investigate it from the log, and not actually crash the program. (It is possible to continue the program and just not finish the operation that excepted.)

e.printStackTrace();
}
}
Expand Down Expand Up @@ -1034,7 +1051,6 @@ private void saveHistory() {
@SuppressWarnings("unchecked")
private void ripNextAlbum() {
isRipping = true;

// Save current state of queue to configuration.
Utils.setConfigList("queue", (Enumeration<Object>) queueListModel.elements());

Expand All @@ -1044,7 +1060,7 @@ private void ripNextAlbum() {
return;
}
String nextAlbum = (String) queueListModel.remove(0);
if (queueListModel.size() == 0) {
if (queueListModel.isEmpty()) {
optionQueue.setText("Queue");
}
else {
Expand Down
1 change: 1 addition & 0 deletions src/main/java/com/rarchives/ripme/ui/RipStatusMessage.java
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ public Object getObject() {
return object;
}

@Override
public String toString() {
return status.value + ": " + object.toString();
}
Expand Down
Loading