Merge pull request #2 from RipMeApp/master

Update Fork

pom.xml

@@ -4,7 +4,7 @@
   <groupId>com.rarchives.ripme</groupId>
   <artifactId>ripme</artifactId>
   <packaging>jar</packaging>
-  <version>1.7.8</version>
+  <version>1.7.11</version>
   <name>ripme</name>
   <url>http://rip.rarchives.com</url>
   <properties>

ripme.json

@@ -1,6 +1,9 @@
 {
-    "latestVersion": "1.7.8",
+    "latestVersion": "1.7.11",
     "changeList": [
+        "1.7.11: Added gwarchives support to the cheveretoRipper; Gfycat Tests & Fix for bad reddit submissions; instagram ripper can now be made to skip videos",
+        "1.7.10: Added basic pornpics.com ripper; Fixed hentai.cafe regex",
+        "1.7.9: FuraffinityRipper can now rip non-public albums; Added 2 new api keys, ripper can now download raw images from tumblr; Erome ripper now matchs links without the www; Tumblr ripper now tells the user if it hits the rate limit",
         "1.7.8: Forced https for tumblr image links; Fixed imgur album filenames; SankakuComplexRipper now downloads full sized images; Added dribbble.com ripper; Added comfirm button for clearing history",
         "1.7.7: Fixed E621 Ripper; Added unit test for zizki.com; Added unit test for Xbooru.com; Updated reddit useragent",
         "1.7.6: Added OglafRipper",

src/main/java/com/rarchives/ripme/App.java

@@ -63,13 +63,21 @@ public static void main(String[] args) throws MalformedURLException {
             SwingUtilities.invokeLater(mw);
         }
     }
-
+    /**
+     * Creates an abstract ripper and instructs it to rip.
+     * @param url URL to be ripped
+     * @throws Exception 
+     */
     private static void rip(URL url) throws Exception {
         AbstractRipper ripper = AbstractRipper.getRipper(url);
         ripper.setup();
         ripper.rip();
     }
 
+    /**
+     * For dealing with command-line arguments.
+     * @param args Array of Command-line arguments
+     */
     private static void handleArguments(String[] args) {
         CommandLine cl = getArgs(args);
         if (cl.hasOption('h')) {
@@ -109,7 +117,7 @@ private static void handleArguments(String[] args) {
         }
         if (cl.hasOption('R')) {
             loadHistory();
-            if (HISTORY.toList().size() == 0) {
+            if (HISTORY.toList().isEmpty()) {
                 logger.error("There are no history entries to re-rip. Rip some albums first");
                 System.exit(-1);
             }
@@ -173,14 +181,18 @@ private static void handleArguments(String[] args) {
         }
     }
 
-    // this function will attempt to rip the provided url
+    /**
+     * Attempt to rip targetURL.
+     * @param targetURL URL to rip
+     * @param saveConfig Whether or not you want to save the config (?)
+     */
     private static void ripURL(String targetURL, boolean saveConfig) {
         try {
             URL url = new URL(targetURL);
             rip(url);
             List<String> history = Utils.getConfigList("download.history");
-            if (!history.contains(url.toExternalForm())) {
-                history.add(url.toExternalForm());
+            if (!history.contains(url.toExternalForm())) {//if you haven't already downloaded the file before
+                history.add(url.toExternalForm());//add it to history so you won't have to redownload
                 Utils.setConfigList("download.history", Arrays.asList(history.toArray()));
                 if (saveConfig) {
                     Utils.saveConfig();
@@ -195,6 +207,10 @@ private static void ripURL(String targetURL, boolean saveConfig) {
         }
     }
 
+    /**
+     * Creates an Options object, returns it.
+     * @return Returns all acceptable command-line options.
+     */
     private static Options getOptions() {
         Options opts = new Options();
         opts.addOption("h", "help", false, "Print the help");
@@ -213,6 +229,11 @@ private static Options getOptions() {
         return opts;
     }
 
+    /**
+     * Tries to parse commandline arguments.
+     * @param args Array of commandline arguments.
+     * @return CommandLine object containing arguments.
+     */
     private static CommandLine getArgs(String[] args) {
         BasicParser parser = new BasicParser();
         try {
@@ -223,7 +244,10 @@ private static CommandLine getArgs(String[] args) {
             return null;
         }
     }
-
+
+    /**
+     * Loads history from history file into memory.
+     */
     private static void loadHistory() {
         File historyFile = new File(Utils.getConfigDir() + File.separator + "history.json");
         HISTORY.clear();

src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java

@@ -140,6 +140,14 @@ public void rip() throws IOException {
         }
         waitForThreads();
     }
+
+    /**
+     * Gets the file name from the URL
+     * @param url 
+     *      URL that you want to get the filename from
+     * @return 
+     *      Filename of the URL
+     */
     private String fileNameFromURL(URL url) {
         String saveAs = url.toExternalForm();
         if (saveAs.substring(saveAs.length() - 1) == "/") { saveAs = saveAs.substring(0,saveAs.length() - 1) ;}
@@ -150,6 +158,20 @@ private String fileNameFromURL(URL url) {
         if (saveAs.indexOf(':') >= 0) { saveAs = saveAs.substring(0, saveAs.indexOf(':')); }
         return saveAs;
     }
+    /**
+     * 
+     * @param url
+     *      Target URL
+     * @param subdirectory
+     *      Path to subdirectory where you want to save it
+     * @param text
+     *      Text you want to save
+     * @param index
+     *      Index in something like an album
+     * @return 
+     *      True if ripped successfully
+     *      False if failed
+     */
     public boolean saveText(URL url, String subdirectory, String text, int index) {
         String saveAs = fileNameFromURL(url);
         return saveText(url,subdirectory,text,index,saveAs);
@@ -189,6 +211,14 @@ private boolean saveText(URL url, String subdirectory, String text, int index, S
         }
         return true;
     }
+
+    /**
+     * Gets prefix based on where in the index it is
+     * @param index 
+     *      The index in question
+     * @return 
+     *      Returns prefix for a file. (?)
+     */
     protected String getPrefix(int index) {
         String prefix = "";
         if (keepSortOrder() && Utils.getConfigBoolean("download.save_order", true)) {

src/main/java/com/rarchives/ripme/ripper/AbstractJSONRipper.java

@@ -20,6 +20,7 @@ protected AbstractJSONRipper(URL url) throws IOException {
     }
 
     protected abstract String getDomain();
+    @Override
     public abstract String getHost();
 
     protected abstract JSONObject getFirstPage() throws IOException;

src/main/java/com/rarchives/ripme/ripper/AbstractRipper.java

@@ -84,7 +84,14 @@ private void writeDownloadedURL(String downloadedURL) throws IOException {
             }
         }
     }
-
+
+    /**
+     * Checks to see if Ripme has already downloaded a URL
+     * @param url URL to check if downloaded
+     * @return 
+     *      Returns true if previously downloaded.
+     *      Returns false if not yet downloaded.
+     */
     private boolean hasDownloadedURL(String url) {
         File file = new File(URLHistoryFile);
         try {
@@ -118,6 +125,15 @@ public AbstractRipper(URL url) throws IOException {
         this.url = sanitizeURL(url);
     }
 
+    /**
+     * Sets ripper's:
+     *      Working directory
+     *      Logger (for debugging)
+     *      FileAppender
+     *      Threadpool
+     * @throws IOException 
+     *      Always be prepared.
+     */
     public void setup() throws IOException {
         setWorkingDir(this.url);
         Logger rootLogger = Logger.getRootLogger();
@@ -155,9 +171,27 @@ public void setObserver(RipStatusHandler obs) {
      * @param cookies
      *      The cookies to send to the server while downloading this file.
      * @return
+     *      True if downloaded successfully
+     *      False if failed to download
      */
     protected abstract boolean addURLToDownload(URL url, File saveAs, String referrer, Map<String, String> cookies);
 
+    /**
+     * Queues image to be downloaded and saved.
+     * @param url
+     *      URL of the file
+     * @param prefix
+     *      Prefix for the downloaded file
+     * @param subdirectory
+     *      Path to get to desired directory from working directory
+     * @param referrer
+     *      The HTTP referrer to use while downloading this file.
+     * @param cookies
+     *      The cookies to send to the server while downloading this file.
+     * @return 
+     *      True if downloaded successfully
+     *      False if failed to download
+     */
     protected boolean addURLToDownload(URL url, String prefix, String subdirectory, String referrer, Map<String, String> cookies) {
         if (Utils.getConfigBoolean("remember.url_history", true) && !isThisATest()) {
             if (hasDownloadedURL(url.toExternalForm())) {
@@ -322,6 +356,11 @@ void checkIfComplete() {
         }
     }
 
+    /**
+     * Gets URL
+     * @return 
+     *      Returns URL that wants to be downloaded.
+     */
     public URL getURL() {
         return url;
     }
@@ -335,8 +374,20 @@ public File getWorkingDir() {
         return workingDir;
     }
 
+    @Override
     public abstract void setWorkingDir(URL url) throws IOException;
 
+    /**
+     * 
+     * @param url 
+     *      The URL you want to get the title of.
+     * @return
+     *      host_URLid
+     *      e.g. (for a reddit post)
+     *      reddit_post_7mg2ur
+     * @throws MalformedURLException 
+     *      If any of those damned URLs gets malformed.
+     */
     public String getAlbumTitle(URL url) throws MalformedURLException {
         return getHost() + "_" + getGID(url);
     }
@@ -391,7 +442,7 @@ public static List<Constructor<?>> getRipperConstructors(String pkg) throws Exce
 
     /**
      * Sends an update message to the relevant observer(s) on this ripper.
-     * @param status
+     * @param status 
      * @param message
      */
     public void sendUpdate(STATUS status, Object message) {
@@ -400,9 +451,17 @@ public void sendUpdate(STATUS status, Object message) {
         }
         observer.update(this, new RipStatusMessage(status, message));
     }
-
+
+    /**
+     * Get the completion percentage.
+     * @return 
+     *      Percentage complete
+     */
     public abstract int getCompletionPercentage();
-
+    /**
+     * @return 
+     *      Text for status
+     */
     public abstract String getStatusText();
 
     /**
@@ -423,7 +482,9 @@ public void run() {
             cleanup();
         }
     }
-
+    /**
+     * Tries to delete any empty directories
+     */
     private void cleanup() {
         if (this.workingDir.list().length == 0) {
             // No files, delete the dir
@@ -434,7 +495,15 @@ private void cleanup() {
             }
         }
     }
-
+
+    /**
+     * Pauses thread for a set amount of time.
+     * @param milliseconds
+     *      Amount of time (in milliseconds) that the thread gets paused for
+     * @return 
+     *      True if paused successfully
+     *      False if failed to pause/got interrupted.
+     */
     protected boolean sleep(int milliseconds) {
         try {
             logger.debug("Sleeping " + milliseconds + "ms");

src/main/java/com/rarchives/ripme/ripper/AlbumRipper.java

@@ -13,6 +13,10 @@
 import com.rarchives.ripme.ui.RipStatusMessage.STATUS;
 import com.rarchives.ripme.utils.Utils;
 
+
+/**'
+ * For ripping delicious albums off the interwebz.
+ */
 public abstract class AlbumRipper extends AbstractRipper {
 
     private Map<URL, File> itemsPending = Collections.synchronizedMap(new HashMap<URL, File>());
@@ -34,10 +38,17 @@ protected boolean allowDuplicates() {
     }
 
     @Override
+    /**
+     * Returns total amount of files attempted.
+     */
     public int getCount() {
         return itemsCompleted.size() + itemsErrored.size();
     }
 
+    @Override
+    /**
+     * Queues multiple URLs of single images to download from a single Album URL
+     */
     public boolean addURLToDownload(URL url, File saveAs, String referrer, Map<String,String> cookies) {
         // Only download one file if this is a test.
         if (super.isThisATest() &&
@@ -101,6 +112,9 @@ protected boolean addURLToDownload(URL url) {
     }
 
     @Override
+    /**
+     * Cleans up & tells user about successful download
+     */
     public void downloadCompleted(URL url, File saveAs) {
         if (observer == null) {
             return;
@@ -119,6 +133,9 @@ public void downloadCompleted(URL url, File saveAs) {
     }
 
     @Override
+    /**
+     * Cleans up & tells user about failed download.
+     */
     public void downloadErrored(URL url, String reason) {
         if (observer == null) {
             return;
@@ -131,6 +148,10 @@ public void downloadErrored(URL url, String reason) {
     }
 
     @Override
+    /**
+     * Tells user that a single file in the album they wish to download has
+     * already been downloaded in the past.
+     */
     public void downloadExists(URL url, File file) {
         if (observer == null) {
             return;