Added timeouts and increased maximum file download sizes

RipMeApp · Mar 13, 2014 · 9576896 · 9576896
1 parent 1cdec76
commit 9576896
Show file tree

Hide file tree

Showing 2 changed files with 23 additions and 2 deletions.
diff --git a/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java b/src/main/java/com/rarchives/ripme/ripper/DownloadFileThread.java
@@ -26,13 +26,18 @@ public class DownloadFileThread extends Thread {
     private AbstractRipper observer;
     private int retries;
 
+    private final int TIMEOUT;
+    private final int MAX_BODY_SIZE;
+
     public DownloadFileThread(URL url, File saveAs, AbstractRipper observer) {
         super();
         this.url = url;
         this.saveAs = saveAs;
         this.prettySaveAs = Utils.removeCWD(saveAs);
         this.observer = observer;
         this.retries = Utils.getConfigInteger("download.retries", 1);
+        this.TIMEOUT = Utils.getConfigInteger("download.timeout", 60000);
+        this.MAX_BODY_SIZE = Utils.getConfigInteger("download.max_bytes", 1024 * 1024 * 100);
     }
 
     /**
@@ -60,13 +65,16 @@ public void run() {
                 Response response;
                 response = Jsoup.connect(url.toExternalForm())
                         .ignoreContentType(true)
+                        .userAgent(AbstractRipper.USER_AGENT)
+                        .timeout(TIMEOUT)
+                        .maxBodySize(MAX_BODY_SIZE)
                         .execute();
                 FileOutputStream out = (new FileOutputStream(saveAs));
                 out.write(response.bodyAsBytes());
                 out.close();
                 break; // Download successful: break out of infinite loop
             } catch (IOException e) {
-                logger.error("[!] Exception while downloading file: " + url + " - " + e.getMessage());
+                logger.error("[!] Exception while downloading file: " + url + " - " + e.getMessage(), e);
             }
             if (tries > this.retries) {
                 logger.error("[!] Exceeded maximum retries (" + this.retries + ") for URL " + url);

diff --git a/src/main/resources/rip.properties b/src/main/resources/rip.properties
@@ -1,6 +1,19 @@
+# Download threads to use per ripper
 threads.size = 5
+
+# Overwrite existing files
 file.overwrite = false
-download.retries = 3
+
+# Number of retries on failed downloads
+download.retries = 1
+
+# File download timeout (in milliseconds)
+download.timeout = 60000
+
+# Maximum size of downloaded files in bytes (required)
+download.max_size = 104857600
+
+# API creds
 twitter.auth = VW9Ybjdjb1pkd2J0U3kwTUh2VXVnOm9GTzVQVzNqM29LQU1xVGhnS3pFZzhKbGVqbXU0c2lHQ3JrUFNNZm8=
 tumblr.auth = v5kUqGQXUtmF7K0itri1DGtgTs0VQpbSEbh1jxYgj9d2Sq18F8
 gw.api = gonewild