1.0.28 - Added Vk video (single and albums) rippers #8

pom.xml

@@ -4,7 +4,7 @@
   <groupId>com.rarchives.ripme</groupId>
   <artifactId>ripme</artifactId>
   <packaging>jar</packaging>
-  <version>1.0.27</version>
+  <version>1.0.28</version>
   <name>ripme</name>
   <url>http://rip.rarchives.com</url>
   <properties>

src/main/java/com/rarchives/ripme/ripper/rippers/VkRipper.java

@@ -32,7 +32,16 @@ public VkRipper(URL url) throws IOException {
 
     @Override
     public boolean canRip(URL url) {
-        return url.getHost().endsWith(DOMAIN);
+        if (!url.getHost().endsWith(DOMAIN)) {
+            return false;
+        }
+        // Ignore /video pages (but not /videos pages)
+        String u = url.toExternalForm();
+        if (u.contains("/video") && !u.contains("videos")) {
+            // Single video page
+            return false;
+        }
+        return true;
     }
 
     @Override
@@ -42,6 +51,50 @@ public URL sanitizeURL(URL url) throws MalformedURLException {
 
     @Override
     public void rip() throws IOException {
+        if (this.url.toExternalForm().contains("/videos")) {
+            ripVideos();
+        }
+        else {
+            ripImages();
+        }
+    }
+
+    private void ripVideos() throws IOException {
+        String oid = getGID(this.url).replace("videos", "");
+        String u = "http://vk.com/al_video.php";
+        Map<String,String> postData = new HashMap<String,String>();
+        postData.put("al", "1");
+        postData.put("act", "load_videos_silent");
+        postData.put("offset", "0");
+        postData.put("oid", oid);
+        Document doc = Jsoup.connect(u)
+                            .header("Referer", this.url.toExternalForm())
+                            .ignoreContentType(true)
+                            .userAgent(USER_AGENT)
+                            .timeout(5000)
+                            .data(postData)
+                            .post();
+        String[] jsonStrings = doc.toString().split("<!>");
+        JSONObject json = new JSONObject(jsonStrings[jsonStrings.length - 1]);
+        JSONArray videos = json.getJSONArray("all");
+        logger.info("Found " + videos.length() + " videos");
+        for (int i = 0; i < videos.length(); i++) {
+            JSONArray jsonVideo = videos.getJSONArray(i);
+            int vidid = jsonVideo.getInt(1);
+            String videoURL = com.rarchives.ripme.ripper.rippers.video.VkRipper.getVideoURLAtPage(
+                    "http://vk.com/video" + oid + "_" + vidid);
+            addURLToDownload(new URL(videoURL), String.format("%03d_", i + 1));
+            try {
+                Thread.sleep(500);
+            } catch (InterruptedException e) {
+                logger.error("Interrupted while waiting to fetch next video URL", e);
+                break;
+            }
+        }
+        waitForThreads();
+    }
+
+    private void ripImages() throws IOException {
         Map<String,String> photoIDsToURLs = new HashMap<String,String>();
         int offset = 0;
         while (true) {
@@ -146,7 +199,7 @@ public String getHost() {
 
     @Override
     public String getGID(URL url) throws MalformedURLException {
-        Pattern p = Pattern.compile("^https?://(www\\.)?vk\\.com/(photos|album)([a-zA-Z0-9_]{1,}).*$");
+        Pattern p = Pattern.compile("^https?://(www\\.)?vk\\.com/(photos|album|videos)([a-zA-Z0-9_]{1,}).*$");
         Matcher m = p.matcher(url.toExternalForm());
         if (!m.matches()) {
             throw new MalformedURLException("Expected format: http://vk.com/album#### or vk.com/photos####");

src/main/java/com/rarchives/ripme/ripper/rippers/video/VkRipper.java

@@ -0,0 +1,83 @@
+package com.rarchives.ripme.ripper.rippers.video;
+
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.log4j.Logger;
+import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
+
+import com.rarchives.ripme.ripper.VideoRipper;
+
+public class VkRipper extends VideoRipper {
+
+    private static final String HOST = "vk";
+    private static final Logger logger = Logger.getLogger(VkRipper.class);
+
+    public VkRipper(URL url) throws IOException {
+        super(url);
+    }
+
+    @Override
+    public String getHost() {
+        return HOST;
+    }
+
+    @Override
+    public boolean canRip(URL url) {
+        Pattern p = Pattern.compile("^https?://[wm.]*vk\\.com/video[0-9]+.*$");
+        Matcher m = p.matcher(url.toExternalForm());
+        return m.matches();
+    }
+
+    @Override
+    public URL sanitizeURL(URL url) throws MalformedURLException {
+        return url;
+    }
+
+    @Override
+    public String getGID(URL url) throws MalformedURLException {
+        Pattern p = Pattern.compile("^https?://[wm.]*vk\\.com/video([0-9]+).*$");
+        Matcher m = p.matcher(url.toExternalForm());
+        if (m.matches()) {
+            return m.group(1);
+        }
+
+        throw new MalformedURLException(
+                "Expected vk video URL format:"
+                        + "vk.com/videos####"
+                        + " Got: " + url);
+    }
+
+    @Override
+    public void rip() throws IOException {
+        logger.info("    Retrieving " + this.url);
+        String videoURL = getVideoURLAtPage(this.url.toExternalForm());
+        addURLToDownload(new URL(videoURL), HOST + "_" + getGID(this.url));
+        waitForThreads();
+    }
+
+    public static String getVideoURLAtPage(String url) throws IOException {
+        Document doc = Jsoup.connect(url)
+                            .userAgent(USER_AGENT)
+                            .get();
+        String html = doc.outerHtml();
+        String videoURL = null;
+        for (String quality : new String[] {"1080", "720", "480", "240"}) {
+            quality = "url" + quality + "\\\":\\\"";
+            if (html.contains(quality)) {
+                videoURL = html.substring(html.indexOf(quality) + quality.length());
+                videoURL = videoURL.substring(0, videoURL.indexOf("\""));
+                videoURL = videoURL.replace("\\", "");
+                break;
+            }
+        }
+        if (videoURL == null) {
+            throw new IOException("Could not find video URL at " + url);
+        }
+        return videoURL;
+    }
+}

src/main/java/com/rarchives/ripme/ui/UpdateUtils.java

@@ -19,7 +19,7 @@
 public class UpdateUtils {
 
     private static final Logger logger = Logger.getLogger(UpdateUtils.class);
-    private static final String DEFAULT_VERSION = "1.0.27";
+    private static final String DEFAULT_VERSION = "1.0.28";
     private static final String updateJsonURL = "http://rarchives.com/ripme.json";
     private static final String updateJarURL = "http://rarchives.com/ripme.jar";
     private static final String mainFileName = "ripme.jar";