-
Notifications
You must be signed in to change notification settings - Fork 631
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Added instagram ripper, integration tests.
Also fixed parts of the imgur ripper.
- Loading branch information
Showing
7 changed files
with
296 additions
and
9 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
140 changes: 140 additions & 0 deletions
140
src/main/java/com/rarchives/ripme/ripper/rippers/InstagramRipper.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,140 @@ | ||
| package com.rarchives.ripme.ripper.rippers; | ||
|
|
||
| import java.io.IOException; | ||
| import java.net.MalformedURLException; | ||
| import java.net.URL; | ||
| import java.util.regex.Matcher; | ||
| import java.util.regex.Pattern; | ||
|
|
||
| import org.apache.log4j.Logger; | ||
| import org.json.JSONArray; | ||
| import org.json.JSONObject; | ||
| import org.jsoup.Jsoup; | ||
| import org.jsoup.nodes.Document; | ||
| import org.jsoup.nodes.Element; | ||
|
|
||
| import com.rarchives.ripme.ripper.AbstractRipper; | ||
|
|
||
| public class InstagramRipper extends AbstractRipper { | ||
|
|
||
| private static final String DOMAIN = "instagram.com", | ||
| HOST = "instagram"; | ||
| private static final Logger logger = Logger.getLogger(ImagearnRipper.class); | ||
|
|
||
| public InstagramRipper(URL url) throws IOException { | ||
| super(url); | ||
| } | ||
|
|
||
| @Override | ||
| public boolean canRip(URL url) { | ||
| return url.getHost().endsWith(DOMAIN); | ||
| } | ||
|
|
||
| @Override | ||
| public URL sanitizeURL(URL url) throws MalformedURLException { | ||
| Pattern p = Pattern.compile("^https?://instagram\\.com/p/([a-zA-Z0-9]{1,}).*$"); | ||
| Matcher m = p.matcher(url.toExternalForm()); | ||
| if (m.matches()) { | ||
| // Link to photo, not the user account | ||
| try { | ||
| url = getUserPageFromImage(url); | ||
| } catch (Exception e) { | ||
| logger.error("[!] Failed to get user page from " + url, e); | ||
| throw new MalformedURLException("Failed to retrieve user page from " + url); | ||
| } | ||
| } | ||
| p = Pattern.compile("^.*instagram.com/([a-zA-Z0-9]{3,}).*$"); | ||
| m = p.matcher(url.toExternalForm()); | ||
| if (!m.matches()) { | ||
| throw new MalformedURLException("Expected username in URL (instagram.com/username and not " + url); | ||
| } | ||
| return new URL("http://statigr.am/" + m.group(1)); | ||
| } | ||
|
|
||
| private URL getUserPageFromImage(URL url) throws IOException { | ||
| Document doc = Jsoup.connect(url.toExternalForm()).get(); | ||
| for (Element element : doc.select("meta[property='og:description']")) { | ||
| String content = element.attr("content"); | ||
| if (content.endsWith("'s photo on Instagram")) { | ||
| return new URL("http://statigr.am/" + content.substring(0, content.indexOf("'"))); | ||
| } | ||
| } | ||
| throw new MalformedURLException("Expected username in URL (instagram.com/username and not " + url); | ||
| } | ||
|
|
||
| private String getUserID(URL url) throws IOException { | ||
| logger.info(" Retrieving " + url); | ||
| Document doc = Jsoup.connect(this.url.toExternalForm()).get(); | ||
| for (Element element : doc.select("input[id=user_public]")) { | ||
| return element.attr("value"); | ||
| } | ||
| throw new IOException("Unable to find userID at " + this.url); | ||
| } | ||
|
|
||
| @Override | ||
| public void rip() throws IOException { | ||
| int index = 0; | ||
| String userID = getUserID(this.url); | ||
| String baseURL = "http://statigr.am/controller_nl.php?action=getPhotoUserPublic&user_id=" + userID; | ||
| String params = ""; | ||
| while (true) { | ||
| String url = baseURL + params; | ||
| logger.info(" Retrieving " + url); | ||
| String jsonString = Jsoup.connect(url).ignoreContentType(true).execute().body(); | ||
| JSONObject json = new JSONObject(jsonString); | ||
| JSONArray datas = json.getJSONArray("data"); | ||
| String nextMaxID = ""; | ||
| if (datas.length() == 0) { | ||
| break; | ||
| } | ||
| for (int i = 0; i < datas.length(); i++) { | ||
| JSONObject data = (JSONObject) datas.get(i); | ||
| if (data.has("id")) { | ||
| nextMaxID = data.getString("id"); | ||
| } | ||
| if (data.has("videos")) { | ||
| index += 1; | ||
| String video = data.getJSONObject("videos").getJSONObject("standard_resolution").getString("url"); | ||
| addURLToDownload(new URL(video), String.format("%03d_", index)); | ||
| } else if (data.has("images")) { | ||
| index += 1; | ||
| String image = data.getJSONObject("images").getJSONObject("standard_resolution").getString("url"); | ||
| // addURLToDownload(new URL(image), String.format("%03d_", index)); | ||
| addURLToDownload(new URL(image)); | ||
| } | ||
| } | ||
| JSONObject pagination = json.getJSONObject("pagination"); | ||
| if (nextMaxID.equals("")) { | ||
| if (!pagination.has("next_max_id")) { | ||
| break; | ||
| } else { | ||
| nextMaxID = pagination.getString("next_max_id"); | ||
| } | ||
| } | ||
| params = "&max_id=" + nextMaxID; | ||
| try { | ||
| Thread.sleep(3000); | ||
| } catch (InterruptedException e) { | ||
| logger.error("[!] Interrupted while waiting to load next album:", e); | ||
| break; | ||
| } | ||
| } | ||
| waitForThreads(); | ||
| } | ||
|
|
||
| @Override | ||
| public String getHost() { | ||
| return HOST; | ||
| } | ||
|
|
||
| @Override | ||
| public String getGID(URL url) throws MalformedURLException { | ||
| Pattern p = Pattern.compile("^https?://statigr.am/([a-zA-Z0-9]{3,}).*$"); | ||
| Matcher m = p.matcher(url.toExternalForm()); | ||
| if (m.matches()) { | ||
| return m.group(1); | ||
| } | ||
| throw new MalformedURLException("Unable to find user in " + url); | ||
| } | ||
|
|
||
| } |
2 changes: 1 addition & 1 deletion
2
...est/java/com/rarchives/ripme/AppTest.java → ...java/com/rarchives/ripme/tst/AppTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
78 changes: 78 additions & 0 deletions
78
src/test/java/com/rarchives/ripme/tst/ripper/rippers/ImgurRipperTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,78 @@ | ||
| package com.rarchives.ripme.tst.ripper.rippers; | ||
|
|
||
| import java.io.IOException; | ||
| import java.net.URL; | ||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
|
|
||
| import com.rarchives.ripme.ripper.rippers.ImgurRipper; | ||
|
|
||
| public class ImgurRipperTest extends RippersTest { | ||
|
|
||
| public void testImgurURLFailures() throws IOException { | ||
| List<URL> failURLs = new ArrayList<URL>(); | ||
| // Imgur urls that should not work | ||
| failURLs.add(new URL("http://imgur.com")); | ||
| failURLs.add(new URL("http://imgur.com/")); | ||
| failURLs.add(new URL("http://i.imgur.com")); | ||
| failURLs.add(new URL("http://i.imgur.com/")); | ||
| failURLs.add(new URL("http://imgur.com/image")); | ||
| failURLs.add(new URL("http://imgur.com/image.jpg")); | ||
| failURLs.add(new URL("http://i.imgur.com/image.jpg")); | ||
| for (URL url : failURLs) { | ||
| try { | ||
| new ImgurRipper(url); | ||
| fail("Instantiated ripper for URL that should not work: " + url); | ||
| } catch (Exception e) { | ||
| // Expected | ||
| continue; | ||
| } | ||
| } | ||
| } | ||
|
|
||
| public void testImgurURLPasses() throws IOException { | ||
| List<URL> passURLs = new ArrayList<URL>(); | ||
| // Imgur URLs that should work | ||
| passURLs.add(new URL("http://imgur.com/a/XPd4F")); | ||
| passURLs.add(new URL("http://imgur.com/a/XPd4F/")); | ||
| passURLs.add(new URL("http://imgur.com/a/WxG6f/all")); | ||
| passURLs.add(new URL("http://imgur.com/a/WxG6f/layout/vertical#0")); | ||
| passURLs.add(new URL("http://imgur.com/a/WxG6f/layout/horizontal#0")); | ||
| passURLs.add(new URL("http://imgur.com/a/WxG6f/layout/grid#0")); | ||
| passURLs.add(new URL("http://imgur.com/YOdjht3,x5VxH9G,5juXjJ2")); | ||
| passURLs.add(new URL("http://markedone911.imgur.com")); | ||
| passURLs.add(new URL("http://markedone911.imgur.com/")); | ||
|
|
||
| for (URL url : passURLs) { | ||
| try { | ||
| ImgurRipper ripper = new ImgurRipper(url); | ||
| assertTrue(ripper.canRip(url)); | ||
| deleteDir(ripper.getWorkingDir()); | ||
| } catch (Exception e) { | ||
| fail("Failed to instantiate ripper for " + url); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| public void testImgurAlbums() throws IOException { | ||
| List<URL> contentURLs = new ArrayList<URL>(); | ||
| // URLs that should return more than 1 image | ||
| contentURLs.add(new URL("http://imgur.com/a/hqJIu")); // Vertical layout | ||
| contentURLs.add(new URL("http://imgur.com/a/dS9OQ#0")); // Horizontal layout | ||
| contentURLs.add(new URL("http://imgur.com/a/YpsW9#0")); // Grid layout | ||
| contentURLs.add(new URL("http://imgur.com/a/WxG6f/layout/vertical#0")); | ||
| contentURLs.add(new URL("http://imgur.com/a/WxG6f/layout/horizontal#0")); | ||
| contentURLs.add(new URL("http://imgur.com/a/WxG6f/layout/grid#0")); | ||
| for (URL url : contentURLs) { | ||
| try { | ||
| ImgurRipper ripper = new ImgurRipper(url); | ||
| ripper.rip(); | ||
| assert(ripper.getWorkingDir().listFiles().length > 1); | ||
| deleteDir(ripper.getWorkingDir()); | ||
| } catch (Exception e) { | ||
| fail("Error while ripping URL " + url + ": " + e.getMessage()); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| } |
28 changes: 28 additions & 0 deletions
28
src/test/java/com/rarchives/ripme/tst/ripper/rippers/InstagramRipperTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,28 @@ | ||
| package com.rarchives.ripme.tst.ripper.rippers; | ||
|
|
||
| import java.io.IOException; | ||
| import java.net.URL; | ||
| import java.util.ArrayList; | ||
| import java.util.List; | ||
|
|
||
| import com.rarchives.ripme.ripper.rippers.InstagramRipper; | ||
|
|
||
|
|
||
| public class InstagramRipperTest extends RippersTest { | ||
|
|
||
| public void testInstagramAlbums() throws IOException { | ||
| List<URL> contentURLs = new ArrayList<URL>(); | ||
| contentURLs.add(new URL("http://instagram.com/feelgoodincc#")); | ||
| for (URL url : contentURLs) { | ||
| try { | ||
| InstagramRipper ripper = new InstagramRipper(url); | ||
| ripper.rip(); | ||
| assert(ripper.getWorkingDir().listFiles().length > 1); | ||
| deleteDir(ripper.getWorkingDir()); | ||
| } catch (Exception e) { | ||
| fail("Error while ripping URL " + url + ": " + e.getMessage()); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| } |
22 changes: 22 additions & 0 deletions
22
src/test/java/com/rarchives/ripme/tst/ripper/rippers/RippersTest.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| package com.rarchives.ripme.tst.ripper.rippers; | ||
|
|
||
| import java.io.File; | ||
|
|
||
| import junit.framework.TestCase; | ||
|
|
||
| public class RippersTest extends TestCase { | ||
|
|
||
| protected void deleteDir(File dir) { | ||
| return; | ||
| /* | ||
| for (File f : dir.listFiles()) { | ||
| if (f.isDirectory()) { | ||
| deleteDir(f); | ||
| } | ||
| f.delete(); | ||
| } | ||
| dir.delete(); | ||
| //*/ | ||
| } | ||
|
|
||
| } |