Merge pull request #101 from JabRef/catalog-crawler

Full text crawlers
JabRef · Oct 9, 2015 · 18a6a0e · 18a6a0e
2 parents 3a89147 + 5f1419a
commit 18a6a0e
Show file tree

Hide file tree

Showing 33 changed files with 750 additions and 502 deletions.
diff --git a/CHANGELOG b/CHANGELOG
@@ -15,6 +15,7 @@
     - Feature: When pasting a Google search URL, meta data will be automatically stripped before insertion.
     - No longer write JabRef version to BibTex file header.
     - No longer add blank lines inside a bibtex entry
+    - Feature: PDF auto download from ACS, arXiv, ScienceDirect, SpringerLink, and Google Scholar
     - Perform syntax improvements enabled by Java 1.7+ (diamond operator, try-with-resources)
     - List of authors is now auto generated `scripts/generate-authors.sh` and inserted into L10N About.html
     - Remove Mr.DLib support as MR.DLib will be shut down in 2015

diff --git a/build.gradle b/build.gradle
@@ -75,6 +75,9 @@ dependencies {
     compile 'commons-logging:commons-logging:1.2'
 
     compile 'junit:junit:4.12'
+
+    compile 'org.jsoup:jsoup:1.8.3'
+    compile 'com.mashape.unirest:unirest-java:1.4.6'
 }
 
 sourceSets {

diff --git a/external-libraries.txt b/external-libraries.txt
@@ -49,6 +49,11 @@ Project: JGoodies Looks
 URL:     http://www.jgoodies.com/downloads/libraries/
 License: BSD
 
+Id:      com.mashape.unirest
+Project: Unirest for Java
+URL:     https://github.com/Mashape/unirest-java
+License: MIT
+
 Id:      mysql:mysql-connector-java
 Project: MySQL Connector/J
 URL:     http://www.mysql.de/downloads/connector/j/
@@ -79,6 +84,11 @@ Project: Apache PDFBox
 URL:     http://pdfbox.apache.org
 License: Apache 2.0
 
+Id:      org.jsoup:jsoup
+Project: jsoup
+URL:     https://github.com/jhy/jsoup/
+License: MIT
+
 Id:      org.openoffice:juh
 Project: OpenOffice.org
 URL:     http://www.openoffice.org/api/SDK

diff --git a/src/main/java/net/sf/jabref/external/ACSPdfDownload.java b/src/main/java/net/sf/jabref/external/ACSPdfDownload.java
diff --git a/src/main/java/net/sf/jabref/external/DownloadExternalFile.java b/src/main/java/net/sf/jabref/external/DownloadExternalFile.java
@@ -49,6 +49,7 @@
  * If the download is cancelled, or failed, the user is informed. The callback is never called.
  */
 public class DownloadExternalFile {
+    private static final Log LOGGER = LogFactory.getLog(DownloadExternalFile.class);
 
     private final JabRefFrame frame;
     private final MetaData metaData;
@@ -57,11 +58,7 @@ public class DownloadExternalFile {
     private boolean downloadFinished;
     private boolean dontShowDialog;
 
-    private static final Log LOGGER = LogFactory.getLog(DownloadExternalFile.class);
-
-
     public DownloadExternalFile(JabRefFrame frame, MetaData metaData, String bibtexKey) {
-
         this.frame = frame;
         this.metaData = metaData;
         this.bibtexKey = bibtexKey;
@@ -99,9 +96,7 @@ public void download(final DownloadCallback callback) throws IOException {
      *                 is complete.
      */
     public void download(URL url, final DownloadCallback callback) throws IOException {
-
         String res = url.toString();
-
         String mimeType;
 
         // First of all, start the download itself in the background to a temporary file:
@@ -110,7 +105,6 @@ public void download(URL url, final DownloadCallback callback) throws IOExceptio
 
         URLDownload udl = MonitoredURLDownload.buildMonitoredDownload(frame, url);
 
-        //long time = System.currentTimeMillis();
         try {
             // TODO: what if this takes long time?
             // TODO: stop editor dialog if this results in an error:
@@ -124,12 +118,10 @@ public void download(URL url, final DownloadCallback callback) throws IOExceptio
         }
         final URL urlF = url;
         final URLDownload udlF = udl;
-        //System.out.println("Time: "+(System.currentTimeMillis()-time));
-        JabRefExecutorService.INSTANCE.execute(new Runnable() {
 
+        JabRefExecutorService.INSTANCE.execute(new Runnable() {
             @Override
             public void run() {
-
                 try {
                     udlF.downloadToFile(tmp);
                 } catch (IOException e2) {
@@ -143,10 +135,8 @@ public void run() {
                     LOGGER.info("Error while downloading " + "'" + urlF + "'", e2);
                     return;
                 }
-
                 // Download finished: call the method that stops the progress bar etc.:
                 SwingUtilities.invokeLater(new Runnable() {
-
                     @Override
                     public void run() {
                         downloadFinished();
@@ -157,10 +147,8 @@ public void run() {
 
         ExternalFileType suggestedType = null;
         if (mimeType != null) {
-            System.out.println("mimetype:" + mimeType);
+            LOGGER.debug("MIME Type suggested: " + mimeType);
             suggestedType = Globals.prefs.getExternalFileTypeByMimeType(mimeType);
-            /*if (suggestedType != null)
-                System.out.println("Found type '"+suggestedType.getName()+"' by MIME type '"+udl.getMimeType()+"'");*/
         }
         // Then, while the download is proceeding, let the user choose the details of the file:
         String suffix;
@@ -172,7 +160,7 @@ public void run() {
             suggestedType = Globals.prefs.getExternalFileTypeByExt(suffix);
         }
 
-        String suggestedName = bibtexKey != null ? getSuggestedFileName(suffix) : "";
+        String suggestedName = getSuggestedFileName(suffix);
         String[] fDirectory = getFileDirectory(res);
         final String directory;
         if (fDirectory.length == 0) {
@@ -182,8 +170,7 @@ public void run() {
         }
         final String suggestDir = directory != null ? directory : System.getProperty("user.home");
         File file = new File(new File(suggestDir), suggestedName);
-        FileListEntry entry = new FileListEntry("", bibtexKey != null ? file.getCanonicalPath() : "",
-                suggestedType);
+        FileListEntry entry = new FileListEntry("", file.getCanonicalPath(), suggestedType);
         editor = new FileListEntryEditor(frame, entry, true, false, metaData);
         editor.getProgressBar().setIndeterminate(true);
         editor.setOkEnabled(false);
@@ -286,10 +273,9 @@ private void downloadFinished() {
         editor.setOkEnabled(true);
         editor.getProgressBar().setValue(editor.getProgressBar().getMaximum());
     }
-
+    // FIXME: will break download if no bibtexkey is present!
     private String getSuggestedFileName(String suffix) {
-
-        String plannedName = bibtexKey;
+        String plannedName = bibtexKey != null ? bibtexKey : "set-filename";
         if (!suffix.isEmpty()) {
             plannedName += "." + suffix;
         }
@@ -298,7 +284,10 @@ private String getSuggestedFileName(String suffix) {
         * [ 1548875 ] download pdf produces unsupported filename
         *
         * http://sourceforge.net/tracker/index.php?func=detail&aid=1548875&group_id=92314&atid=600306
-        *
+        * FIXME: rework this! just allow alphanumeric stuff or so?
+        * https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx#naming_conventions
+        * http://superuser.com/questions/358855/what-characters-are-safe-in-cross-platform-file-names-for-linux-windows-and-os
+        * https://support.apple.com/en-us/HT202808
         */
         if (OS.WINDOWS) {
             plannedName = plannedName.replaceAll("\\?|\\*|\\<|\\>|\\||\\\"|\\:|\\.$|\\[|\\]", "");
@@ -310,7 +299,7 @@ private String getSuggestedFileName(String suffix) {
     }
 
     /**
-     * Look for the last '.' in the link, and returnthe following characters.
+     * Look for the last '.' in the link, and return the following characters.
      * This gives the extension for most reasonably named links.
      *
      * @param link The link