Improve .bib from .pdf

CHANGELOG.md

@@ -12,6 +12,8 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv
 ### Added
 
 - We added support for selecting and using CSL Styles in JabRef's OpenOffice/LibreOffice integration for inserting bibliographic and in-text citations into a document. [#2146](https://github.com/JabRef/jabref/issues/2146), [#8893](https://github.com/JabRef/jabref/issues/8893)
+- We added Tools > New library based on references in PDF file... to create a new library based on the references section in a PDF file. [#11522](https://github.com/JabRef/jabref/pull/11522)
+- When converting the references section of a paper (PDF file), more than the last page is treated. [#11522](https://github.com/JabRef/jabref/pull/11522)
 - Added minimal support for [biblatex data annotation](https://mirrors.ctan.org/macros/latex/contrib/biblatex/doc/biblatex.pdf#subsection.3.7) fields in .layout files. [#11505](https://github.com/JabRef/jabref/issues/11505)
 - Added saving of selected options in the [Lookup -> Search for unlinked local files dialog](https://docs.jabref.org/collect/findunlinkedfiles#link-the-pdfs-to-your-bib-library). [#11439](https://github.com/JabRef/jabref/issues/11439)
 

src/main/java/org/jabref/architecture/ApacheCommonsLang3Allowed.java → src/main/java/org/jabref/architecture/AllowedToUseApacheCommonsLang3.java

@@ -4,7 +4,7 @@
  * Annotation to indicate that usage of ApacheCommonsLang3 is explicitly allowed.
  * The intention is to fully switch to Google Guava and only use Apache Commons Lang3 if there is no other possibility
  */
-public @interface ApacheCommonsLang3Allowed {
+public @interface AllowedToUseApacheCommonsLang3 {
 
     // The rationale
     String value();

src/main/java/org/jabref/gui/actions/StandardActions.java

@@ -93,6 +93,8 @@ public enum StandardActions implements Action {
 
     PARSE_LATEX(Localization.lang("Search for citations in LaTeX files..."), IconTheme.JabRefIcons.LATEX_CITATIONS),
     NEW_SUB_LIBRARY_FROM_AUX(Localization.lang("New sublibrary based on AUX file") + "...", Localization.lang("New BibTeX sublibrary") + Localization.lang("This feature generates a new library based on which entries are needed in an existing LaTeX document."), IconTheme.JabRefIcons.NEW),
+    NEW_LIBRARY_FROM_PDF_ONLINE(Localization.lang("New library based on references in PDF file... (online)"), Localization.lang("This feature generates a new library based on the list of references in a PDF file. Thereby, it uses GROBID's functionality."), IconTheme.JabRefIcons.NEW),
+    NEW_LIBRARY_FROM_PDF_OFFLINE(Localization.lang("New library based on references in PDF file... (offline)"), Localization.lang("This feature generates a new library based on the list of references in a PDF file. Thereby, it uses JabRef's build-in functionality.."), IconTheme.JabRefIcons.NEW),
     WRITE_METADATA_TO_PDF(Localization.lang("Write metadata to PDF files"), Localization.lang("Will write metadata to the PDFs linked from selected entries."), KeyBinding.WRITE_METADATA_TO_PDF),
 
     START_NEW_STUDY(Localization.lang("Start new systematic literature review")),

src/main/java/org/jabref/gui/frame/MainMenu.java

@@ -51,6 +51,7 @@
 import org.jabref.gui.journals.AbbreviateAction;
 import org.jabref.gui.libraryproperties.LibraryPropertiesAction;
 import org.jabref.gui.linkedfile.RedownloadMissingFilesAction;
+import org.jabref.gui.maintable.NewLibraryFromPdfAction;
 import org.jabref.gui.mergeentries.MergeEntriesAction;
 import org.jabref.gui.preferences.ShowPreferencesAction;
 import org.jabref.gui.preview.CopyCitationAction;
@@ -80,6 +81,8 @@
 import org.jabref.model.util.FileUpdateMonitor;
 import org.jabref.preferences.PreferencesService;
 
+import com.tobiasdiez.easybind.EasyBind;
+
 public class MainMenu extends MenuBar {
     private final JabRefFrame frame;
     private final FileHistoryMenu fileHistoryMenu;
@@ -273,9 +276,17 @@ private void createMenu() {
         final MenuItem pushToApplicationMenuItem = factory.createMenuItem(pushToApplicationCommand.getAction(), pushToApplicationCommand);
         pushToApplicationCommand.registerReconfigurable(pushToApplicationMenuItem);
 
+        NewLibraryFromPdfAction newLibraryFromPdfAction = new NewLibraryFromPdfAction(frame, stateManager, dialogService, preferencesService, taskExecutor);
+        // Action used twice, because it distinguishes internally between online and offline
+        // We want the UI to show "online" and "offline" explicitly
+        MenuItem newLibraryFromPdfMenuItemOnline = factory.createMenuItem(StandardActions.NEW_LIBRARY_FROM_PDF_ONLINE, newLibraryFromPdfAction);
+        MenuItem newLibraryFromPdfMenuItemOffline = factory.createMenuItem(StandardActions.NEW_LIBRARY_FROM_PDF_OFFLINE, newLibraryFromPdfAction);
+
         tools.getItems().addAll(
                 factory.createMenuItem(StandardActions.PARSE_LATEX, new ParseLatexAction(stateManager)),
                 factory.createMenuItem(StandardActions.NEW_SUB_LIBRARY_FROM_AUX, new NewSubLibraryAction(frame, stateManager, dialogService)),
+                newLibraryFromPdfMenuItemOnline,
+                newLibraryFromPdfMenuItemOffline,
 
                 new SeparatorMenuItem(),
 
@@ -303,6 +314,9 @@ private void createMenu() {
 
                 factory.createMenuItem(StandardActions.REDOWNLOAD_MISSING_FILES, new RedownloadMissingFilesAction(stateManager, dialogService, preferencesService.getFilePreferences(), taskExecutor))
         );
+
+        EasyBind.subscribe(preferencesService.getGrobidPreferences().grobidEnabledProperty(), newLibraryFromPdfMenuItemOnline::setVisible);
+
         SidePaneType webSearchPane = SidePaneType.WEB_SEARCH;
         SidePaneType groupsPane = SidePaneType.GROUPS;
         SidePaneType openOfficePane = SidePaneType.OPEN_OFFICE;

src/main/java/org/jabref/gui/maintable/ExtractReferencesAction.java

@@ -15,7 +15,6 @@
 import org.jabref.gui.actions.SimpleCommand;
 import org.jabref.gui.importer.ImportEntriesDialog;
 import org.jabref.gui.util.BackgroundTask;
-import org.jabref.gui.util.TaskExecutor;
 import org.jabref.logic.importer.ParserResult;
 import org.jabref.logic.importer.fileformat.BibliographyFromPdfImporter;
 import org.jabref.logic.importer.util.GrobidService;
@@ -33,30 +32,28 @@
 /**
  * SIDE EFFECT: Sets the "cites" field of the entry having the linked files
  *
- * Mode choice A: online or offline
- * Mode choice B: complete entry or single file (the latter is not implemented)
+ * <ul>
+ *   <li>Mode choice A: online or offline</li>
+ *   <li>Mode choice B: complete entry or single file (the latter is not implemented)</li>
+ * </ul>
  *
- * The different modes should be implemented as sub classes. However, this was too complicated, thus we use variables at the constructor to parameterize this class.
+ * The mode is selected by the preferences whether to use Grobid or not.
  */
 public class ExtractReferencesAction extends SimpleCommand {
     private final int FILES_LIMIT = 10;
 
-    private final boolean online;
     private final DialogService dialogService;
     private final StateManager stateManager;
     private final PreferencesService preferencesService;
     private final BibEntry entry;
     private final LinkedFile linkedFile;
-    private final TaskExecutor taskExecutor;
 
     private final BibliographyFromPdfImporter bibliographyFromPdfImporter;
 
-    public ExtractReferencesAction(boolean online,
-                                   DialogService dialogService,
+    public ExtractReferencesAction(DialogService dialogService,
                                    StateManager stateManager,
-                                   PreferencesService preferencesService,
-                                   TaskExecutor taskExecutor) {
-        this(online, dialogService, stateManager, preferencesService, null, null, taskExecutor);
+                                   PreferencesService preferencesService) {
+        this(dialogService, stateManager, preferencesService, null, null);
     }
 
     /**
@@ -65,20 +62,16 @@ public ExtractReferencesAction(boolean online,
      * @param entry the entry to handle (can be null)
      * @param linkedFile the linked file (can be null)
      */
-    private ExtractReferencesAction(boolean online,
-                                    @NonNull DialogService dialogService,
+    private ExtractReferencesAction(@NonNull DialogService dialogService,
                                     @NonNull StateManager stateManager,
                                     @NonNull PreferencesService preferencesService,
                                     @Nullable BibEntry entry,
-                                    @Nullable LinkedFile linkedFile,
-                                    @NonNull TaskExecutor taskExecutor) {
-        this.online = online;
+                                    @Nullable LinkedFile linkedFile) {
         this.dialogService = dialogService;
         this.stateManager = stateManager;
         this.preferencesService = preferencesService;
         this.entry = entry;
         this.linkedFile = linkedFile;
-        this.taskExecutor = taskExecutor;
         bibliographyFromPdfImporter = new BibliographyFromPdfImporter(preferencesService.getCitationKeyPatternPreferences());
 
         if (this.linkedFile == null) {
@@ -98,15 +91,14 @@ public void execute() {
 
     private void extractReferences() {
         stateManager.getActiveDatabase().ifPresent(databaseContext -> {
-            assert online == this.preferencesService.getGrobidPreferences().isGrobidEnabled();
-
             List<BibEntry> selectedEntries;
             if (entry == null) {
                 selectedEntries = stateManager.getSelectedEntries();
             } else {
                 selectedEntries = List.of(entry);
             }
 
+            boolean online = this.preferencesService.getGrobidPreferences().isGrobidEnabled();
             Callable<ParserResult> parserResultCallable;
             if (online) {
                 Optional<Callable<ParserResult>> parserResultCallableOnline = getParserResultCallableOnline(databaseContext, selectedEntries);
@@ -166,9 +158,21 @@ private void extractReferences(Iterator<Path> fileListIterator, ParserResult res
             result.getDatabase().insertEntries(bibliographyFromPdfImporter.importDatabase(fileListIterator.next()).getDatabase().getEntries());
         }
 
+        String cites = getCites(result.getDatabase().getEntries(), currentEntry);
+        currentEntry.setField(StandardField.CITES, cites);
+    }
+
+    /**
+     * Creates the field content for the "cites" field. The field contains the citation keys of the imported entries.
+     *
+     * TODO: Move this part to logic somehow
+     *
+     * @param currentEntry used to create citation keys if the importer did not provide one from the imported entry
+     */
+    private static String getCites(List<BibEntry> entries, BibEntry currentEntry) {
         StringJoiner cites = new StringJoiner(",");
         int count = 0;
-        for (BibEntry importedEntry : result.getDatabase().getEntries()) {
+        for (BibEntry importedEntry : entries) {
             count++;
             Optional<String> citationKey = importedEntry.getCitationKey();
             String citationKeyToAdd;
@@ -195,7 +199,7 @@ private void extractReferences(Iterator<Path> fileListIterator, ParserResult res
             }
             cites.add(citationKeyToAdd);
         }
-        currentEntry.setField(StandardField.CITES, cites.toString());
+        return cites.toString();
     }
 
     private Optional<Callable<ParserResult>> getParserResultCallableOnline(BibDatabaseContext databaseContext, List<BibEntry> selectedEntries) {

src/main/java/org/jabref/gui/maintable/NewLibraryFromPdfAction.java

@@ -0,0 +1,100 @@
+package org.jabref.gui.maintable;
+
+import java.nio.file.Path;
+import java.util.concurrent.Callable;
+
+import javafx.application.Platform;
+
+import org.jabref.gui.DialogService;
+import org.jabref.gui.LibraryTabContainer;
+import org.jabref.gui.StateManager;
+import org.jabref.gui.actions.SimpleCommand;
+import org.jabref.gui.util.BackgroundTask;
+import org.jabref.gui.util.FileDialogConfiguration;
+import org.jabref.gui.util.TaskExecutor;
+import org.jabref.logic.importer.ParserResult;
+import org.jabref.logic.importer.fileformat.BibliographyFromPdfImporter;
+import org.jabref.logic.importer.util.GrobidService;
+import org.jabref.logic.l10n.Localization;
+import org.jabref.logic.util.StandardFileType;
+import org.jabref.model.database.BibDatabaseContext;
+import org.jabref.preferences.PreferencesService;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Similar to {@link ExtractReferencesAction}. This action creates a new library, the other action "just" appends to the current library
+ *
+ * <ul>
+ *   <li>Mode choice A: online or offline</li>
+ *   <li>Mode choice B: complete entry or single file (the latter is not implemented)</li>
+ * </ul>
+ * <p>
+ * The mode is selected by the preferences whether to use Grobid or not.
+ * <p>
+ * The different modes should be implemented as sub classes. Moreover, there are synergies with {@link ExtractReferencesAction}. However, this was too complicated.
+ */
+public class NewLibraryFromPdfAction extends SimpleCommand {
+    private static final Logger LOGGER = LoggerFactory.getLogger(NewLibraryFromPdfAction .class);
+
+    private final LibraryTabContainer libraryTabContainer;
+    private final StateManager stateManager;
+    private final DialogService dialogService;
+    private final PreferencesService preferencesService;
+    private final BibliographyFromPdfImporter bibliographyFromPdfImporter;
+    private final TaskExecutor taskExecutor;
+
+    public NewLibraryFromPdfAction(
+            LibraryTabContainer libraryTabContainer,
+            StateManager stateManager,
+            DialogService dialogService,
+            PreferencesService preferencesService,
+            TaskExecutor taskExecutor) {
+        this.libraryTabContainer = libraryTabContainer;
+        this.stateManager = stateManager;
+        this.dialogService = dialogService;
+        this.preferencesService = preferencesService;
+        // Instruct the importer to keep the numbers (instead of generating keys)
+        this.bibliographyFromPdfImporter = new BibliographyFromPdfImporter();
+        this.taskExecutor = taskExecutor;
+    }
+
+    @Override
+    public void execute() {
+        final FileDialogConfiguration.Builder builder = new FileDialogConfiguration.Builder();
+        builder.withDefaultExtension(StandardFileType.PDF);
+        // Sensible default for the directory to start browsing is the directory of the currently opened library. The pdf storage dir seems not to be feasible, because extracting references from a PDF itself can be done by the context menu of the respective entry.
+        stateManager.getActiveDatabase()
+                    .flatMap(BibDatabaseContext::getDatabasePath)
+                    .ifPresent(path -> builder.withInitialDirectory(path.getParent()));
+        FileDialogConfiguration fileDialogConfiguration = builder.build();
+
+        LOGGER.trace("Opening file dialog with configuration: {}", fileDialogConfiguration);
+
+        dialogService.showFileOpenDialog(fileDialogConfiguration).ifPresent(path -> {
+            LOGGER.trace("Selected file: {}", path);
+            Callable<ParserResult> parserResultCallable = getParserResultCallable(path);
+            BackgroundTask.wrap(parserResultCallable)
+                          .withInitialMessage(Localization.lang("Processing PDF(s)"))
+                          .onFailure(dialogService::showErrorDialogAndWait)
+                          .onSuccess(result -> {
+                              LOGGER.trace("Finished processing PDF(s): {}", result);
+                              libraryTabContainer.addTab(result.getDatabaseContext(), true);
+                          })
+                          .executeWith(taskExecutor);
+        });
+    }
+
+    private Callable<ParserResult> getParserResultCallable(Path path) {
+        Callable<ParserResult> parserResultCallable;
+        boolean online = this.preferencesService.getGrobidPreferences().isGrobidEnabled();
+        if (online) {
+            parserResultCallable = () -> new ParserResult(
+                    new GrobidService(this.preferencesService.getGrobidPreferences()).processReferences(path, preferencesService.getImportFormatPreferences()));
+        } else {
+            parserResultCallable = () -> bibliographyFromPdfImporter.importDatabase(path);
+        }
+        return parserResultCallable;
+    }
+}

src/main/java/org/jabref/gui/maintable/RightClickMenu.java

@@ -53,8 +53,10 @@ public static ContextMenu create(BibEntryTableViewModel entry,
         ActionFactory factory = new ActionFactory();
         ContextMenu contextMenu = new ContextMenu();
 
-        MenuItem extractFileReferencesOnline = factory.createMenuItem(StandardActions.EXTRACT_FILE_REFERENCES_ONLINE, new ExtractReferencesAction(true, dialogService, stateManager, preferencesService, taskExecutor));
-        MenuItem extractFileReferencesOffline = factory.createMenuItem(StandardActions.EXTRACT_FILE_REFERENCES_OFFLINE, new ExtractReferencesAction(false, dialogService, stateManager, preferencesService, taskExecutor));
+        ExtractReferencesAction extractReferencesAction = new ExtractReferencesAction(dialogService, stateManager, preferencesService);
+        // Two menu items required, because of menu item display. Action checks preference internal what to do
+        MenuItem extractFileReferencesOnline = factory.createMenuItem(StandardActions.EXTRACT_FILE_REFERENCES_ONLINE, extractReferencesAction);
+        MenuItem extractFileReferencesOffline = factory.createMenuItem(StandardActions.EXTRACT_FILE_REFERENCES_OFFLINE, extractReferencesAction);
 
         contextMenu.getItems().addAll(
                 factory.createMenuItem(StandardActions.COPY, new EditAction(StandardActions.COPY, () -> libraryTab, stateManager, undoManager)),

src/main/java/org/jabref/gui/util/UiTaskExecutor.java

@@ -38,9 +38,6 @@ public class UiTaskExecutor implements TaskExecutor {
     private final ScheduledExecutorService scheduledExecutor = Executors.newScheduledThreadPool(2);
     private final WeakHashMap<DelayTaskThrottler, Void> throttlers = new WeakHashMap<>();
 
-    /**
-     *
-     */
     public static <V> V runInJavaFXThread(Callable<V> callable) {
         if (Platform.isFxApplicationThread()) {
             try {