Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add logic for parsing references from last page of PDF #11156

Merged
merged 19 commits into from
Apr 8, 2024
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv

### Added

- We added support for parsing the "References" section of IEEE papers offline. [#11156](https://github.com/JabRef/jabref/pull/11156)

### Changed

### Fixed
Expand Down
7 changes: 4 additions & 3 deletions src/main/java/org/jabref/gui/actions/ActionFactory.java
Original file line number Diff line number Diff line change
Expand Up @@ -78,8 +78,11 @@ private static Label getAssociatedNode(MenuItem menuItem) {
public MenuItem configureMenuItem(Action action, Command command, MenuItem menuItem) {
ActionUtils.configureMenuItem(new JabRefAction(action, command, keyBindingRepository, Sources.FromMenu), menuItem);
setGraphic(menuItem, action);
enableTooltips(command, menuItem);
return menuItem;
}

// Show tooltips
private static void enableTooltips(Command command, MenuItem menuItem) {
if (command instanceof SimpleCommand simpleCommand) {
EasyBind.subscribe(
simpleCommand.statusMessageProperty(),
Expand All @@ -96,8 +99,6 @@ public MenuItem configureMenuItem(Action action, Command command, MenuItem menuI
}
);
}

return menuItem;
}

public MenuItem createMenuItem(Action action, Command command) {
Expand Down
4 changes: 0 additions & 4 deletions src/main/java/org/jabref/gui/actions/SimpleCommand.java
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,6 @@ public abstract class SimpleCommand extends CommandBase {

protected ReadOnlyStringWrapper statusMessage = new ReadOnlyStringWrapper("");

public String getStatusMessage() {
return statusMessage.get();
}

public ReadOnlyStringProperty statusMessageProperty() {
return statusMessage.getReadOnlyProperty();
}
Expand Down
3 changes: 2 additions & 1 deletion src/main/java/org/jabref/gui/actions/StandardActions.java
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,8 @@ public enum StandardActions implements Action {
REBUILD_FULLTEXT_SEARCH_INDEX(Localization.lang("Rebuild fulltext search index"), IconTheme.JabRefIcons.FILE),
REDOWNLOAD_MISSING_FILES(Localization.lang("Redownload missing files"), IconTheme.JabRefIcons.DOWNLOAD),
OPEN_EXTERNAL_FILE(Localization.lang("Open file"), IconTheme.JabRefIcons.FILE, KeyBinding.OPEN_FILE),
EXTRACT_FILE_REFERENCES(Localization.lang("Extract references from file"), IconTheme.JabRefIcons.FILE_STAR),
EXTRACT_FILE_REFERENCES_ONLINE(Localization.lang("Extract references from file (online)"), IconTheme.JabRefIcons.FILE_STAR),
EXTRACT_FILE_REFERENCES_OFFLINE(Localization.lang("Extract references from file (offline)"), IconTheme.JabRefIcons.FILE_STAR),
OPEN_URL(Localization.lang("Open URL or DOI"), IconTheme.JabRefIcons.WWW, KeyBinding.OPEN_URL_OR_DOI),
SEARCH_SHORTSCIENCE(Localization.lang("Search ShortScience")),
MERGE_WITH_FETCHED_ENTRY(Localization.lang("Get bibliographic data from %0", "DOI/ISBN/...")),
Expand Down
159 changes: 136 additions & 23 deletions src/main/java/org/jabref/gui/maintable/ExtractReferencesAction.java
Original file line number Diff line number Diff line change
@@ -1,9 +1,13 @@
package org.jabref.gui.maintable;

import java.nio.file.Path;
import java.util.LinkedList;
import java.util.Iterator;
import java.util.List;
import java.util.Optional;
import java.util.StringJoiner;
import java.util.concurrent.Callable;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.jabref.gui.DialogService;
import org.jabref.gui.StateManager;
Expand All @@ -13,48 +17,78 @@
import org.jabref.gui.util.BackgroundTask;
import org.jabref.gui.util.TaskExecutor;
import org.jabref.logic.importer.ParserResult;
import org.jabref.logic.importer.fetcher.GrobidPreferences;
import org.jabref.logic.importer.fileformat.BibliographyFromPdfImporter;
import org.jabref.logic.importer.util.GrobidService;
import org.jabref.logic.l10n.Localization;
import org.jabref.logic.util.io.FileUtil;
import org.jabref.model.database.BibDatabaseContext;
import org.jabref.model.entry.BibEntry;
import org.jabref.model.entry.LinkedFile;
import org.jabref.model.entry.field.StandardField;
import org.jabref.preferences.PreferencesService;

import org.jspecify.annotations.NonNull;
import org.jspecify.annotations.Nullable;

/**
* SIDE EFFECT: Sets the "cites" field of the entry having the linked files
*
* Mode choice A: online or offline
* Mode choice B: complete entry or single file (the latter is not implemented)
*
* The different modes should be implemented as sub classes. However, this was too complicated, thus we use variables at the constructor to parameterize this class.
*/
public class ExtractReferencesAction extends SimpleCommand {
private final int FILES_LIMIT = 10;

private final boolean online;
private final DialogService dialogService;
private final StateManager stateManager;
private final PreferencesService preferencesService;
private final BibEntry entry;
private final LinkedFile linkedFile;
private final TaskExecutor taskExecutor;

public ExtractReferencesAction(DialogService dialogService,
private final BibliographyFromPdfImporter bibliographyFromPdfImporter;

public ExtractReferencesAction(boolean online,
DialogService dialogService,
StateManager stateManager,
PreferencesService preferencesService,
TaskExecutor taskExecutor) {
this(dialogService, stateManager, preferencesService, null, null, taskExecutor);
this(online, dialogService, stateManager, preferencesService, null, null, taskExecutor);
}

public ExtractReferencesAction(DialogService dialogService,
StateManager stateManager,
PreferencesService preferencesService,
BibEntry entry,
LinkedFile linkedFile,
TaskExecutor taskExecutor) {
/**
* Can be used to bind the action on a context menu in the linked file view (future work)
*
* @param entry the entry to handle (can be null)
* @param linkedFile the linked file (can be null)
*/
private ExtractReferencesAction(boolean online,
@NonNull DialogService dialogService,
@NonNull StateManager stateManager,
@NonNull PreferencesService preferencesService,
@Nullable BibEntry entry,
@Nullable LinkedFile linkedFile,
@NonNull TaskExecutor taskExecutor) {
this.online = online;
this.dialogService = dialogService;
this.stateManager = stateManager;
this.preferencesService = preferencesService;
this.entry = entry;
this.linkedFile = linkedFile;
this.taskExecutor = taskExecutor;
bibliographyFromPdfImporter = new BibliographyFromPdfImporter(preferencesService.getCitationKeyPatternPreferences());

String text;
GrobidPreferences grobidPreferences = preferencesService.getGrobidPreferences();

if (this.linkedFile == null) {
this.executable.bind(
ActionHelper.needsEntriesSelected(stateManager)
.and(ActionHelper.hasLinkedFileForSelectedEntries(stateManager))
.and(this.preferencesService.getGrobidPreferences().grobidEnabledProperty())
);
} else {
this.setExecutable(true);
Expand All @@ -68,34 +102,113 @@ public void execute() {

private void extractReferences() {
stateManager.getActiveDatabase().ifPresent(databaseContext -> {
List<BibEntry> selectedEntries = new LinkedList<>();
assert online == this.preferencesService.getGrobidPreferences().isGrobidEnabled();

List<BibEntry> selectedEntries;
if (entry == null) {
selectedEntries = stateManager.getSelectedEntries();
} else {
selectedEntries.add(entry);
selectedEntries = List.of(entry);
}

List<Path> fileList = FileUtil.getListOfLinkedFiles(selectedEntries, databaseContext.getFileDirectories(preferencesService.getFilePreferences()));
if (fileList.size() > FILES_LIMIT) {
boolean continueOpening = dialogService.showConfirmationDialogAndWait(Localization.lang("Processing a large number of files"),
Localization.lang("You are about to process %0 files. Continue?", fileList.size()),
Localization.lang("Continue"), Localization.lang("Cancel"));
if (!continueOpening) {
Callable<ParserResult> parserResultCallable;
if (online) {
parserResultCallable = getParserResultCallableOnline(databaseContext, selectedEntries);
if (parserResultCallable == null) {
return;
}
} else {
parserResultCallable = getParserResultCallableOffline(databaseContext, selectedEntries);
}

Callable<ParserResult> parserResultCallable = () -> new ParserResult(
new GrobidService(this.preferencesService.getGrobidPreferences()).processReferences(fileList, preferencesService.getImportFormatPreferences())
);
BackgroundTask<ParserResult> task = BackgroundTask.wrap(parserResultCallable)
.withInitialMessage(Localization.lang("Processing PDF(s)"));

task.onFailure(dialogService::showErrorDialogAndWait);

ImportEntriesDialog dialog = new ImportEntriesDialog(stateManager.getActiveDatabase().get(), task);
dialog.setTitle(Localization.lang("Extract References"));
String title;
if (online) {
title = Localization.lang("Extract References (online)");
} else {
title = Localization.lang("Extract References (offline)");
}
dialog.setTitle(title);
dialogService.showCustomDialogAndWait(dialog);
});
}

private @NonNull Callable<ParserResult> getParserResultCallableOffline(BibDatabaseContext databaseContext, List<BibEntry> selectedEntries) {
return () -> {
BibEntry currentEntry = selectedEntries.getFirst();
List<Path> fileList = FileUtil.getListOfLinkedFiles(selectedEntries, databaseContext.getFileDirectories(preferencesService.getFilePreferences()));

// We need to have ParserResult handled at the importer, because it imports the meta data (library type, encoding, ...)
ParserResult result = bibliographyFromPdfImporter.importDatabase(fileList.getFirst());

// subsequent files are just appended to result
Iterator<Path> fileListIterator = fileList.iterator();
fileListIterator.next(); // skip first file
extractReferences(fileListIterator, result, currentEntry);

// handle subsequent entries
Iterator<BibEntry> selectedEntriesIterator = selectedEntries.iterator();
selectedEntriesIterator.next(); // skip first entry
while (selectedEntriesIterator.hasNext()) {
currentEntry = selectedEntriesIterator.next();
fileList = FileUtil.getListOfLinkedFiles(List.of(currentEntry), databaseContext.getFileDirectories(preferencesService.getFilePreferences()));
fileListIterator = fileList.iterator();
extractReferences(fileListIterator, result, currentEntry);
}

return result;
};
}

private void extractReferences(Iterator<Path> fileListIterator, ParserResult result, BibEntry currentEntry) {
while (fileListIterator.hasNext()) {
result.getDatabase().insertEntries(bibliographyFromPdfImporter.importDatabase(fileListIterator.next()).getDatabase().getEntries());
}

StringJoiner cites = new StringJoiner(",");
int count = 0;
for (BibEntry importedEntry : result.getDatabase().getEntries()) {
count++;
Optional<String> citationKey = importedEntry.getCitationKey();
if (citationKey.isPresent()) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

citationKey.map(cites:add).orElseGet( () ->

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure if new code is more readable --> "orElseGet" result needs to be added to the list, too. Uses outer variable "count", which is non final. I needed to wrap in anonymous object.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

then better use the original code

cites.add(citationKey.get());
} else {
String sourceCitationKey = currentEntry.getCitationKey().orElse("unknown");
String newCitationKey;
// Could happen if no author and no year is present
// We use the number of the comment field (because there is no other way to get the number reliable)
Pattern pattern = Pattern.compile("^\\[(\\d+)\\]");
Matcher matcher = pattern.matcher(importedEntry.getField(StandardField.COMMENT).orElse(""));
if (matcher.hasMatch()) {
newCitationKey = sourceCitationKey + "-" + matcher.group(1);
} else {
newCitationKey = sourceCitationKey + "-" + count;
}
importedEntry.setCitationKey(newCitationKey);
cites.add(newCitationKey);
}
}
currentEntry.setField(StandardField.CITES, cites.toString());
}

private @Nullable Callable<ParserResult> getParserResultCallableOnline(BibDatabaseContext databaseContext, List<BibEntry> selectedEntries) {
koppor marked this conversation as resolved.
Show resolved Hide resolved
Callable<ParserResult> parserResultCallable;
List<Path> fileList = FileUtil.getListOfLinkedFiles(selectedEntries, databaseContext.getFileDirectories(preferencesService.getFilePreferences()));
if (fileList.size() > FILES_LIMIT) {
boolean continueOpening = dialogService.showConfirmationDialogAndWait(Localization.lang("Processing a large number of files"),
Localization.lang("You are about to process %0 files. Continue?", fileList.size()),
Localization.lang("Continue"), Localization.lang("Cancel"));
if (!continueOpening) {
return null;
}
}
parserResultCallable = () -> new ParserResult(
new GrobidService(this.preferencesService.getGrobidPreferences()).processReferences(fileList, preferencesService.getImportFormatPreferences())
);
return parserResultCallable;
}
}
14 changes: 13 additions & 1 deletion src/main/java/org/jabref/gui/maintable/RightClickMenu.java
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

import javafx.scene.control.ContextMenu;
import javafx.scene.control.Menu;
import javafx.scene.control.MenuItem;
import javafx.scene.control.SeparatorMenuItem;

import org.jabref.gui.ClipBoardManager;
Expand Down Expand Up @@ -34,6 +35,8 @@
import org.jabref.preferences.PreferencesService;
import org.jabref.preferences.PreviewPreferences;

import com.tobiasdiez.easybind.EasyBind;

public class RightClickMenu {

public static ContextMenu create(BibEntryTableViewModel entry,
Expand All @@ -50,6 +53,9 @@ public static ContextMenu create(BibEntryTableViewModel entry,
ActionFactory factory = new ActionFactory(keyBindingRepository);
ContextMenu contextMenu = new ContextMenu();

MenuItem extractFileReferencesOnline = factory.createMenuItem(StandardActions.EXTRACT_FILE_REFERENCES_ONLINE, new ExtractReferencesAction(true, dialogService, stateManager, preferencesService, taskExecutor));
MenuItem extractFileReferencesOffline = factory.createMenuItem(StandardActions.EXTRACT_FILE_REFERENCES_OFFLINE, new ExtractReferencesAction(false, dialogService, stateManager, preferencesService, taskExecutor));

contextMenu.getItems().addAll(
factory.createMenuItem(StandardActions.COPY, new EditAction(StandardActions.COPY, () -> libraryTab, stateManager, undoManager)),
createCopySubMenu(factory, dialogService, stateManager, preferencesService, clipBoardManager, abbreviationRepository, taskExecutor),
Expand All @@ -75,7 +81,8 @@ public static ContextMenu create(BibEntryTableViewModel entry,
factory.createMenuItem(StandardActions.ATTACH_FILE_FROM_URL, new AttachFileFromURLAction(dialogService, stateManager, taskExecutor, preferencesService)),
factory.createMenuItem(StandardActions.OPEN_FOLDER, new OpenFolderAction(dialogService, stateManager, preferencesService, taskExecutor)),
factory.createMenuItem(StandardActions.OPEN_EXTERNAL_FILE, new OpenExternalFileAction(dialogService, stateManager, preferencesService, taskExecutor)),
factory.createMenuItem(StandardActions.EXTRACT_FILE_REFERENCES, new ExtractReferencesAction(dialogService, stateManager, preferencesService, taskExecutor)),
extractFileReferencesOnline,
extractFileReferencesOffline,

factory.createMenuItem(StandardActions.OPEN_URL, new OpenUrlAction(dialogService, stateManager, preferencesService)),
factory.createMenuItem(StandardActions.SEARCH_SHORTSCIENCE, new SearchShortScienceAction(dialogService, stateManager, preferencesService)),
Expand All @@ -86,6 +93,11 @@ public static ContextMenu create(BibEntryTableViewModel entry,
factory.createMenuItem(StandardActions.MERGE_WITH_FETCHED_ENTRY, new MergeWithFetchedEntryAction(dialogService, stateManager, taskExecutor, preferencesService, undoManager))
);

EasyBind.subscribe(preferencesService.getGrobidPreferences().grobidEnabledProperty(), enabled -> {
extractFileReferencesOnline.setVisible(enabled);
extractFileReferencesOffline.setVisible(!enabled);
});

return contextMenu;
}

Expand Down
11 changes: 0 additions & 11 deletions src/main/java/org/jabref/gui/menus/ChangeEntryTypeAction.java
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@

import javax.swing.undo.UndoManager;

import javafx.beans.property.ReadOnlyStringProperty;
import javafx.beans.property.ReadOnlyStringWrapper;

import org.jabref.gui.EntryTypeView;
Expand Down Expand Up @@ -36,14 +35,4 @@ public void execute() {
.ifPresent(change -> compound.addEdit(new UndoableChangeType(change))));
undoManager.addEdit(compound);
}

@Override
public String getStatusMessage() {
return statusMessage.get();
}

@Override
public ReadOnlyStringProperty statusMessageProperty() {
return statusMessageProperty.getReadOnlyProperty();
}
}
Loading