From 32c7b34d4c650faf874b5df4cab4bd5e91d78242 Mon Sep 17 00:00:00 2001 From: Dominik Voigt Date: Tue, 24 Nov 2020 21:33:49 +0100 Subject: [PATCH 01/24] Enable automated cross library search using a cross library query language. Signed-off-by: Dominik Voigt --- CHANGELOG.md | 1 + build.gradle | 2 + src/main/java/module-info.java | 1 + src/main/java/org/jabref/gui/JabRefFrame.java | 4 +- .../gui/StartLiteratureReviewAction.java | 72 ++++ .../jabref/gui/actions/StandardActions.java | 1 + .../org/jabref/logic/crawler/Crawler.java | 52 +++ .../LibraryEntryToFetcherConverter.java | 65 ++++ .../jabref/logic/crawler/StudyFetcher.java | 80 ++++ .../jabref/logic/crawler/StudyRepository.java | 344 ++++++++++++++++++ .../jabref/logic/crawler/git/GitHandler.java | 83 +++++ .../importer/fetcher/SpringerFetcher.java | 2 +- .../model/entry/types/EntryTypeFactory.java | 1 + ...tematicLiteratureReviewStudyEntryType.java | 33 ++ ...ratureReviewStudyEntryTypeDefinitions.java | 60 +++ .../org/jabref/model/study/FetchResult.java | 24 ++ .../org/jabref/model/study/QueryResult.java | 24 ++ .../java/org/jabref/model/study/Study.java | 98 +++++ .../model/study/StudyMetaDataField.java | 24 ++ src/main/resources/l10n/JabRef_en.properties | 19 +- .../org/jabref/logic/crawler/CrawlerTest.java | 95 +++++ .../LibraryEntryToFetcherConverterTest.java | 61 ++++ .../logic/crawler/StudyRepositoryTest.java | 287 +++++++++++++++ .../SearchBasedFetcherCapabilityTest.java | 2 +- .../org/jabref/model/study/StudyTest.java | 93 +++++ .../jabref/logic/crawler/ArXivQuantumMock.bib | 15 + .../crawler/SpringerCloud ComputingMock.bib | 9 + .../logic/crawler/SpringerQuantumMock.bib | 9 + .../org/jabref/logic/crawler/study.bib | 37 ++ 29 files changed, 1579 insertions(+), 19 deletions(-) create mode 100644 src/main/java/org/jabref/gui/StartLiteratureReviewAction.java create mode 100644 src/main/java/org/jabref/logic/crawler/Crawler.java create mode 100644 src/main/java/org/jabref/logic/crawler/LibraryEntryToFetcherConverter.java create mode 100644 src/main/java/org/jabref/logic/crawler/StudyFetcher.java create mode 100644 src/main/java/org/jabref/logic/crawler/StudyRepository.java create mode 100644 src/main/java/org/jabref/logic/crawler/git/GitHandler.java create mode 100644 src/main/java/org/jabref/model/entry/types/SystematicLiteratureReviewStudyEntryType.java create mode 100644 src/main/java/org/jabref/model/entry/types/SystematicLiteratureReviewStudyEntryTypeDefinitions.java create mode 100644 src/main/java/org/jabref/model/study/FetchResult.java create mode 100644 src/main/java/org/jabref/model/study/QueryResult.java create mode 100644 src/main/java/org/jabref/model/study/Study.java create mode 100644 src/main/java/org/jabref/model/study/StudyMetaDataField.java create mode 100644 src/test/java/org/jabref/logic/crawler/CrawlerTest.java create mode 100644 src/test/java/org/jabref/logic/crawler/LibraryEntryToFetcherConverterTest.java create mode 100644 src/test/java/org/jabref/logic/crawler/StudyRepositoryTest.java create mode 100644 src/test/java/org/jabref/model/study/StudyTest.java create mode 100644 src/test/resources/org/jabref/logic/crawler/ArXivQuantumMock.bib create mode 100644 src/test/resources/org/jabref/logic/crawler/SpringerCloud ComputingMock.bib create mode 100644 src/test/resources/org/jabref/logic/crawler/SpringerQuantumMock.bib create mode 100644 src/test/resources/org/jabref/logic/crawler/study.bib diff --git a/CHANGELOG.md b/CHANGELOG.md index 38c9e5bd2dc..bfcd7cb3ce1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ to the page field for cases where the page numbers are missing. [#7019](https:// - We added a new formatter to output shorthand month format. [#6579](https://github.com/JabRef/jabref/issues/6579) - We added support for the new Microsoft Edge browser in all platforms. [#7056](https://github.com/JabRef/jabref/pull/7056) - We reintroduced emacs/bash-like keybindings. [#6017](https://github.com/JabRef/jabref/issues/6017) +- We added a feature to provide automated cross library search using a cross library query language. This provides support for the search step of systematic literature reviews (SLRs). [koppor#369](https://github.com/koppor/jabref/issues/369) ### Changed diff --git a/build.gradle b/build.gradle index 6e04f5b0b22..6e2326393b8 100644 --- a/build.gradle +++ b/build.gradle @@ -139,6 +139,8 @@ dependencies { exclude group: 'org.apache.lucene', module: 'lucene-sandbox' } + implementation group: 'org.eclipse.jgit', name: 'org.eclipse.jgit', version: '5.9.0.202009080501-r' + implementation group: 'org.mariadb.jdbc', name: 'mariadb-java-client', version: '2.7.0' implementation 'org.postgresql:postgresql:42.2.18' diff --git a/src/main/java/module-info.java b/src/main/java/module-info.java index eb7e0102e92..c080fccc99e 100644 --- a/src/main/java/module-info.java +++ b/src/main/java/module-info.java @@ -91,4 +91,5 @@ requires com.h2database.mvstore; requires lucene.queryparser; requires lucene.core; + requires org.eclipse.jgit; } diff --git a/src/main/java/org/jabref/gui/JabRefFrame.java b/src/main/java/org/jabref/gui/JabRefFrame.java index 963df97bb57..1b48ad339fc 100644 --- a/src/main/java/org/jabref/gui/JabRefFrame.java +++ b/src/main/java/org/jabref/gui/JabRefFrame.java @@ -815,7 +815,9 @@ private MenuBar createMenu() { new SeparatorMenuItem(), factory.createMenuItem(StandardActions.SEND_AS_EMAIL, new SendAsEMailAction(dialogService, stateManager)), - pushToApplicationMenuItem + pushToApplicationMenuItem, + new SeparatorMenuItem(), + factory.createMenuItem(StandardActions.START_SYSTEMATIC_LITERATURE_REVIEW, new StartLiteratureReviewAction(this)) ); SidePaneComponent webSearch = sidePaneManager.getComponent(SidePaneType.WEB_SEARCH); diff --git a/src/main/java/org/jabref/gui/StartLiteratureReviewAction.java b/src/main/java/org/jabref/gui/StartLiteratureReviewAction.java new file mode 100644 index 00000000000..a641bd1e514 --- /dev/null +++ b/src/main/java/org/jabref/gui/StartLiteratureReviewAction.java @@ -0,0 +1,72 @@ +package org.jabref.gui; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.Optional; + +import org.jabref.gui.actions.SimpleCommand; +import org.jabref.gui.importer.actions.OpenDatabaseAction; +import org.jabref.gui.util.BackgroundTask; +import org.jabref.gui.util.FileDialogConfiguration; +import org.jabref.logic.crawler.Crawler; +import org.jabref.logic.importer.ParseException; +import org.jabref.logic.l10n.Localization; +import org.jabref.preferences.JabRefPreferences; + +import org.eclipse.jgit.api.errors.GitAPIException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class StartLiteratureReviewAction extends SimpleCommand { + private static final Logger LOGGER = LoggerFactory.getLogger(StartLiteratureReviewAction.class); + private final JabRefFrame frame; + private final DialogService dialogService; + + public StartLiteratureReviewAction(JabRefFrame frame) { + this.frame = frame; + this.dialogService = frame.getDialogService(); + } + + @Override + public void execute() { + FileDialogConfiguration fileDialogConfiguration = new FileDialogConfiguration.Builder() + .withInitialDirectory(getInitialDirectory()) + .build(); + + Optional studyDefinitionFile = dialogService.showFileOpenDialog(fileDialogConfiguration); + if (studyDefinitionFile.isEmpty()) { + // Do nothing if selection was canceled + return; + } + final Crawler crawler; + try { + crawler = new Crawler(studyDefinitionFile.get(), Globals.getFileUpdateMonitor(), JabRefPreferences.getInstance().getSavePreferences(), Globals.entryTypesManager); + } catch (IOException | ParseException | GitAPIException e) { + LOGGER.info("Error during reading of study definition file.", e); + dialogService.showErrorDialogAndWait(Localization.lang("Error during reading of study definition file.")); + return; + } + BackgroundTask.wrap(() -> { + crawler.performCrawl(); + return 0; // Return any value to make this a callable instead of a runnable. This allows throwing exceptions. + }) + .onFailure(e -> { + LOGGER.info("Error during persistence of crawling results."); + dialogService.showErrorDialogAndWait(Localization.lang("Error during persistence of crawling results."), e); + }) + .onSuccess(unused -> new OpenDatabaseAction(frame).openFile(Path.of(studyDefinitionFile.get().getParent().toString(), "studyResult.bib"), true)) + .executeWith(Globals.TASK_EXECUTOR); + } + + /** + * @return Path of current panel database directory or the working directory + */ + private Path getInitialDirectory() { + if (frame.getBasePanelCount() == 0) { + return Globals.prefs.getWorkingDir(); + } else { + Optional databasePath = frame.getCurrentLibraryTab().getBibDatabaseContext().getDatabasePath(); + return databasePath.map(Path::getParent).orElse(Globals.prefs.getWorkingDir()); + } + } +} diff --git a/src/main/java/org/jabref/gui/actions/StandardActions.java b/src/main/java/org/jabref/gui/actions/StandardActions.java index 2b316dca80a..0767d4021bb 100644 --- a/src/main/java/org/jabref/gui/actions/StandardActions.java +++ b/src/main/java/org/jabref/gui/actions/StandardActions.java @@ -88,6 +88,7 @@ public enum StandardActions implements Action { PARSE_LATEX(Localization.lang("Search for citations in LaTeX files..."), IconTheme.JabRefIcons.LATEX_CITATIONS), NEW_SUB_LIBRARY_FROM_AUX(Localization.lang("New sublibrary based on AUX file") + "...", Localization.lang("New BibTeX sublibrary") + Localization.lang("This feature generates a new library based on which entries are needed in an existing LaTeX document."), IconTheme.JabRefIcons.NEW), WRITE_XMP(Localization.lang("Write XMP metadata to PDFs"), Localization.lang("Will write XMP metadata to the PDFs linked from selected entries."), KeyBinding.WRITE_XMP), + START_SYSTEMATIC_LITERATURE_REVIEW(Localization.lang("Start systematic literature review")), OPEN_DATABASE_FOLDER(Localization.lang("Reveal in file explorer")), OPEN_FOLDER(Localization.lang("Open folder"), Localization.lang("Open folder"), KeyBinding.OPEN_FOLDER), OPEN_FILE(Localization.lang("Open file"), Localization.lang("Open file"), IconTheme.JabRefIcons.FILE, KeyBinding.OPEN_FILE), diff --git a/src/main/java/org/jabref/logic/crawler/Crawler.java b/src/main/java/org/jabref/logic/crawler/Crawler.java new file mode 100644 index 00000000000..f3114100e87 --- /dev/null +++ b/src/main/java/org/jabref/logic/crawler/Crawler.java @@ -0,0 +1,52 @@ +package org.jabref.logic.crawler; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; + +import org.jabref.logic.crawler.git.GitHandler; +import org.jabref.logic.exporter.SavePreferences; +import org.jabref.logic.importer.ParseException; +import org.jabref.model.entry.BibEntryTypesManager; +import org.jabref.model.study.QueryResult; +import org.jabref.model.study.Study; +import org.jabref.model.util.FileUpdateMonitor; +import org.jabref.preferences.JabRefPreferences; + +import org.eclipse.jgit.api.errors.GitAPIException; + +/** + * This class provides a service for SLR support by conducting an automated search and persistance + * of studies using the queries and E-Libraries specified in the provided study definition file. + * + * It composes a StudyRepository for repository management, + * and a StudyFetcher that manages the crawling over the selected E-Libraries. + */ +public class Crawler { + private final StudyRepository studyRepository; + private final StudyFetcher studyFetcher; + + /** + * Creates a crawler for retrieving studies from E-Libraries + * + * @param studyDefinitionFile The path to the study definition file that contains the list of targeted E-Libraries and used cross-library queries + */ + public Crawler(Path studyDefinitionFile, FileUpdateMonitor fileUpdateMonitor, SavePreferences savePreferences, BibEntryTypesManager bibEntryTypesManager) throws IllegalArgumentException, IOException, ParseException, GitAPIException { + Path studyRepositoryRoot = studyDefinitionFile.getParent(); + studyRepository = new StudyRepository(studyRepositoryRoot, new GitHandler(studyRepositoryRoot), JabRefPreferences.getInstance().getImportFormatPreferences(), fileUpdateMonitor, savePreferences, bibEntryTypesManager); + Study study = studyRepository.getStudy(); + LibraryEntryToFetcherConverter libraryEntryToFetcherConverter = new LibraryEntryToFetcherConverter(study.getActiveLibraryEntries(), JabRefPreferences.getInstance().getImportFormatPreferences()); + this.studyFetcher = new StudyFetcher(libraryEntryToFetcherConverter.getActiveFetchers(), study.getSearchQueryStrings()); + } + + /** + * This methods performs the crawling of the active libraries defined in the study definition file. + * This method also persists the results in the same folder the study definition file is stored in. + * + * @throws IOException Thrown if a problem occurred during the persistence of the result. + */ + public void performCrawl() throws IOException, GitAPIException { + List results = studyFetcher.crawl(); + studyRepository.persist(results); + } +} diff --git a/src/main/java/org/jabref/logic/crawler/LibraryEntryToFetcherConverter.java b/src/main/java/org/jabref/logic/crawler/LibraryEntryToFetcherConverter.java new file mode 100644 index 00000000000..ee2f0f00c71 --- /dev/null +++ b/src/main/java/org/jabref/logic/crawler/LibraryEntryToFetcherConverter.java @@ -0,0 +1,65 @@ +package org.jabref.logic.crawler; + +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.stream.Collectors; + +import org.jabref.logic.importer.ImportFormatPreferences; +import org.jabref.logic.importer.SearchBasedFetcher; +import org.jabref.logic.importer.WebFetchers; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.field.UnknownField; + +/** + * Converts library entries from the given study into their corresponding fetchers. + */ +class LibraryEntryToFetcherConverter { + private final List libraryEntries; + private final ImportFormatPreferences importFormatPreferences; + + public LibraryEntryToFetcherConverter(List libraryEntries, ImportFormatPreferences importFormatPreferences) { + this.libraryEntries = libraryEntries; + this.importFormatPreferences = importFormatPreferences; + } + + /** + * Returns a list of instances of all active library fetchers. + * + * A fetcher is considered active if there exists an library entry of the library the fetcher is associated with that is enabled. + * + * @return Instances of all active fetchers defined in the study definition. + */ + public List getActiveFetchers() { + return getFetchersFromLibraryEntries(this.libraryEntries); + } + + /** + * Transforms a list of libraryEntries into a list of SearchBasedFetcher instances. + * + * @param libraryEntries List of entries + * @return List of fetcher instances + */ + private List getFetchersFromLibraryEntries(List libraryEntries) { + return libraryEntries.parallelStream() + .filter(bibEntry -> bibEntry.getType().getName().equals("library")) + .map(this::createFetcherFromLibraryEntry) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + + /** + * Transforms a library entry into a SearchBasedFetcher instance. This only works if the library entry specifies a supported fetcher. + * + * @param libraryEntry the entry that will be converted + * @return An instance of the fetcher defined by the library entry. + */ + private SearchBasedFetcher createFetcherFromLibraryEntry(BibEntry libraryEntry) { + Set searchBasedFetchers = WebFetchers.getSearchBasedFetchers(importFormatPreferences); + String libraryNameFromFetcher = libraryEntry.getField(new UnknownField("name")).orElse(""); + return searchBasedFetchers.stream() + .filter(searchBasedFetcher -> searchBasedFetcher.getName().toLowerCase().equals(libraryNameFromFetcher.toLowerCase())) + .findAny() + .orElse(null); + } +} diff --git a/src/main/java/org/jabref/logic/crawler/StudyFetcher.java b/src/main/java/org/jabref/logic/crawler/StudyFetcher.java new file mode 100644 index 00000000000..c39ba7efe52 --- /dev/null +++ b/src/main/java/org/jabref/logic/crawler/StudyFetcher.java @@ -0,0 +1,80 @@ +package org.jabref.logic.crawler; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +import org.jabref.logic.importer.FetcherException; +import org.jabref.logic.importer.PagedSearchBasedFetcher; +import org.jabref.logic.importer.SearchBasedFetcher; +import org.jabref.model.database.BibDatabase; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.study.FetchResult; +import org.jabref.model.study.QueryResult; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Delegates the search of the provided set of targeted E-Libraries with the provided queries to the E-Library specific fetchers, + * and aggregates the results returned by the fetchers by query and E-Library. + */ +class StudyFetcher { + private static final Logger LOGGER = LoggerFactory.getLogger(StudyFetcher.class); + private static final int MAX_AMOUNT_OF_RESULTS_PER_FETCHER = 100; + + private final List activeFetchers; + private final List searchQueries; + + StudyFetcher(List activeFetchers, List searchQueries) throws IllegalArgumentException { + this.searchQueries = searchQueries; + this.activeFetchers = activeFetchers; + } + + /** + * Each Map Entry contains the results for one search term for all libraries. + * Each entry of the internal map contains the results for a given library. + * If any library API is not available, its corresponding entry is missing from the internal map. + */ + public List crawl() { + return searchQueries.parallelStream() + .map(this::getQueryResult) + .collect(Collectors.toList()); + } + + private QueryResult getQueryResult(String searchQuery) { + return new QueryResult(searchQuery, performSearchOnQuery(searchQuery)); + } + + /** + * Queries all Databases on the given searchQuery. + * + * @param searchQuery The query the search is performed for. + * @return Mapping of each fetcher by name and all their retrieved publications as a BibDatabase + */ + private List performSearchOnQuery(String searchQuery) { + return activeFetchers.parallelStream() + .map(fetcher -> performSearchOnQueryForFetcher(searchQuery, fetcher)) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } + + private FetchResult performSearchOnQueryForFetcher(String searchQuery, SearchBasedFetcher fetcher) { + try { + List fetchResult = new ArrayList<>(); + if (fetcher instanceof PagedSearchBasedFetcher) { + int pages = ((int) Math.ceil(((double) MAX_AMOUNT_OF_RESULTS_PER_FETCHER) / ((PagedSearchBasedFetcher) fetcher).getPageSize())); + for (int page = 0; page < pages; page++) { + fetchResult.addAll(((PagedSearchBasedFetcher) fetcher).performSearchPaged(searchQuery, page).getContent()); + } + } else { + fetchResult = fetcher.performSearch(searchQuery); + } + return new FetchResult(fetcher.getName(), new BibDatabase(fetchResult)); + } catch (FetcherException e) { + LOGGER.warn(String.format("%s API request failed", fetcher.getName()), e); + return null; + } + } +} diff --git a/src/main/java/org/jabref/logic/crawler/StudyRepository.java b/src/main/java/org/jabref/logic/crawler/StudyRepository.java new file mode 100644 index 00000000000..6b16d1e07dc --- /dev/null +++ b/src/main/java/org/jabref/logic/crawler/StudyRepository.java @@ -0,0 +1,344 @@ +package org.jabref.logic.crawler; + +import java.io.FileWriter; +import java.io.IOException; +import java.io.InputStream; +import java.io.Writer; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import org.jabref.logic.citationkeypattern.CitationKeyGenerator; +import org.jabref.logic.crawler.git.GitHandler; +import org.jabref.logic.database.DatabaseMerger; +import org.jabref.logic.exporter.BibtexDatabaseWriter; +import org.jabref.logic.exporter.SavePreferences; +import org.jabref.logic.importer.ImportFormatPreferences; +import org.jabref.logic.importer.OpenDatabase; +import org.jabref.logic.importer.ParseException; +import org.jabref.logic.importer.SearchBasedFetcher; +import org.jabref.logic.importer.fileformat.BibtexParser; +import org.jabref.model.database.BibDatabase; +import org.jabref.model.database.BibDatabaseContext; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.BibEntryTypesManager; +import org.jabref.model.entry.field.UnknownField; +import org.jabref.model.entry.types.SystematicLiteratureReviewStudyEntryType; +import org.jabref.model.study.FetchResult; +import org.jabref.model.study.QueryResult; +import org.jabref.model.study.Study; +import org.jabref.model.util.FileUpdateMonitor; +import org.jabref.preferences.JabRefPreferences; + +import org.eclipse.jgit.api.errors.GitAPIException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This class manages all aspects of the study process related to the repository. + * + * It includes the parsing of the study definition file (study.bib) into a Study instance, + * the structured persistence of the crawling results for the study within the file based repository, + * as well as the sharing, and versioning of results using git. + */ +class StudyRepository { + // Tests work with study.bib + private static final String STUDY_DEFINITION_FILE_NAME = "study.bib"; + private static final Logger LOGGER = LoggerFactory.getLogger(StudyRepository.class); + + private final Path repositoryPath; + private final Path studyDefinitionBib; + private final GitHandler gitHandler; + private final Study study; + private final ImportFormatPreferences importFormatPreferences; + private final FileUpdateMonitor fileUpdateMonitor; + private final SavePreferences savePreferences; + private final BibEntryTypesManager bibEntryTypesManager; + + /** + * Creates a study repository. + * + * @param pathToRepository Where the repository root is located. + * @param gitHandler The git handler that managages any interaction with the remote repository + * @throws IllegalArgumentException If the repository root directory does not exist, or the root directory does not contain the study definition file. + * @throws IOException Thrown if the given repository does not exists, or the study definition file does not exist + * @throws ParseException Problem parsing the study definition file. + */ + public StudyRepository(Path pathToRepository, GitHandler gitHandler, ImportFormatPreferences importFormatPreferences, FileUpdateMonitor fileUpdateMonitor, SavePreferences savePreferences, BibEntryTypesManager bibEntryTypesManager) throws IOException, ParseException, GitAPIException { + this.repositoryPath = pathToRepository; + this.gitHandler = gitHandler; + try { + gitHandler.updateLocalRepository(); + } catch (GitAPIException e) { + LOGGER.info("Updating repository from remote failed"); + } + this.importFormatPreferences = importFormatPreferences; + this.fileUpdateMonitor = fileUpdateMonitor; + this.studyDefinitionBib = Path.of(repositoryPath.toString(), STUDY_DEFINITION_FILE_NAME); + this.savePreferences = savePreferences; + this.bibEntryTypesManager = bibEntryTypesManager; + + if (Files.notExists(repositoryPath)) { + throw new IOException("The given repository does not exists."); + } else if (Files.notExists(studyDefinitionBib)) { + throw new IOException("The study definition file does not exist in the given repository."); + } + study = parseStudyFile(); + this.setUpRepositoryStructure(); + } + + /** + * Returns entries stored in the repository for a certain query and fetcher + */ + public BibDatabaseContext getFetcherResultEntries(String query, String fetcherName) throws IOException { + return OpenDatabase.loadDatabase(getPathToFetcherResultFile(query, fetcherName), importFormatPreferences, fileUpdateMonitor).getDatabaseContext(); + } + + /** + * Returns the merged entries stored in the repository for a certain query + */ + public BibDatabaseContext getQueryResultEntries(String query) throws IOException { + return OpenDatabase.loadDatabase(getPathToQueryResultFile(query), importFormatPreferences, fileUpdateMonitor).getDatabaseContext(); + } + + /** + * Returns the merged entries stored in the repository for all queries + */ + public BibDatabaseContext getStudyResultEntries() throws IOException { + return OpenDatabase.loadDatabase(getPathToStudyResultFile(), importFormatPreferences, fileUpdateMonitor).getDatabaseContext(); + } + + /** + * The study definition file contains all the definitions of a study. This method extracts the BibEntries from the study BiB file. + * + * @return Returns the BibEntries parsed from the study definition file. + * @throws IOException Problem opening the input stream. + * @throws ParseException Problem parsing the study definition file. + */ + private Study parseStudyFile() throws IOException, ParseException { + BibtexParser parser = new BibtexParser(importFormatPreferences, fileUpdateMonitor); + List parsedEntries = new ArrayList<>(); + try (InputStream inputStream = Files.newInputStream(studyDefinitionBib)) { + parsedEntries.addAll(parser.parseEntries(inputStream)); + } + + BibEntry studyEntry = parsedEntries.parallelStream() + .filter(bibEntry -> bibEntry.getType().equals(SystematicLiteratureReviewStudyEntryType.STUDY_ENTRY)).findAny() + .orElseThrow(() -> new ParseException("Study definition file does not contain a study entry")); + List queryEntries = parsedEntries.parallelStream() + .filter(bibEntry -> bibEntry.getType().equals(SystematicLiteratureReviewStudyEntryType.SEARCH_QUERY_ENTRY)) + .collect(Collectors.toList()); + List libraryEntries = parsedEntries.parallelStream() + .filter(bibEntry -> bibEntry.getType().equals(SystematicLiteratureReviewStudyEntryType.LIBRARY_ENTRY)) + .collect(Collectors.toList()); + + return new Study(studyEntry, queryEntries, libraryEntries); + } + + public Study getStudy() { + return study; + } + + public void persist(List crawlResults) throws IOException, GitAPIException { + try { + gitHandler.updateLocalRepository(); + } catch (GitAPIException e) { + LOGGER.info("Updating repository from remote failed"); + } + persistResults(crawlResults); + study.setLastSearchDate(LocalDate.now()); + persistStudy(); + try { + gitHandler.updateRemoteRepository("Conducted search " + LocalDate.now()); + } catch (GitAPIException e) { + LOGGER.info("Updating remote repository failed"); + } + } + + private void persistStudy() throws IOException { + writeResultToFile(studyDefinitionBib, new BibDatabase(study.getAllEntries())); + } + + /** + * Create for each query a folder, and for each fetcher a bib file in the query folder to store its results. + */ + private void setUpRepositoryStructure() throws IOException { + // Cannot use stream here since IOException has to be thrown + LibraryEntryToFetcherConverter converter = new LibraryEntryToFetcherConverter(study.getActiveLibraryEntries(), importFormatPreferences); + for (String query : study.getSearchQueryStrings()) { + createQueryResultFolder(query); + converter.getActiveFetchers() + .forEach(searchBasedFetcher -> createFetcherResultFile(query, searchBasedFetcher)); + createQueryResultFile(query); + } + createStudyResultFile(); + } + + /** + * Creates a folder using the query and its corresponding query id. + * This folder name is unique for each query, as long as the query id in the study definition is unique for each query. + * + * @param query The query the folder is created for + */ + private void createQueryResultFolder(String query) throws IOException { + Path queryResultFolder = getPathToQueryDirectory(query); + createFolder(queryResultFolder); + } + + private void createFolder(Path folder) throws IOException { + if (Files.notExists(folder)) { + try { + Files.createDirectory(folder); + } catch (IOException e) { + throw new IOException("Error during creation of repository structure.", e); + } + } + } + + private void createFetcherResultFile(String query, SearchBasedFetcher searchBasedFetcher) { + String fetcherName = searchBasedFetcher.getName(); + Path fetcherResultFile = getPathToFetcherResultFile(query, fetcherName); + createBibFile(fetcherResultFile); + } + + private void createQueryResultFile(String query) { + Path queryResultFile = getPathToFetcherResultFile(query, "result"); + createBibFile(queryResultFile); + } + + private void createStudyResultFile() { + createBibFile(getPathToStudyResultFile()); + } + + private void createBibFile(Path file) { + if (Files.notExists(file)) { + try { + Files.createFile(file); + } catch (IOException e) { + throw new IllegalStateException("Error during creation of repository structure.", e); + } + } + } + + /** + * Returns a string that can be used as a folder name. + * This removes all characters from the query that are illegal for directory names. + * Structure: ID-trimmed query + * + * Examples: + * Input: '(title: test-title AND abstract: Test)' as a query entry with id 1 + * Output: '1 - title= test-title AND abstract= Test' + * + * Input: 'abstract: Test*' as a query entry with id 1 + * Output: '1 - abstract= Test' + * + * Input: '"test driven"' as a query entry with id 1 + * Output: '1 - test driven' + * + * @param query that is trimmed and combined with its query id + * @return a unique folder name for any query. + */ + private String trimNameAndAddID(String query) { + // Replace all field: with field= for folder name + String trimmedNamed = query.replaceAll(":", "="); + trimmedNamed = trimmedNamed.replaceAll("[^A-Za-z0-9_.\\s=-]", ""); + if (query.length() > 240) { + trimmedNamed = query.substring(0, 240); + } + String id = findQueryIDByQueryString(query); + return id + " - " + trimmedNamed; + } + + /** + * Helper to find the query id for folder name creation. + * Returns the id of the first SearchQuery BibEntry with a query field that matches the given query. + * + * @param query The query whose ID is searched + * @return ID of the query defined in the study definition. + */ + private String findQueryIDByQueryString(String query) { + return study.getSearchQueryEntries() + .parallelStream() + .filter(bibEntry -> bibEntry.getField(new UnknownField("query")).orElse("").equals(query)) + .map(BibEntry::getCitationKey) + .filter(Optional::isPresent) + .map(Optional::get) + .findFirst() + .orElseThrow() + .replaceFirst("query", ""); + } + + /** + * Persists the crawling results in the local file based repository. + * + * @param crawlResults The results that shall be persisted. + */ + private void persistResults(List crawlResults) throws IOException { + DatabaseMerger merger = new DatabaseMerger(); + BibDatabase newStudyResultEntries = new BibDatabase(); + + for (QueryResult result : crawlResults) { + BibDatabase queryResultEntries = new BibDatabase(); + for (FetchResult fetcherResult : result.getResultsPerFetcher()) { + BibDatabase fetcherEntries = fetcherResult.getFetchResult(); + BibDatabaseContext existingFetcherResult = getFetcherResultEntries(result.getQuery(), fetcherResult.getFetcherName()); + + // Create citation keys for all entries that do not have one + generateCiteKeys(existingFetcherResult, fetcherEntries); + + // Merge new entries into fetcher result file + merger.merge(existingFetcherResult.getDatabase(), fetcherEntries); + // Aggregate each fetcher result into the query result + merger.merge(queryResultEntries, fetcherEntries); + + writeResultToFile(getPathToFetcherResultFile(result.getQuery(), fetcherResult.getFetcherName()), existingFetcherResult.getDatabase()); + } + BibDatabase existingQueryEntries = getQueryResultEntries(result.getQuery()).getDatabase(); + + // Merge new entries into query result file + merger.merge(existingQueryEntries, queryResultEntries); + // Aggregate all new entries for every query into the study result + merger.merge(newStudyResultEntries, queryResultEntries); + + writeResultToFile(getPathToQueryResultFile(result.getQuery()), existingQueryEntries); + } + BibDatabase existingStudyResultEntries = getStudyResultEntries().getDatabase(); + + // Merge new entries into study result file + merger.merge(existingStudyResultEntries, newStudyResultEntries); + + writeResultToFile(getPathToStudyResultFile(), existingStudyResultEntries); + } + + private void generateCiteKeys(BibDatabaseContext existingEntries, BibDatabase targetEntries) { + CitationKeyGenerator citationKeyGenerator = new CitationKeyGenerator(existingEntries, JabRefPreferences.getInstance().getCitationKeyPatternPreferences()); + targetEntries.getEntries().stream().filter(bibEntry -> !bibEntry.hasCitationKey()).forEach(citationKeyGenerator::generateAndSetKey); + } + + private void writeResultToFile(Path pathToFile, BibDatabase entries) throws IOException { + try (Writer fileWriter = new FileWriter(pathToFile.toFile())) { + BibtexDatabaseWriter databaseWriter = new BibtexDatabaseWriter(fileWriter, savePreferences, bibEntryTypesManager); + databaseWriter.saveDatabase(new BibDatabaseContext(entries)); + } + } + + private Path getPathToFetcherResultFile(String query, String fetcherName) { + return Path.of(repositoryPath.toString(), trimNameAndAddID(query), fetcherName + ".bib"); + } + + private Path getPathToQueryResultFile(String query) { + return Path.of(repositoryPath.toString(), trimNameAndAddID(query), "result.bib"); + } + + private Path getPathToStudyResultFile() { + return Path.of(repositoryPath.toString(), "studyResult.bib"); + } + + private Path getPathToQueryDirectory(String query) { + return Path.of(repositoryPath.toString(), trimNameAndAddID(query)); + } +} diff --git a/src/main/java/org/jabref/logic/crawler/git/GitHandler.java b/src/main/java/org/jabref/logic/crawler/git/GitHandler.java new file mode 100644 index 00000000000..439f08dfccd --- /dev/null +++ b/src/main/java/org/jabref/logic/crawler/git/GitHandler.java @@ -0,0 +1,83 @@ +package org.jabref.logic.crawler.git; + +import java.io.IOException; +import java.nio.file.Path; + +import org.eclipse.jgit.api.Git; +import org.eclipse.jgit.api.RmCommand; +import org.eclipse.jgit.api.Status; +import org.eclipse.jgit.api.errors.GitAPIException; +import org.eclipse.jgit.transport.CredentialsProvider; +import org.eclipse.jgit.transport.UsernamePasswordCredentialsProvider; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This class handles the updating of the local and remote git repository that is located at the repository path + */ +public class GitHandler { + private static final Logger LOGGER = LoggerFactory.getLogger(GitHandler.class); + private final Path repositoryPath; + private final CredentialsProvider credentialsProvider = new UsernamePasswordCredentialsProvider(System.getenv("GIT_EMAIL"), System.getenv("GIT_PW")); + + /** + * Initialize the handler for the given repository + * + * @param repositoryPath The root of the intialized git repository + */ + public GitHandler(Path repositoryPath) { + this.repositoryPath = repositoryPath; + } + + /** + * Updates the local repository based on the main branch of the original remote repository + */ + public void updateLocalRepository() throws IOException, GitAPIException { + try (Git git = Git.open(this.repositoryPath.toFile())) { + git.pull() + .setRemote("origin") + .setRemoteBranchName("main") + .setCredentialsProvider(credentialsProvider) + .call(); + } + } + + /** + * Adds all the added, changed, and removed files to the index and updates the remote origin repository + * If pushiong to remote fails it fails silently + * + * @param commitMessage The commit message used for the commit to the remote repository + */ + public void updateRemoteRepository(String commitMessage) throws IOException, GitAPIException { + // First get up to date + this.updateLocalRepository(); + try (Git git = Git.open(this.repositoryPath.toFile())) { + Status status = git.status().call(); + if (!status.isClean()) { + // Add new and changed files to index + git.add() + .addFilepattern(".") + .call(); + // Add all removed files to index + if (!status.getMissing().isEmpty()) { + RmCommand removeCommand = git.rm() + .setCached(true); + status.getMissing().forEach(removeCommand::addFilepattern); + removeCommand.call(); + } + git.commit() + .setAllowEmpty(false) + .setMessage(commitMessage) + .call(); + try { + + git.push() + .setCredentialsProvider(credentialsProvider) + .call(); + } catch (GitAPIException e) { + LOGGER.info("Failed to push"); + } + } + } + } +} diff --git a/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java index a547dbe2175..1064a7f272e 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java @@ -178,7 +178,7 @@ private String constructComplexQueryString(ComplexSearchQuery complexSearchQuery complexSearchQuery.getTitlePhrases().forEach(title -> searchTerms.add("title:" + title)); complexSearchQuery.getJournal().ifPresent(journal -> searchTerms.add("journal:" + journal)); // Since Springer API does not support year range search, we ignore formYear and toYear and use "singleYear" only - complexSearchQuery.getSingleYear().ifPresent(year -> searchTerms.add("year:" + year.toString())); + complexSearchQuery.getSingleYear().ifPresent(year -> searchTerms.add("date:" + year.toString() + "*")); searchTerms.addAll(complexSearchQuery.getDefaultFieldPhrases()); return String.join(" AND ", searchTerms); } diff --git a/src/main/java/org/jabref/model/entry/types/EntryTypeFactory.java b/src/main/java/org/jabref/model/entry/types/EntryTypeFactory.java index 1ecf238382f..29422891f4e 100644 --- a/src/main/java/org/jabref/model/entry/types/EntryTypeFactory.java +++ b/src/main/java/org/jabref/model/entry/types/EntryTypeFactory.java @@ -50,6 +50,7 @@ public static EntryType parse(String typeName) { List types = new ArrayList<>(Arrays.asList(StandardEntryType.values())); types.addAll(Arrays.asList(IEEETranEntryType.values())); + types.addAll(Arrays.asList(SystematicLiteratureReviewStudyEntryType.values())); return types.stream().filter(type -> type.getName().equals(typeName.toLowerCase(Locale.ENGLISH))).findFirst().orElse(new UnknownEntryType(typeName)); } diff --git a/src/main/java/org/jabref/model/entry/types/SystematicLiteratureReviewStudyEntryType.java b/src/main/java/org/jabref/model/entry/types/SystematicLiteratureReviewStudyEntryType.java new file mode 100644 index 00000000000..1d9bd4be112 --- /dev/null +++ b/src/main/java/org/jabref/model/entry/types/SystematicLiteratureReviewStudyEntryType.java @@ -0,0 +1,33 @@ +package org.jabref.model.entry.types; + +import java.util.Arrays; +import java.util.Locale; +import java.util.Optional; + +public enum SystematicLiteratureReviewStudyEntryType implements EntryType { + STUDY_ENTRY("Study"), + SEARCH_QUERY_ENTRY("SearchQuery"), + LIBRARY_ENTRY("Library"); + + private final String displayName; + + SystematicLiteratureReviewStudyEntryType(String displayName) { + this.displayName = displayName; + } + + public static Optional fromName(String name) { + return Arrays.stream(SystematicLiteratureReviewStudyEntryType.values()) + .filter(field -> field.getName().equalsIgnoreCase(name)) + .findAny(); + } + + @Override + public String getName() { + return displayName.toLowerCase(Locale.ENGLISH); + } + + @Override + public String getDisplayName() { + return displayName; + } +} diff --git a/src/main/java/org/jabref/model/entry/types/SystematicLiteratureReviewStudyEntryTypeDefinitions.java b/src/main/java/org/jabref/model/entry/types/SystematicLiteratureReviewStudyEntryTypeDefinitions.java new file mode 100644 index 00000000000..e9f31b2e630 --- /dev/null +++ b/src/main/java/org/jabref/model/entry/types/SystematicLiteratureReviewStudyEntryTypeDefinitions.java @@ -0,0 +1,60 @@ +package org.jabref.model.entry.types; + +import java.util.Arrays; +import java.util.List; + +import org.jabref.model.entry.BibEntryType; +import org.jabref.model.entry.BibEntryTypeBuilder; +import org.jabref.model.entry.field.StandardField; +import org.jabref.model.entry.field.UnknownField; + +/** + * This class represents all supported entry types used in a study definition file + */ +public class SystematicLiteratureReviewStudyEntryTypeDefinitions { + + /** + * Entry type used for study meta data within a study definition file + * + *
    + *
  • Required fields: author, lastsearchdate, name, enabled
    • + *
  • Optional fields:
    • + *
+ */ + private static final BibEntryType STUDY_ENTRY = new BibEntryTypeBuilder() + .withType(SystematicLiteratureReviewStudyEntryType.STUDY_ENTRY) + .withRequiredFields(StandardField.AUTHOR, new UnknownField("lastsearchdate"), new UnknownField("name"), new UnknownField("researchquestions")) + .build(); + + /** + * Entry type for the queries within the study definition file + * + *
    + *
  • Required fields: query
    • + *
  • Optional fields:
    • + *
+ */ + private static final BibEntryType SEARCH_QUERY_ENTRY = new BibEntryTypeBuilder() + .withType(SystematicLiteratureReviewStudyEntryType.SEARCH_QUERY_ENTRY) + .withRequiredFields(new UnknownField("query")) + .build(); + + /** + * Entry type for the targeted libraries within a study definition file + * + *
    + *
  • Required fields: name, enabled
    • + *
  • Optional fields: comment
    • + *
+ */ + private static final BibEntryType LIBRARY_ENTRY = new BibEntryTypeBuilder() + .withType(SystematicLiteratureReviewStudyEntryType.STUDY_ENTRY) + .withRequiredFields(new UnknownField("name"), new UnknownField("enabled")) + .withImportantFields(new UnknownField("comment")) + .build(); + + public static final List ALL = Arrays.asList(STUDY_ENTRY, SEARCH_QUERY_ENTRY, LIBRARY_ENTRY); + + private SystematicLiteratureReviewStudyEntryTypeDefinitions() { + } +} diff --git a/src/main/java/org/jabref/model/study/FetchResult.java b/src/main/java/org/jabref/model/study/FetchResult.java new file mode 100644 index 00000000000..80637feb4ab --- /dev/null +++ b/src/main/java/org/jabref/model/study/FetchResult.java @@ -0,0 +1,24 @@ +package org.jabref.model.study; + +import org.jabref.model.database.BibDatabase; + +/** + * Represents the result of fetching the results for a query for a specific library + */ +public class FetchResult { + private final String fetcherName; + private final BibDatabase fetchResult; + + public FetchResult(String fetcherName, BibDatabase fetcherResult) { + this.fetcherName = fetcherName; + this.fetchResult = fetcherResult; + } + + public String getFetcherName() { + return fetcherName; + } + + public BibDatabase getFetchResult() { + return fetchResult; + } +} diff --git a/src/main/java/org/jabref/model/study/QueryResult.java b/src/main/java/org/jabref/model/study/QueryResult.java new file mode 100644 index 00000000000..2976b5224fe --- /dev/null +++ b/src/main/java/org/jabref/model/study/QueryResult.java @@ -0,0 +1,24 @@ +package org.jabref.model.study; + +import java.util.List; + +/** + * Represents the result of fetching the results from all active fetchers for a specific query. + */ +public class QueryResult { + private final String query; + private final List resultsPerLibrary; + + public QueryResult(String query, List resultsPerLibrary) { + this.query = query; + this.resultsPerLibrary = resultsPerLibrary; + } + + public String getQuery() { + return query; + } + + public List getResultsPerFetcher() { + return resultsPerLibrary; + } +} diff --git a/src/main/java/org/jabref/model/study/Study.java b/src/main/java/org/jabref/model/study/Study.java new file mode 100644 index 00000000000..37ed6e2328a --- /dev/null +++ b/src/main/java/org/jabref/model/study/Study.java @@ -0,0 +1,98 @@ +package org.jabref.model.study; + +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.stream.Collectors; + +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.field.UnknownField; + +/** + * This class represents a scientific study. + * + * This class defines all aspects of a scientific study relevant to the application. It is a proxy for the file based study definition. + */ +public class Study { + private static final String SEARCH_QUERY_FIELD_NAME = "query"; + + private final BibEntry studyEntry; + private final List queryEntries; + private final List libraryEntries; + + public Study(BibEntry studyEntry, List queryEntries, List libraryEntries) { + this.studyEntry = studyEntry; + this.queryEntries = queryEntries; + this.libraryEntries = libraryEntries; + } + + public List getAllEntries() { + List allEntries = new ArrayList<>(); + allEntries.add(studyEntry); + allEntries.addAll(queryEntries); + allEntries.addAll(libraryEntries); + return allEntries; + } + + /** + * Returns all query strings + * + * @return List of all queries as Strings. + */ + public List getSearchQueryStrings() { + return queryEntries.parallelStream() + .map(bibEntry -> bibEntry.getField(new UnknownField(SEARCH_QUERY_FIELD_NAME))) + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.toList()); + } + + /** + * This method returns the SearchQuery entries. + * This is required when the BibKey of the search term entry is required in combination with the search query (e.g. + * for the creation of the study repository structure). + */ + public List getSearchQueryEntries() { + return queryEntries; + } + + /** + * Returns a meta data entry of the first study entry found in the study definition file of the provided type. + * + * @param metaDataField The type of requested meta-data + * @return returns the requested meta data type of the first found study entry + * @throws IllegalArgumentException If the study file does not contain a study entry. + */ + public Optional getStudyMetaDataField(StudyMetaDataField metaDataField) throws IllegalArgumentException { + return studyEntry.getField(metaDataField.toField()); + } + + /** + * Sets the lastSearchDate field of the study entry + * + * @param date date the last time a search was conducted + */ + public void setLastSearchDate(LocalDate date) { + studyEntry.setField(StudyMetaDataField.STUDY_LAST_SEARCH.toField(), date.toString()); + } + + /** + * Extracts all active LibraryEntries from the BibEntries. + * + * @return List of BibEntries of type Library + * @throws IllegalArgumentException If a transformation from Library entry to LibraryDefinition fails + */ + public List getActiveLibraryEntries() throws IllegalArgumentException { + return libraryEntries + .parallelStream() + .filter(bibEntry -> { + // If enabled is not defined, the fetcher is active. + return bibEntry.getField(new UnknownField("enabled")) + .map(enabled -> enabled.equals("true")) + .orElse(true); + }) + .collect(Collectors.toList()); + } +} + diff --git a/src/main/java/org/jabref/model/study/StudyMetaDataField.java b/src/main/java/org/jabref/model/study/StudyMetaDataField.java new file mode 100644 index 00000000000..6dbea2a2dc8 --- /dev/null +++ b/src/main/java/org/jabref/model/study/StudyMetaDataField.java @@ -0,0 +1,24 @@ +package org.jabref.model.study; + +import org.jabref.model.entry.field.Field; +import org.jabref.model.entry.field.StandardField; +import org.jabref.model.entry.field.UnknownField; + +/** + * This enum represents the different fields in the study entry + */ +public enum StudyMetaDataField { + STUDY_NAME(new UnknownField("name")), STUDY_RESEARCH_QUESTIONS(new UnknownField("researchQuestions")), + STUDY_AUTHORS(StandardField.AUTHOR), STUDY_GIT_REPOSITORY(new UnknownField("gitRepositoryURL")), + STUDY_LAST_SEARCH(new UnknownField("lastSearchDate")); + + private final Field field; + + StudyMetaDataField(Field field) { + this.field = field; + } + + public Field toField() { + return this.field; + } +} diff --git a/src/main/resources/l10n/JabRef_en.properties b/src/main/resources/l10n/JabRef_en.properties index b58510e6d5c..d66d978e315 100644 --- a/src/main/resources/l10n/JabRef_en.properties +++ b/src/main/resources/l10n/JabRef_en.properties @@ -295,32 +295,21 @@ Entry\ owner=Entry owner Entry\ preview=Entry preview Entry\ table=Entry table - Entry\ table\ columns=Entry table columns Entry\ Title\ (Required\ to\ deliver\ recommendations.)=Entry Title (Required to deliver recommendations.) - Entry\ type=Entry type - Error=Error - Error\ occurred\ when\ parsing\ entry=Error occurred when parsing entry - Error\ opening\ file=Error opening file - Error\ while\ writing=Error while writing - +Error\ during\ persistence\ of\ crawling\ results.=Error during persistence of crawling results. +Error\ during\ reading\ of\ study\ definition\ file.=Error during reading of study definition file. '%0'\ exists.\ Overwrite\ file?='%0' exists. Overwrite file? - Export=Export - Export\ preferences=Export preferences - Export\ preferences\ to\ file=Export preferences to file - Export\ to\ clipboard=Export to clipboard - Export\ to\ text\ file.=Export to text file. - Exporting=Exporting Extension=Extension @@ -644,11 +633,9 @@ Previous\ preview\ layout=Previous preview layout Available=Available Selected=Selected Selected\ Layouts\ can\ not\ be\ empty=Selected Layouts can not be empty - +Start\ systematic\ literature\ review=Start systematic literature review Reset\ default\ preview\ style=Reset default preview style - Previous\ entry=Previous entry - Primary\ sort\ criterion=Primary sort criterion Problem\ with\ parsing\ entry=Problem with parsing entry Processing\ %0=Processing %0 diff --git a/src/test/java/org/jabref/logic/crawler/CrawlerTest.java b/src/test/java/org/jabref/logic/crawler/CrawlerTest.java new file mode 100644 index 00000000000..daeb90b4f07 --- /dev/null +++ b/src/test/java/org/jabref/logic/crawler/CrawlerTest.java @@ -0,0 +1,95 @@ +package org.jabref.logic.crawler; + +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.jabref.logic.exporter.SavePreferences; +import org.jabref.logic.util.io.FileUtil; +import org.jabref.model.entry.BibEntryTypesManager; +import org.jabref.model.metadata.SaveOrderConfig; +import org.jabref.model.util.DummyFileUpdateMonitor; + +import org.eclipse.jgit.api.Git; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Answers; + +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +/** + * Integration test of the components used for SLR support + */ +class CrawlerTest { + @TempDir + Path tempRepositoryDirectory; + SavePreferences preferences; + BibEntryTypesManager entryTypesManager; + + @Test + public void testWhetherAllFilesAreCreated() throws Exception { + setUp(); + Crawler testCrawler = new Crawler(getPathToStudyDefinitionFile(), + new DummyFileUpdateMonitor(), + preferences, + entryTypesManager + ); + + testCrawler.performCrawl(); + + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "1 - Quantum"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "2 - Cloud Computing"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "3 - TestSearchQuery3"))); + + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "1 - Quantum", "ArXiv.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "2 - Cloud Computing", "ArXiv.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "3 - TestSearchQuery3", "ArXiv.bib"))); + + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "1 - Quantum", "Springer.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "2 - Cloud Computing", "Springer.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "3 - TestSearchQuery3", "Springer.bib"))); + + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "1 - Quantum", "result.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "2 - Cloud Computing", "result.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "3 - TestSearchQuery3", "result.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "studyResult.bib"))); + } + + private Path getPathToStudyDefinitionFile() { + return tempRepositoryDirectory.resolve("study.bib"); + } + + /** + * Set up mocks and copies the study definition file into the test repository + */ + private void setUp() throws Exception { + setUpRepository(); + preferences = mock(SavePreferences.class, Answers.RETURNS_DEEP_STUBS); + when(preferences.getSaveOrder()).thenReturn(new SaveOrderConfig()); + when(preferences.getEncoding()).thenReturn(null); + when(preferences.takeMetadataSaveOrderInAccount()).thenReturn(true); + entryTypesManager = new BibEntryTypesManager(); + } + + private void setUpRepository() throws Exception { + Git git = Git.init() + .setDirectory(tempRepositoryDirectory.toFile()) + .call(); + setUpTestStudyDefinitionFile(); + git.add() + .addFilepattern(".") + .call(); + git.commit() + .setMessage("Initialize") + .call(); + git.close(); + } + + private void setUpTestStudyDefinitionFile() throws Exception { + Path destination = tempRepositoryDirectory.resolve("study.bib"); + URL studyDefinition = this.getClass().getResource("study.bib"); + FileUtil.copyFile(Path.of(studyDefinition.toURI()), destination, false); + } +} diff --git a/src/test/java/org/jabref/logic/crawler/LibraryEntryToFetcherConverterTest.java b/src/test/java/org/jabref/logic/crawler/LibraryEntryToFetcherConverterTest.java new file mode 100644 index 00000000000..cce997fe564 --- /dev/null +++ b/src/test/java/org/jabref/logic/crawler/LibraryEntryToFetcherConverterTest.java @@ -0,0 +1,61 @@ +package org.jabref.logic.crawler; + +import java.net.URL; +import java.nio.file.Path; +import java.util.List; + +import org.jabref.logic.crawler.git.GitHandler; +import org.jabref.logic.exporter.SavePreferences; +import org.jabref.logic.importer.SearchBasedFetcher; +import org.jabref.logic.util.io.FileUtil; +import org.jabref.model.entry.BibEntryTypesManager; +import org.jabref.model.metadata.SaveOrderConfig; +import org.jabref.model.study.Study; +import org.jabref.model.util.DummyFileUpdateMonitor; +import org.jabref.preferences.JabRefPreferences; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Answers; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class LibraryEntryToFetcherConverterTest { + SavePreferences preferences; + BibEntryTypesManager entryTypesManager; + GitHandler gitHandler; + @TempDir + Path tempRepositoryDirectory; + + @BeforeEach + void setUpMocks() { + preferences = mock(SavePreferences.class, Answers.RETURNS_DEEP_STUBS); + when(preferences.getSaveOrder()).thenReturn(new SaveOrderConfig()); + when(preferences.getEncoding()).thenReturn(null); + when(preferences.takeMetadataSaveOrderInAccount()).thenReturn(true); + entryTypesManager = new BibEntryTypesManager(); + gitHandler = mock(GitHandler.class, Answers.RETURNS_DEFAULTS); + } + + @Test + public void getActiveFetcherInstances() throws Exception { + Path studyDefinition = tempRepositoryDirectory.resolve("study.bib"); + copyTestStudyDefinitionFileIntoDirectory(studyDefinition); + + Study study = new StudyRepository(tempRepositoryDirectory, gitHandler, JabRefPreferences.getInstance().getImportFormatPreferences(), new DummyFileUpdateMonitor(), preferences, entryTypesManager).getStudy(); + LibraryEntryToFetcherConverter converter = new LibraryEntryToFetcherConverter(study.getActiveLibraryEntries(), JabRefPreferences.getInstance().getImportFormatPreferences()); + List result = converter.getActiveFetchers(); + + Assertions.assertEquals(2, result.size()); + Assertions.assertEquals(result.get(0).getName(), "Springer"); + Assertions.assertEquals(result.get(1).getName(), "ArXiv"); + } + + private void copyTestStudyDefinitionFileIntoDirectory(Path destination) throws Exception { + URL studyDefinition = this.getClass().getResource("study.bib"); + FileUtil.copyFile(Path.of(studyDefinition.toURI()), destination, false); + } +} diff --git a/src/test/java/org/jabref/logic/crawler/StudyRepositoryTest.java b/src/test/java/org/jabref/logic/crawler/StudyRepositoryTest.java new file mode 100644 index 00000000000..72717b2a0a2 --- /dev/null +++ b/src/test/java/org/jabref/logic/crawler/StudyRepositoryTest.java @@ -0,0 +1,287 @@ +package org.jabref.logic.crawler; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; + +import org.jabref.logic.citationkeypattern.CitationKeyGenerator; +import org.jabref.logic.crawler.git.GitHandler; +import org.jabref.logic.database.DatabaseMerger; +import org.jabref.logic.exporter.SavePreferences; +import org.jabref.logic.util.io.FileUtil; +import org.jabref.model.database.BibDatabase; +import org.jabref.model.database.BibDatabaseContext; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.BibEntryTypesManager; +import org.jabref.model.entry.field.StandardField; +import org.jabref.model.entry.field.UnknownField; +import org.jabref.model.entry.types.StandardEntryType; +import org.jabref.model.metadata.SaveOrderConfig; +import org.jabref.model.study.FetchResult; +import org.jabref.model.study.QueryResult; +import org.jabref.model.study.Study; +import org.jabref.model.study.StudyMetaDataField; +import org.jabref.model.util.DummyFileUpdateMonitor; +import org.jabref.preferences.JabRefPreferences; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.mockito.Answers; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +class StudyRepositoryTest { + private static final String NON_EXISTING_DIRECTORY = "nonExistingTestRepositoryDirectory"; + SavePreferences preferences; + BibEntryTypesManager entryTypesManager; + @TempDir + Path tempRepositoryDirectory; + StudyRepository studyRepository; + GitHandler gitHandler = mock(GitHandler.class, Answers.RETURNS_DEFAULTS); + + @Test + void providePathToNonExistentRepositoryThrowsException() { + Path nonExistingRepositoryDirectory = tempRepositoryDirectory.resolve(NON_EXISTING_DIRECTORY); + + assertThrows(IOException.class, () -> new StudyRepository(nonExistingRepositoryDirectory, gitHandler, JabRefPreferences.getInstance().getImportFormatPreferences(), new DummyFileUpdateMonitor(), preferences, entryTypesManager)); + } + + @Test + void providePathToExistentRepositoryWithOutStudyDefinitionFileThrowsException() { + assertThrows(IOException.class, () -> new StudyRepository(tempRepositoryDirectory, gitHandler, JabRefPreferences.getInstance().getImportFormatPreferences(), new DummyFileUpdateMonitor(), preferences, entryTypesManager)); + } + + /** + * Tests whether the StudyRepository correctly imports the study file. + */ + @Test + void studyFileCorrectlyImported() throws Exception { + setUpTestRepository(); + List expectedSearchterms = List.of("Quantum", "Cloud Computing", "TestSearchQuery3"); + List expectedActiveFetchersByName = List.of("Springer", "ArXiv"); + + Study study = new StudyRepository(tempRepositoryDirectory, gitHandler, JabRefPreferences.getInstance().getImportFormatPreferences(), new DummyFileUpdateMonitor(), preferences, entryTypesManager).getStudy(); + + assertEquals(expectedSearchterms, study.getSearchQueryStrings()); + assertEquals("TestStudyName", study.getStudyMetaDataField(StudyMetaDataField.STUDY_NAME).get()); + assertEquals("Jab Ref", study.getStudyMetaDataField(StudyMetaDataField.STUDY_AUTHORS).get()); + assertEquals("Question1; Question2", study.getStudyMetaDataField(StudyMetaDataField.STUDY_RESEARCH_QUESTIONS).get()); + assertEquals(expectedActiveFetchersByName, study.getActiveLibraryEntries() + .stream() + .filter(bibEntry -> bibEntry.getType().getName().equals("library")) + .map(bibEntry -> bibEntry.getField(new UnknownField("name")).orElse("")) + .collect(Collectors.toList()) + ); + } + + /** + * Tests whether the file structure of the repository is created correctly from the study definitions file. + */ + @Test + void repositoryStructureCorrectlyCreated() throws Exception { + // When repository is instantiated the directory structure is created + getTestStudyRepository(); + + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "1 - Quantum"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "2 - Cloud Computing"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "3 - TestSearchQuery3"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "1 - Quantum", "ArXiv.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "2 - Cloud Computing", "ArXiv.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "3 - TestSearchQuery3", "ArXiv.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "1 - Quantum", "Springer.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "2 - Cloud Computing", "Springer.bib"))); + assertTrue(Files.exists(Path.of(tempRepositoryDirectory.toString(), "3 - TestSearchQuery3", "Springer.bib"))); + assertTrue(Files.notExists(Path.of(tempRepositoryDirectory.toString(), "1 - Quantum", "IEEEXplore.bib"))); + assertTrue(Files.notExists(Path.of(tempRepositoryDirectory.toString(), "2 - Cloud Computing", "IEEEXplore.bib"))); + assertTrue(Files.notExists(Path.of(tempRepositoryDirectory.toString(), "3 - TestSearchQuery3", "IEEEXplore.bib"))); + } + + /** + * This tests whether the repository returns the stored bib entries correctly. + */ + @Test + void bibEntriesCorrectlyStored() throws Exception { + StudyRepository repository = getTestStudyRepository(); + setUpTestResultFile(); + List result = repository.getFetcherResultEntries("Quantum", "ArXiv").getEntries(); + assertEquals(getArXivQuantumMockResults(), result); + } + + @Test + void fetcherResultsPersistedCorrectly() throws Exception { + List mockResults = getMockResults(); + + getTestStudyRepository().persist(mockResults); + + assertEquals(getArXivQuantumMockResults(), getTestStudyRepository().getFetcherResultEntries("Quantum", "ArXiv").getEntries()); + assertEquals(getSpringerQuantumMockResults(), getTestStudyRepository().getFetcherResultEntries("Quantum", "Springer").getEntries()); + assertEquals(getSpringerCloudComputingMockResults(), getTestStudyRepository().getFetcherResultEntries("Cloud Computing", "Springer").getEntries()); + } + + @Test + void mergedResultsPersistedCorrectly() throws Exception { + List mockResults = getMockResults(); + List expected = new ArrayList<>(); + expected.addAll(getArXivQuantumMockResults()); + expected.add(getSpringerQuantumMockResults().get(1)); + expected.add(getSpringerQuantumMockResults().get(2)); + + getTestStudyRepository().persist(mockResults); + + // All Springer results are duplicates for "Quantum" + assertEquals(expected, getTestStudyRepository().getQueryResultEntries("Quantum").getEntries()); + assertEquals(getSpringerCloudComputingMockResults(), getTestStudyRepository().getQueryResultEntries("Cloud Computing").getEntries()); + } + + @Test + void setsLastSearchDatePersistedCorrectly() throws Exception { + List mockResults = getMockResults(); + + getTestStudyRepository().persist(mockResults); + + assertEquals(LocalDate.now().toString(), getTestStudyRepository().getStudy().getStudyMetaDataField(StudyMetaDataField.STUDY_LAST_SEARCH).get()); + } + + @Test + void studyResultsPersistedCorrectly() throws Exception { + List mockResults = getMockResults(); + + getTestStudyRepository().persist(mockResults); + + assertEquals(new HashSet<>(getNonDuplicateBibEntryResult().getEntries()), new HashSet<>(getTestStudyRepository().getStudyResultEntries().getEntries())); + } + + private StudyRepository getTestStudyRepository() throws Exception { + if (Objects.isNull(studyRepository)) { + setUpTestRepository(); + studyRepository = new StudyRepository(tempRepositoryDirectory, gitHandler, JabRefPreferences.getInstance().getImportFormatPreferences(), new DummyFileUpdateMonitor(), preferences, entryTypesManager); + } + return studyRepository; + } + + /** + * Set up mocks and copies the study definition file into the test repository + */ + private void setUpTestRepository() throws URISyntaxException { + setUpTestStudyDefinitionFile(); + preferences = mock(SavePreferences.class, Answers.RETURNS_DEEP_STUBS); + when(preferences.getSaveOrder()).thenReturn(new SaveOrderConfig()); + when(preferences.getEncoding()).thenReturn(null); + when(preferences.takeMetadataSaveOrderInAccount()).thenReturn(true); + entryTypesManager = new BibEntryTypesManager(); + } + + private void setUpTestStudyDefinitionFile() throws URISyntaxException { + Path destination = tempRepositoryDirectory.resolve("study.bib"); + URL studyDefinition = this.getClass().getResource("study.bib"); + FileUtil.copyFile(Path.of(studyDefinition.toURI()), destination, false); + } + + /** + * This overwrites the existing result file in the repository with a result file containing multiple BibEntries. + * The repository has to exist before this method is called. + */ + private void setUpTestResultFile() throws URISyntaxException { + Path queryDirectory = Path.of(tempRepositoryDirectory.toString(), "1 - Quantum"); + Path resultFileLocation = Path.of(queryDirectory.toString(), "ArXiv" + ".bib"); + URL resultFile = this.getClass().getResource("ArXivQuantumMock.bib"); + FileUtil.copyFile(Path.of(resultFile.toURI()), resultFileLocation, true); + resultFileLocation = Path.of(queryDirectory.toString(), "Springer" + ".bib"); + resultFile = this.getClass().getResource("SpringerQuantumMock.bib"); + FileUtil.copyFile(Path.of(resultFile.toURI()), resultFileLocation, true); + } + + private BibDatabase getNonDuplicateBibEntryResult() { + BibDatabase mockResults = new BibDatabase(getSpringerCloudComputingMockResults()); + DatabaseMerger merger = new DatabaseMerger(); + merger.merge(mockResults, new BibDatabase(getSpringerQuantumMockResults())); + merger.merge(mockResults, new BibDatabase(getArXivQuantumMockResults())); + return mockResults; + } + + private List getMockResults() { + QueryResult resultQuantum = + new QueryResult("Quantum", List.of( + new FetchResult("ArXiv", new BibDatabase(stripCitationKeys(getArXivQuantumMockResults()))), + new FetchResult("Springer", new BibDatabase(stripCitationKeys(getSpringerQuantumMockResults()))))); + QueryResult resultCloudComputing = new QueryResult("Cloud Computing", List.of(new FetchResult("Springer", new BibDatabase(getSpringerCloudComputingMockResults())))); + return List.of(resultQuantum, resultCloudComputing); + } + + /** + * Strips the citation key from fetched entries as these normally do not have a citation key + */ + private List stripCitationKeys(List entries) { + entries.forEach(bibEntry -> bibEntry.setCitationKey("")); + return entries; + } + + private List getArXivQuantumMockResults() { + BibEntry entry1 = new BibEntry() + .withCitationKey("Blaha") + .withField(StandardField.AUTHOR, "Stephen Blaha") + .withField(StandardField.TITLE, "Quantum Computers and Quantum Computer Languages: Quantum Assembly Language and Quantum C Language"); + entry1.setType(StandardEntryType.Article); + BibEntry entry2 = new BibEntry() + .withCitationKey("Kaye") + .withField(StandardField.AUTHOR, "Phillip Kaye and Michele Mosca") + .withField(StandardField.TITLE, "Quantum Networks for Generating Arbitrary Quantum States"); + entry2.setType(StandardEntryType.Article); + BibEntry entry3 = new BibEntry() + .withCitationKey("Watrous") + .withField(StandardField.AUTHOR, "John Watrous") + .withField(StandardField.TITLE, "Quantum Computational Complexity"); + entry3.setType(StandardEntryType.Article); + + return List.of(entry1, entry2, entry3); + } + + private List getSpringerQuantumMockResults() { + // This is a duplicate of entry 1 of ArXiv + BibEntry entry1 = new BibEntry() + .withCitationKey("Blaha") + .withField(StandardField.AUTHOR, "Stephen Blaha") + .withField(StandardField.TITLE, "Quantum Computers and Quantum Computer Languages: Quantum Assembly Language and Quantum C Language"); + entry1.setType(StandardEntryType.Article); + BibEntry entry2 = new BibEntry() + .withCitationKey("Kroeger") + .withField(StandardField.AUTHOR, "H. Kröger") + .withField(StandardField.TITLE, "Nonlinear Dynamics In Quantum Physics -- Quantum Chaos and Quantum Instantons"); + entry2.setType(StandardEntryType.Article); + BibEntry entry3 = new BibEntry() + .withField(StandardField.AUTHOR, "Zieliński, Cezary") + .withField(StandardField.TITLE, "Automatic Control, Robotics, and Information Processing"); + entry3.setType(StandardEntryType.Article); + + CitationKeyGenerator citationKeyGenerator = new CitationKeyGenerator(new BibDatabaseContext(), JabRefPreferences.getInstance().getCitationKeyPatternPreferences()); + citationKeyGenerator.generateAndSetKey(entry3); + + return List.of(entry1, entry2, entry3); + } + + private List getSpringerCloudComputingMockResults() { + BibEntry entry1 = new BibEntry() + .withCitationKey("Gritzalis") + .withField(StandardField.AUTHOR, "Gritzalis, Dimitris and Stergiopoulos, George and Vasilellis, Efstratios and Anagnostopoulou, Argiro") + .withField(StandardField.TITLE, "Readiness Exercises: Are Risk Assessment Methodologies Ready for the Cloud?"); + entry1.setType(StandardEntryType.Article); + BibEntry entry2 = new BibEntry() + .withCitationKey("Rangras") + .withField(StandardField.AUTHOR, "Rangras, Jimit and Bhavsar, Sejal") + .withField(StandardField.TITLE, "Design of Framework for Disaster Recovery in Cloud Computing"); + entry2.setType(StandardEntryType.Article); + return List.of(entry1, entry2); + } +} diff --git a/src/test/java/org/jabref/logic/importer/fetcher/SearchBasedFetcherCapabilityTest.java b/src/test/java/org/jabref/logic/importer/fetcher/SearchBasedFetcherCapabilityTest.java index 73afa6d63c9..c31bd348b0c 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/SearchBasedFetcherCapabilityTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/SearchBasedFetcherCapabilityTest.java @@ -21,7 +21,7 @@ /** * Defines the set of capability tests that each tests a given search capability, e.g. author based search. * The idea is to code the capabilities of a fetcher into Java code. - * This way, a) the capbilities of a fetcher are checked automatically (because they can change from time-to-time by the provider) + * This way, a) the capabilities of a fetcher are checked automatically (because they can change from time-to-time by the provider) * and b) the queries sent to the fetchers can be debugged directly without a route through to some fetcher code. */ interface SearchBasedFetcherCapabilityTest { diff --git a/src/test/java/org/jabref/model/study/StudyTest.java b/src/test/java/org/jabref/model/study/StudyTest.java new file mode 100644 index 00000000000..cd22449fcd1 --- /dev/null +++ b/src/test/java/org/jabref/model/study/StudyTest.java @@ -0,0 +1,93 @@ +package org.jabref.model.study; + +import java.time.LocalDate; +import java.util.List; + +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.field.StandardField; +import org.jabref.model.entry.field.UnknownField; +import org.jabref.model.entry.types.SystematicLiteratureReviewStudyEntryType; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class StudyTest { + Study testStudy; + + @BeforeEach + public void setUpTestStudy() { + BibEntry studyEntry = new BibEntry() + .withField(new UnknownField("name"), "TestStudyName") + .withField(StandardField.AUTHOR, "Jab Ref") + .withField(new UnknownField("researchQuestions"), "Question1; Question2") + .withField(new UnknownField("gitRepositoryURL"), "https://github.com/eclipse/jgit.git"); + studyEntry.setType(SystematicLiteratureReviewStudyEntryType.STUDY_ENTRY); + + // Create three SearchTerm entries. + BibEntry searchQuery1 = new BibEntry() + .withField(new UnknownField("query"), "TestSearchQuery1"); + searchQuery1.setType(SystematicLiteratureReviewStudyEntryType.SEARCH_QUERY_ENTRY); + searchQuery1.setCitationKey("query1"); + + BibEntry searchQuery2 = new BibEntry() + .withField(new UnknownField("query"), "TestSearchQuery2"); + searchQuery2.setType(SystematicLiteratureReviewStudyEntryType.SEARCH_QUERY_ENTRY); + searchQuery2.setCitationKey("query2"); + + BibEntry searchQuery3 = new BibEntry() + .withField(new UnknownField("query"), "TestSearchQuery3"); + searchQuery3.setType(SystematicLiteratureReviewStudyEntryType.SEARCH_QUERY_ENTRY); + searchQuery3.setCitationKey("query3"); + + // Create two Library entries + BibEntry library1 = new BibEntry() + .withField(new UnknownField("name"), "acm") + .withField(new UnknownField("enabled"), "false") + .withField(new UnknownField("comment"), "disabled, because no good results"); + library1.setType(SystematicLiteratureReviewStudyEntryType.LIBRARY_ENTRY); + library1.setCitationKey("library1"); + + BibEntry library2 = new BibEntry() + .withField(new UnknownField("name"), "arxiv") + .withField(new UnknownField("enabled"), "true") + .withField(new UnknownField("Comment"), ""); + library2.setType(SystematicLiteratureReviewStudyEntryType.LIBRARY_ENTRY); + library2.setCitationKey("library2"); + + testStudy = new Study(studyEntry, List.of(searchQuery1, searchQuery2, searchQuery3), List.of(library1, library2)); + } + + @Test + void getSearchTermsAsStrings() { + List expectedSearchTerms = List.of("TestSearchQuery1", "TestSearchQuery2", "TestSearchQuery3"); + Assertions.assertEquals(expectedSearchTerms, testStudy.getSearchQueryStrings()); + } + + @Test + void setLastSearchTime() { + LocalDate date = LocalDate.now(); + testStudy.setLastSearchDate(date); + Assertions.assertEquals(date.toString(), testStudy.getStudyMetaDataField(StudyMetaDataField.STUDY_LAST_SEARCH).get()); + } + + @Test + void getStudyName() { + Assertions.assertEquals("TestStudyName", testStudy.getStudyMetaDataField(StudyMetaDataField.STUDY_NAME).get()); + } + + @Test + void getStudyAuthor() { + Assertions.assertEquals("Jab Ref", testStudy.getStudyMetaDataField(StudyMetaDataField.STUDY_AUTHORS).get()); + } + + @Test + void getResearchQuestions() { + Assertions.assertEquals("Question1; Question2", testStudy.getStudyMetaDataField(StudyMetaDataField.STUDY_RESEARCH_QUESTIONS).get()); + } + + @Test + void getGitRepositoryURL() { + Assertions.assertEquals("https://github.com/eclipse/jgit.git", testStudy.getStudyMetaDataField(StudyMetaDataField.STUDY_GIT_REPOSITORY).get()); + } +} diff --git a/src/test/resources/org/jabref/logic/crawler/ArXivQuantumMock.bib b/src/test/resources/org/jabref/logic/crawler/ArXivQuantumMock.bib new file mode 100644 index 00000000000..85df0f1060b --- /dev/null +++ b/src/test/resources/org/jabref/logic/crawler/ArXivQuantumMock.bib @@ -0,0 +1,15 @@ + +@Article{Blaha, + author = {Stephen Blaha}, + title = {Quantum Computers and Quantum Computer Languages: Quantum Assembly Language and Quantum C Language}, +} + +@Article{Kaye, + author = {Phillip Kaye and Michele Mosca}, + title = {Quantum Networks for Generating Arbitrary Quantum States}, +} + +@Article{Watrous, + author = {John Watrous}, + title = {Quantum Computational Complexity}, +} diff --git a/src/test/resources/org/jabref/logic/crawler/SpringerCloud ComputingMock.bib b/src/test/resources/org/jabref/logic/crawler/SpringerCloud ComputingMock.bib new file mode 100644 index 00000000000..627166213fa --- /dev/null +++ b/src/test/resources/org/jabref/logic/crawler/SpringerCloud ComputingMock.bib @@ -0,0 +1,9 @@ +@InCollection{Gritzalis, + author = {Gritzalis, Dimitris and Stergiopoulos, George and Vasilellis, Efstratios and Anagnostopoulou, Argiro}, + title = {Readiness Exercises: Are Risk Assessment Methodologies Ready for the Cloud?}, +} + +@InCollection{Rangras, + author = {Rangras, Jimit and Bhavsar, Sejal}, + title = {Design of Framework for Disaster Recovery in Cloud Computing}, +} diff --git a/src/test/resources/org/jabref/logic/crawler/SpringerQuantumMock.bib b/src/test/resources/org/jabref/logic/crawler/SpringerQuantumMock.bib new file mode 100644 index 00000000000..3cfa2f88487 --- /dev/null +++ b/src/test/resources/org/jabref/logic/crawler/SpringerQuantumMock.bib @@ -0,0 +1,9 @@ +@Article{Zielinski, + author = {Zieliński, Cezary}, + title = {Quantum Computers and Quantum Computer Languages: Quantum Assembly Language and Quantum C Language}, +} + +@Article{Kaye, + author = {H. Kröger}, + title = {Quantum Networks for Generating Arbitrary Quantum States}, +} diff --git a/src/test/resources/org/jabref/logic/crawler/study.bib b/src/test/resources/org/jabref/logic/crawler/study.bib new file mode 100644 index 00000000000..3f9809a82e5 --- /dev/null +++ b/src/test/resources/org/jabref/logic/crawler/study.bib @@ -0,0 +1,37 @@ +% Encoding: UTF-8 + +@Study{v10, + name={TestStudyName}, + author={Jab Ref}, + researchQuestions={Question1; Question2}, +} + +@SearchQuery{query1, + query={Quantum}, +} + +@SearchQuery{query2, + query={Cloud Computing}, +} + +@SearchQuery{query3, + query={TestSearchQuery3}, +} + +@Library{library1, + name = {Springer}, + enabled = {true}, + comment = {}, +} + +@Library{library2, + name = {ArXiv}, + enabled = {true}, + comment = {}, +} + +@Library{library3, + name = {IEEEXplore}, + enabled = {false}, + comment = {}, +} From 73bad8e9516f2919004df240627ea8907f8ec021 Mon Sep 17 00:00:00 2001 From: Dominik Voigt Date: Tue, 24 Nov 2020 23:36:01 +0100 Subject: [PATCH 02/24] Implement Springer transformator --- .../importer/SpringerQueryTransformator.java | 115 ++++++++++++++++++ .../logic/importer/QueryParserTest.java | 2 +- .../SpringerQueryTransformatorTest.java | 66 ++++++++++ 3 files changed, 182 insertions(+), 1 deletion(-) create mode 100644 src/main/java/org/jabref/logic/importer/SpringerQueryTransformator.java create mode 100644 src/test/java/org/jabref/logic/importer/SpringerQueryTransformatorTest.java diff --git a/src/main/java/org/jabref/logic/importer/SpringerQueryTransformator.java b/src/main/java/org/jabref/logic/importer/SpringerQueryTransformator.java new file mode 100644 index 00000000000..1fcd970e338 --- /dev/null +++ b/src/main/java/org/jabref/logic/importer/SpringerQueryTransformator.java @@ -0,0 +1,115 @@ +package org.jabref.logic.importer; + +import java.util.Optional; +import java.util.StringJoiner; +import java.util.stream.Collectors; + +import org.jabref.logic.JabRefException; + +import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; +import org.apache.lucene.queryparser.flexible.core.nodes.AndQueryNode; +import org.apache.lucene.queryparser.flexible.core.nodes.BooleanQueryNode; +import org.apache.lucene.queryparser.flexible.core.nodes.FieldQueryNode; +import org.apache.lucene.queryparser.flexible.core.nodes.OrQueryNode; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; +import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * This class converts a query string written in lucene syntax into a complex query. + * + * For simplicity this is currently limited to fielded data and the boolean AND operator. + */ +public class SpringerQueryTransformator { + public static final Logger LOGGER = LoggerFactory.getLogger(SpringerQueryTransformator.class); + + /** + * Transforms a and b and c to (a AND b AND c), where + * a, b, and c can be complex expressions + */ + private Optional transform(BooleanQueryNode query) { + String delimiter; + if (query instanceof OrQueryNode) { + delimiter = " OR "; + } else { + delimiter = " AND "; + } + + String result = query.getChildren().stream() + .map(this::transform) + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.joining(delimiter, "(", ")")); + if (result.equals("()")) { + return Optional.empty(); + } + return Optional.of(result); + } + + private Optional transform(FieldQueryNode query) { + String convertedField; + String convertedValue; + switch (query.getFieldAsString()) { + case "author" -> { + convertedField = "name"; + convertedValue = "\"" + query.getTextAsString() + "\""; + } + case "journal", "title" -> { + convertedField = query.getFieldAsString(); + convertedValue = "\"" + query.getTextAsString() + "\""; + } + case "year" -> { + convertedField = "date"; + convertedValue = query.getTextAsString() + "*"; + } + case "year-range" -> { + convertedField = ""; + String range = query.getTextAsString(); + String[] split = range.split("-"); + StringJoiner resultBuilder = new StringJoiner("* OR date:", "(date:", "*)"); + for (int i = Integer.parseInt(split[0]); i <= Integer.parseInt(split[1]); i++) { + resultBuilder.add(String.valueOf(i)); + } + convertedValue = resultBuilder.toString(); + } + default -> { + convertedField = ""; + convertedValue = "\"" + query.getTextAsString() + "\""; + } // Just add unkown fields as default + } + if (convertedField.isEmpty()) { + return Optional.of(convertedValue); + } else { + return Optional.of(convertedField + ":" + convertedValue); + } + } + + private Optional transform(QueryNode query) { + if (query instanceof BooleanQueryNode) { + return transform((BooleanQueryNode) query); + } else if (query instanceof FieldQueryNode) { + return transform((FieldQueryNode) query); + } else { + LOGGER.error("Unsupported case when transforming the query"); + return Optional.empty(); + } + } + + /** + * Parses the given query string into a complex query using lucene. + * Note: For unique fields, the alphabetically and numerically first instance in the query string is used in the complex query. + * + * @param query The given query string + * @return A complex query containing all fields of the query string + */ + public Optional parseQueryStringIntoComplexQuery(String query) throws JabRefException { + StandardSyntaxParser parser = new StandardSyntaxParser(); + try { + QueryNode luceneQuery = parser.parse(query, "default"); + return transform(luceneQuery); + } catch (QueryNodeParseException e) { + throw new JabRefException("Error parsing query", e); + } + } +} diff --git a/src/test/java/org/jabref/logic/importer/QueryParserTest.java b/src/test/java/org/jabref/logic/importer/QueryParserTest.java index 970788e86c3..acb8c41a40c 100644 --- a/src/test/java/org/jabref/logic/importer/QueryParserTest.java +++ b/src/test/java/org/jabref/logic/importer/QueryParserTest.java @@ -6,7 +6,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; -class QueryParserTest { +public class QueryParserTest { QueryParser parser = new QueryParser(); @Test diff --git a/src/test/java/org/jabref/logic/importer/SpringerQueryTransformatorTest.java b/src/test/java/org/jabref/logic/importer/SpringerQueryTransformatorTest.java new file mode 100644 index 00000000000..8d8823c12f2 --- /dev/null +++ b/src/test/java/org/jabref/logic/importer/SpringerQueryTransformatorTest.java @@ -0,0 +1,66 @@ +package org.jabref.logic.importer; + +import org.jabref.logic.importer.fetcher.ComplexSearchQuery; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class SpringerQueryTransformatorTest { + SpringerQueryTransformator parser = new SpringerQueryTransformator(); + + @Test + public void convertAuthorField() throws Exception { + String searchQuery = parser.parseQueryStringIntoComplexQuery("author:\"Igor Steinmacher\"").get(); + ComplexSearchQuery expectedQuery = ComplexSearchQuery.builder().author("\"Igor Steinmacher\"").build(); + assertEquals("name:\"Igor Steinmacher\"", searchQuery); + } + + @Test + public void convertDefaultField() throws Exception { + String searchQuery = parser.parseQueryStringIntoComplexQuery("\"default value\"").get(); + assertEquals("\"default value\"", searchQuery); + } + + @Test + public void convertExplicitDefaultField() throws Exception { + String searchQuery = parser.parseQueryStringIntoComplexQuery("default:\"default value\"").get(); + assertEquals("\"default value\"", searchQuery); + } + + @Test + public void convertJournalField() throws Exception { + String searchQuery = parser.parseQueryStringIntoComplexQuery("journal:Nature").get(); + assertEquals("journal:\"Nature\"", searchQuery); + } + + @Test + public void convertAlphabeticallyFirstJournalField() throws Exception { + String searchQuery = parser.parseQueryStringIntoComplexQuery("journal:Nature journal:\"Complex Networks\"").get(); + assertEquals("(journal:\"Nature\" AND journal:\"Complex Networks\")", searchQuery); + } + + @Test + public void convertYearField() throws Exception { + String searchQuery = parser.parseQueryStringIntoComplexQuery("year:2015").get(); + assertEquals("date:2015*", searchQuery); + } + + @Test + public void convertNumericallyFirstYearField() throws Exception { + String searchQuery = parser.parseQueryStringIntoComplexQuery("year:2015 year:2014").get(); + assertEquals("(date:2015* AND date:2014*)", searchQuery); + } + + @Test + public void convertYearRangeField() throws Exception { + String searchQuery = parser.parseQueryStringIntoComplexQuery("year-range:2012-2015").get(); + assertEquals("(date:2012* OR date:2013* OR date:2014* OR date:2015*)", searchQuery); + } + + @Test + public void convertMultipleValuesWithTheSameField() throws Exception { + String searchQuery = parser.parseQueryStringIntoComplexQuery("author:\"Igor Steinmacher\" author:\"Christoph Treude\"").get(); + assertEquals("(name:\"Igor Steinmacher\" AND name:\"Christoph Treude\")", searchQuery); + } +} From 5e7f56efdd44f2396c9b584ff2c3ad1faeeffa82 Mon Sep 17 00:00:00 2001 From: Dominik Voigt Date: Thu, 7 Jan 2021 20:58:03 +0100 Subject: [PATCH 03/24] Introduce generified transformator and reimplement Springer transformator --- .../importer/AbstractQueryTransformator.java | 153 ++++++++++++++++++ .../importer/SpringerQueryTransformator.java | 118 +++++--------- .../SpringerQueryTransformatorTest.java | 6 + 3 files changed, 196 insertions(+), 81 deletions(-) create mode 100644 src/main/java/org/jabref/logic/importer/AbstractQueryTransformator.java diff --git a/src/main/java/org/jabref/logic/importer/AbstractQueryTransformator.java b/src/main/java/org/jabref/logic/importer/AbstractQueryTransformator.java new file mode 100644 index 00000000000..40ecb7decf1 --- /dev/null +++ b/src/main/java/org/jabref/logic/importer/AbstractQueryTransformator.java @@ -0,0 +1,153 @@ +package org.jabref.logic.importer; + +import java.util.Optional; +import java.util.stream.Collectors; + +import org.jabref.logic.JabRefException; + +import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; +import org.apache.lucene.queryparser.flexible.core.nodes.BooleanQueryNode; +import org.apache.lucene.queryparser.flexible.core.nodes.FieldQueryNode; +import org.apache.lucene.queryparser.flexible.core.nodes.GroupQueryNode; +import org.apache.lucene.queryparser.flexible.core.nodes.OrQueryNode; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; +import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public abstract class AbstractQueryTransformator { + public static final Logger LOGGER = LoggerFactory.getLogger(SpringerQueryTransformator.class); + public static final String NO_EXPLICIT_FIELD = "default"; + + /** + * Transforms a and b and c to (a AND b AND c), where + * a, b, and c can be complex expressions + */ + private Optional transform(BooleanQueryNode query) { + String delimiter; + if (query instanceof OrQueryNode) { + delimiter = getLogicalOrOperator(); + } else { + // We define the logical AND operator as the default implementation + delimiter = getLogicalAndOperator(); + } + + String result = query.getChildren().stream() + .map(this::transform) + .filter(Optional::isPresent) + .map(Optional::get) + .collect(Collectors.joining(delimiter, "(", ")")); + if (result.equals("()")) { + return Optional.empty(); + } + return Optional.of(result); + } + + /** + * Returns the logical AND operator used by the library + * Note: whitespaces have to be included around the operator, e.g. " AND " + */ + public abstract String getLogicalAndOperator(); + + /** + * Returns the logial OR operator used by the library + * Note: whitespaces have to be included around the operator, e.g. " OR " + */ + public abstract String getLogicalOrOperator(); + + private Optional transform(FieldQueryNode query) { + String term = query.getTextAsString(); + switch (query.getFieldAsString()) { + case "author" -> { + return Optional.of(handleAuthor(term)); + } + case "title" -> { + return Optional.of(handleTitle(term)); + } + case "journal" -> { + return Optional.of(handleJournal(term)); + } + case "year" -> { + return Optional.of(handleYear(term)); + } + case "year-range" -> { + return Optional.of(handleYearRange(term)); + } + case NO_EXPLICIT_FIELD -> { + return Optional.of(handleUnFieldedTerm(term)); + } + default -> { + return handleOtherField(query.getFieldAsString(), term); + } // Just add unkown fields as default + } + } + + /** + * Return a string representation of the author fielded term + */ + protected abstract String handleAuthor(String textAsString); + + /** + * Return a string representation of the title fielded term + */ + protected abstract String handleTitle(String textAsString); + + /** + * Return a string representation of the journal fielded term + */ + protected abstract String handleJournal(String textAsString); + + /** + * Return a string representation of the year fielded term + */ + protected abstract String handleYear(String textAsString); + + /** + * Return a string representation of the year-range fielded term + */ + protected abstract String handleYearRange(String textAsString); + + /** + * Return a string representation of the author fielded term + */ + protected abstract String handleUnFieldedTerm(String term); + + /** + * Return a string representation of the provided field + * If it is not supported return an empty optional. + */ + protected Optional handleOtherField(String fieldAsString, String term){ + return Optional.of(fieldAsString + ": \"" + term + "\""); + } + + private Optional transform(QueryNode query) { + if (query instanceof BooleanQueryNode) { + return transform((BooleanQueryNode) query); + } else if (query instanceof FieldQueryNode) { + return transform((FieldQueryNode) query); + } else if (query instanceof GroupQueryNode) { + return transform(((GroupQueryNode) query).getChild()); + } else { + LOGGER.error("Unsupported case when transforming the query:\n {}", query); + return Optional.empty(); + } + } + + /** + * Parses the given query string into a complex query using lucene. + * Note: For unique fields, the alphabetically and numerically first instance in the query string is used in the complex query. + * + * @param query The given query string + * @return A complex query containing all fields of the query string + */ + public Optional parseQueryStringIntoComplexQuery(String query) throws JabRefException { + StandardSyntaxParser parser = new StandardSyntaxParser(); + try { + QueryNode luceneQuery = parser.parse(query, NO_EXPLICIT_FIELD); + return transform(luceneQuery); + } catch (QueryNodeParseException e) { + throw new JabRefException("Error parsing query", e); + } + } +} diff --git a/src/main/java/org/jabref/logic/importer/SpringerQueryTransformator.java b/src/main/java/org/jabref/logic/importer/SpringerQueryTransformator.java index 1fcd970e338..2e48a82cd77 100644 --- a/src/main/java/org/jabref/logic/importer/SpringerQueryTransformator.java +++ b/src/main/java/org/jabref/logic/importer/SpringerQueryTransformator.java @@ -21,95 +21,51 @@ * * For simplicity this is currently limited to fielded data and the boolean AND operator. */ -public class SpringerQueryTransformator { - public static final Logger LOGGER = LoggerFactory.getLogger(SpringerQueryTransformator.class); +public class SpringerQueryTransformator extends AbstractQueryTransformator { - /** - * Transforms a and b and c to (a AND b AND c), where - * a, b, and c can be complex expressions - */ - private Optional transform(BooleanQueryNode query) { - String delimiter; - if (query instanceof OrQueryNode) { - delimiter = " OR "; - } else { - delimiter = " AND "; - } + @Override + public String getLogicalAndOperator() { + return " AND "; + } - String result = query.getChildren().stream() - .map(this::transform) - .filter(Optional::isPresent) - .map(Optional::get) - .collect(Collectors.joining(delimiter, "(", ")")); - if (result.equals("()")) { - return Optional.empty(); - } - return Optional.of(result); + @Override + public String getLogicalOrOperator() { + return " OR "; } - private Optional transform(FieldQueryNode query) { - String convertedField; - String convertedValue; - switch (query.getFieldAsString()) { - case "author" -> { - convertedField = "name"; - convertedValue = "\"" + query.getTextAsString() + "\""; - } - case "journal", "title" -> { - convertedField = query.getFieldAsString(); - convertedValue = "\"" + query.getTextAsString() + "\""; - } - case "year" -> { - convertedField = "date"; - convertedValue = query.getTextAsString() + "*"; - } - case "year-range" -> { - convertedField = ""; - String range = query.getTextAsString(); - String[] split = range.split("-"); - StringJoiner resultBuilder = new StringJoiner("* OR date:", "(date:", "*)"); - for (int i = Integer.parseInt(split[0]); i <= Integer.parseInt(split[1]); i++) { - resultBuilder.add(String.valueOf(i)); - } - convertedValue = resultBuilder.toString(); - } - default -> { - convertedField = ""; - convertedValue = "\"" + query.getTextAsString() + "\""; - } // Just add unkown fields as default - } - if (convertedField.isEmpty()) { - return Optional.of(convertedValue); - } else { - return Optional.of(convertedField + ":" + convertedValue); - } + @Override + protected String handleAuthor(String textAsString) { + return String.format("name:\"%s\"", textAsString); } - private Optional transform(QueryNode query) { - if (query instanceof BooleanQueryNode) { - return transform((BooleanQueryNode) query); - } else if (query instanceof FieldQueryNode) { - return transform((FieldQueryNode) query); - } else { - LOGGER.error("Unsupported case when transforming the query"); - return Optional.empty(); - } + @Override + protected String handleTitle(String textAsString) { + return String.format("title:\"%s\"", textAsString); } - /** - * Parses the given query string into a complex query using lucene. - * Note: For unique fields, the alphabetically and numerically first instance in the query string is used in the complex query. - * - * @param query The given query string - * @return A complex query containing all fields of the query string - */ - public Optional parseQueryStringIntoComplexQuery(String query) throws JabRefException { - StandardSyntaxParser parser = new StandardSyntaxParser(); - try { - QueryNode luceneQuery = parser.parse(query, "default"); - return transform(luceneQuery); - } catch (QueryNodeParseException e) { - throw new JabRefException("Error parsing query", e); + @Override + protected String handleJournal(String textAsString) { + return String.format("journal:\"%s\"", textAsString); + + } + + @Override + protected String handleYear(String textAsString) { + return String.format("year:\"%s\"", textAsString); + } + + @Override + protected String handleYearRange(String textAsString) { + String[] split = textAsString.split("-"); + StringJoiner resultBuilder = new StringJoiner("* OR date:", "(date:", "*)"); + for (int i = Integer.parseInt(split[0]); i <= Integer.parseInt(split[1]); i++) { + resultBuilder.add(String.valueOf(i)); } + return resultBuilder.toString(); + } + + @Override + protected String handleUnFieldedTerm(String term) { + return "\"" + term + "\""; } } diff --git a/src/test/java/org/jabref/logic/importer/SpringerQueryTransformatorTest.java b/src/test/java/org/jabref/logic/importer/SpringerQueryTransformatorTest.java index 8d8823c12f2..75aee32c65c 100644 --- a/src/test/java/org/jabref/logic/importer/SpringerQueryTransformatorTest.java +++ b/src/test/java/org/jabref/logic/importer/SpringerQueryTransformatorTest.java @@ -63,4 +63,10 @@ public void convertMultipleValuesWithTheSameField() throws Exception { String searchQuery = parser.parseQueryStringIntoComplexQuery("author:\"Igor Steinmacher\" author:\"Christoph Treude\"").get(); assertEquals("(name:\"Igor Steinmacher\" AND name:\"Christoph Treude\")", searchQuery); } + + @Test + public void groupedOperations() throws Exception { + String searchQuery = parser.parseQueryStringIntoComplexQuery("(author:\"Igor Steinmacher\" OR author:\"Christoph Treude\" AND name:\"Christoph Freunde\") AND year:2015").get(); + assertEquals("(name:\"Igor Steinmacher\" OR (name:\"Christoph Treude\" AND name:\"Christoph Freunde\")) AND date:2015*", searchQuery); + } } From 5093f8092afee69227e232972673f444881a9f77 Mon Sep 17 00:00:00 2001 From: Dominik Voigt Date: Thu, 7 Jan 2021 21:13:54 +0100 Subject: [PATCH 04/24] Remove outermost braces as they are unnecessary adapt tests accordingly --- .../importer/AbstractQueryTransformator.java | 17 +++++++++++++++-- .../importer/SpringerQueryTransformator.java | 2 +- .../SpringerQueryTransformatorTest.java | 10 +++++----- 3 files changed, 21 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/AbstractQueryTransformator.java b/src/main/java/org/jabref/logic/importer/AbstractQueryTransformator.java index 40ecb7decf1..22205cef3d3 100644 --- a/src/main/java/org/jabref/logic/importer/AbstractQueryTransformator.java +++ b/src/main/java/org/jabref/logic/importer/AbstractQueryTransformator.java @@ -118,7 +118,7 @@ private Optional transform(FieldQueryNode query) { * If it is not supported return an empty optional. */ protected Optional handleOtherField(String fieldAsString, String term){ - return Optional.of(fieldAsString + ": \"" + term + "\""); + return Optional.of(String.format("%s:\"%s\"", fieldAsString, term)); } private Optional transform(QueryNode query) { @@ -145,9 +145,22 @@ public Optional parseQueryStringIntoComplexQuery(String query) throws Ja StandardSyntaxParser parser = new StandardSyntaxParser(); try { QueryNode luceneQuery = parser.parse(query, NO_EXPLICIT_FIELD); - return transform(luceneQuery); + System.out.println(luceneQuery); + Optional transformedQuery = transform(luceneQuery); + transformedQuery = transformedQuery.map(this::removeOuterBraces); + return transformedQuery; } catch (QueryNodeParseException e) { throw new JabRefException("Error parsing query", e); } } + + /** + * Removes the outer braces as they are unnecessary + */ + private String removeOuterBraces(String query) { + if (query.startsWith("(") && query.endsWith(")")) { + return query.substring(1, query.length() - 1); + } + return query; + } } diff --git a/src/main/java/org/jabref/logic/importer/SpringerQueryTransformator.java b/src/main/java/org/jabref/logic/importer/SpringerQueryTransformator.java index 2e48a82cd77..cfbfb844cb7 100644 --- a/src/main/java/org/jabref/logic/importer/SpringerQueryTransformator.java +++ b/src/main/java/org/jabref/logic/importer/SpringerQueryTransformator.java @@ -51,7 +51,7 @@ protected String handleJournal(String textAsString) { @Override protected String handleYear(String textAsString) { - return String.format("year:\"%s\"", textAsString); + return String.format("date:%s*", textAsString); } @Override diff --git a/src/test/java/org/jabref/logic/importer/SpringerQueryTransformatorTest.java b/src/test/java/org/jabref/logic/importer/SpringerQueryTransformatorTest.java index 75aee32c65c..73fef0506ba 100644 --- a/src/test/java/org/jabref/logic/importer/SpringerQueryTransformatorTest.java +++ b/src/test/java/org/jabref/logic/importer/SpringerQueryTransformatorTest.java @@ -37,7 +37,7 @@ public void convertJournalField() throws Exception { @Test public void convertAlphabeticallyFirstJournalField() throws Exception { String searchQuery = parser.parseQueryStringIntoComplexQuery("journal:Nature journal:\"Complex Networks\"").get(); - assertEquals("(journal:\"Nature\" AND journal:\"Complex Networks\")", searchQuery); + assertEquals("journal:\"Nature\" AND journal:\"Complex Networks\"", searchQuery); } @Test @@ -49,24 +49,24 @@ public void convertYearField() throws Exception { @Test public void convertNumericallyFirstYearField() throws Exception { String searchQuery = parser.parseQueryStringIntoComplexQuery("year:2015 year:2014").get(); - assertEquals("(date:2015* AND date:2014*)", searchQuery); + assertEquals("date:2015* AND date:2014*", searchQuery); } @Test public void convertYearRangeField() throws Exception { String searchQuery = parser.parseQueryStringIntoComplexQuery("year-range:2012-2015").get(); - assertEquals("(date:2012* OR date:2013* OR date:2014* OR date:2015*)", searchQuery); + assertEquals("date:2012* OR date:2013* OR date:2014* OR date:2015*", searchQuery); } @Test public void convertMultipleValuesWithTheSameField() throws Exception { String searchQuery = parser.parseQueryStringIntoComplexQuery("author:\"Igor Steinmacher\" author:\"Christoph Treude\"").get(); - assertEquals("(name:\"Igor Steinmacher\" AND name:\"Christoph Treude\")", searchQuery); + assertEquals("name:\"Igor Steinmacher\" AND name:\"Christoph Treude\"", searchQuery); } @Test public void groupedOperations() throws Exception { - String searchQuery = parser.parseQueryStringIntoComplexQuery("(author:\"Igor Steinmacher\" OR author:\"Christoph Treude\" AND name:\"Christoph Freunde\") AND year:2015").get(); + String searchQuery = parser.parseQueryStringIntoComplexQuery("(author:\"Igor Steinmacher\" OR author:\"Christoph Treude\" AND author:\"Christoph Freunde\") AND year:2015").get(); assertEquals("(name:\"Igor Steinmacher\" OR (name:\"Christoph Treude\" AND name:\"Christoph Freunde\")) AND date:2015*", searchQuery); } } From ec954046eba457f1d10cb28f30467de22278ed4f Mon Sep 17 00:00:00 2001 From: Dominik Voigt Date: Fri, 8 Jan 2021 18:04:47 +0100 Subject: [PATCH 05/24] Add tranformers and move them into a package --- .../importer/SpringerQueryTransformator.java | 71 -------------- .../jabref/logic/importer/fetcher/ArXiv.java | 3 +- .../AbstractQueryTransformer.java} | 78 ++++++++++----- .../transformators/ArXivQueryTransformer.java | 58 ++++++++++++ .../transformators/IEEEQueryTransformer.java | 68 ++++++++++++++ .../ScholarQueryTransformer.java | 65 +++++++++++++ .../SpringerQueryTransformer.java | 62 ++++++++++++ .../SpringerQueryTransformatorTest.java | 72 -------------- .../transformators/InfixTransformerTest.java | 94 +++++++++++++++++++ .../SpringerQueryTransformerTest.java | 51 ++++++++++ 10 files changed, 456 insertions(+), 166 deletions(-) delete mode 100644 src/main/java/org/jabref/logic/importer/SpringerQueryTransformator.java rename src/main/java/org/jabref/logic/importer/{AbstractQueryTransformator.java => fetcher/transformators/AbstractQueryTransformer.java} (68%) create mode 100644 src/main/java/org/jabref/logic/importer/fetcher/transformators/ArXivQueryTransformer.java create mode 100644 src/main/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformer.java create mode 100644 src/main/java/org/jabref/logic/importer/fetcher/transformators/ScholarQueryTransformer.java create mode 100644 src/main/java/org/jabref/logic/importer/fetcher/transformators/SpringerQueryTransformer.java delete mode 100644 src/test/java/org/jabref/logic/importer/SpringerQueryTransformatorTest.java create mode 100644 src/test/java/org/jabref/logic/importer/fetcher/transformators/InfixTransformerTest.java create mode 100644 src/test/java/org/jabref/logic/importer/fetcher/transformators/SpringerQueryTransformerTest.java diff --git a/src/main/java/org/jabref/logic/importer/SpringerQueryTransformator.java b/src/main/java/org/jabref/logic/importer/SpringerQueryTransformator.java deleted file mode 100644 index cfbfb844cb7..00000000000 --- a/src/main/java/org/jabref/logic/importer/SpringerQueryTransformator.java +++ /dev/null @@ -1,71 +0,0 @@ -package org.jabref.logic.importer; - -import java.util.Optional; -import java.util.StringJoiner; -import java.util.stream.Collectors; - -import org.jabref.logic.JabRefException; - -import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; -import org.apache.lucene.queryparser.flexible.core.nodes.AndQueryNode; -import org.apache.lucene.queryparser.flexible.core.nodes.BooleanQueryNode; -import org.apache.lucene.queryparser.flexible.core.nodes.FieldQueryNode; -import org.apache.lucene.queryparser.flexible.core.nodes.OrQueryNode; -import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; -import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * This class converts a query string written in lucene syntax into a complex query. - * - * For simplicity this is currently limited to fielded data and the boolean AND operator. - */ -public class SpringerQueryTransformator extends AbstractQueryTransformator { - - @Override - public String getLogicalAndOperator() { - return " AND "; - } - - @Override - public String getLogicalOrOperator() { - return " OR "; - } - - @Override - protected String handleAuthor(String textAsString) { - return String.format("name:\"%s\"", textAsString); - } - - @Override - protected String handleTitle(String textAsString) { - return String.format("title:\"%s\"", textAsString); - } - - @Override - protected String handleJournal(String textAsString) { - return String.format("journal:\"%s\"", textAsString); - - } - - @Override - protected String handleYear(String textAsString) { - return String.format("date:%s*", textAsString); - } - - @Override - protected String handleYearRange(String textAsString) { - String[] split = textAsString.split("-"); - StringJoiner resultBuilder = new StringJoiner("* OR date:", "(date:", "*)"); - for (int i = Integer.parseInt(split[0]); i <= Integer.parseInt(split[1]); i++) { - resultBuilder.add(String.valueOf(i)); - } - return resultBuilder.toString(); - } - - @Override - protected String handleUnFieldedTerm(String term) { - return "\"" + term + "\""; - } -} diff --git a/src/main/java/org/jabref/logic/importer/fetcher/ArXiv.java b/src/main/java/org/jabref/logic/importer/fetcher/ArXiv.java index 9b640cdd7f0..ea2e099fdb3 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/ArXiv.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/ArXiv.java @@ -262,8 +262,7 @@ public Page performSearchPaged(ComplexSearchQuery complexSearchQuery, complexSearchQuery.getTitlePhrases().forEach(title -> searchTerms.add("ti:" + title)); complexSearchQuery.getAbstractPhrases().forEach(abstr -> searchTerms.add("abs:" + abstr)); complexSearchQuery.getJournal().ifPresent(journal -> searchTerms.add("jr:" + journal)); - // Since ArXiv API does not support year search, we ignore the year related terms - complexSearchQuery.getToYear().ifPresent(year -> searchTerms.add(year.toString())); + complexSearchQuery.getSingleYear().ifPresent(year -> searchTerms.add(year.toString())); searchTerms.addAll(complexSearchQuery.getDefaultFieldPhrases()); String complexQueryString = String.join(" AND ", searchTerms); diff --git a/src/main/java/org/jabref/logic/importer/AbstractQueryTransformator.java b/src/main/java/org/jabref/logic/importer/fetcher/transformators/AbstractQueryTransformer.java similarity index 68% rename from src/main/java/org/jabref/logic/importer/AbstractQueryTransformator.java rename to src/main/java/org/jabref/logic/importer/fetcher/transformators/AbstractQueryTransformer.java index 22205cef3d3..802fe1c635a 100644 --- a/src/main/java/org/jabref/logic/importer/AbstractQueryTransformator.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformators/AbstractQueryTransformer.java @@ -1,4 +1,4 @@ -package org.jabref.logic.importer; +package org.jabref.logic.importer.fetcher.transformators; import java.util.Optional; import java.util.stream.Collectors; @@ -9,6 +9,7 @@ import org.apache.lucene.queryparser.flexible.core.nodes.BooleanQueryNode; import org.apache.lucene.queryparser.flexible.core.nodes.FieldQueryNode; import org.apache.lucene.queryparser.flexible.core.nodes.GroupQueryNode; +import org.apache.lucene.queryparser.flexible.core.nodes.ModifierQueryNode; import org.apache.lucene.queryparser.flexible.core.nodes.OrQueryNode; import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; @@ -16,13 +17,17 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -public abstract class AbstractQueryTransformator { - public static final Logger LOGGER = LoggerFactory.getLogger(SpringerQueryTransformator.class); +/** + * In case the transformator contains state for a query transformation (such as the {@link IEEEQueryTransformer}), it has to be noted at the JavaDoc. + * Otherwise, a single instance QueryTransformer can be used. + */ +public abstract class AbstractQueryTransformer { + public static final Logger LOGGER = LoggerFactory.getLogger(SpringerQueryTransformer.class); public static final String NO_EXPLICIT_FIELD = "default"; /** * Transforms a and b and c to (a AND b AND c), where - * a, b, and c can be complex expressions + * a, b, and c can be complex expressions. */ private Optional transform(BooleanQueryNode query) { String delimiter; @@ -35,8 +40,7 @@ private Optional transform(BooleanQueryNode query) { String result = query.getChildren().stream() .map(this::transform) - .filter(Optional::isPresent) - .map(Optional::get) + .flatMap(Optional::stream) .collect(Collectors.joining(delimiter, "(", ")")); if (result.equals("()")) { return Optional.empty(); @@ -46,15 +50,27 @@ private Optional transform(BooleanQueryNode query) { /** * Returns the logical AND operator used by the library - * Note: whitespaces have to be included around the operator, e.g. " AND " + * Note: whitespaces have to be included around the operator + * + * Example: " AND " */ - public abstract String getLogicalAndOperator(); + protected abstract String getLogicalAndOperator(); /** - * Returns the logial OR operator used by the library - * Note: whitespaces have to be included around the operator, e.g. " OR " + * Returns the logical OR operator used by the library + * Note: whitespaces have to be included around the operator + * + * Example: " OR " */ - public abstract String getLogicalOrOperator(); + protected abstract String getLogicalOrOperator(); + + /** + * Returns the logical NOT operator used by the library + * + * Example: "!" + */ + protected abstract String getLogicalNotOperator(); + private Optional transform(FieldQueryNode query) { String term = query.getTextAsString(); @@ -69,47 +85,65 @@ private Optional transform(FieldQueryNode query) { return Optional.of(handleJournal(term)); } case "year" -> { - return Optional.of(handleYear(term)); + String s = handleYear(term); + return s.isEmpty() ? Optional.empty() : Optional.of(s); } case "year-range" -> { - return Optional.of(handleYearRange(term)); + String s = handleYearRange(term); + return s.isEmpty() ? Optional.empty() : Optional.of(s); } case NO_EXPLICIT_FIELD -> { return Optional.of(handleUnFieldedTerm(term)); } default -> { + // Just add unknown fields as default return handleOtherField(query.getFieldAsString(), term); - } // Just add unkown fields as default + } } } /** - * Return a string representation of the author fielded term + * Handles the not modifier, all other cases are silently ignored */ - protected abstract String handleAuthor(String textAsString); + private Optional transform(ModifierQueryNode query) { + ModifierQueryNode.Modifier modifier = query.getModifier(); + if (modifier == ModifierQueryNode.Modifier.MOD_NOT) { + return transform(query.getChild()).map(s -> getLogicalNotOperator() + s); + } else { + return transform(query.getChild()); + } + } + + /** + * Return a string representation of the author fielded term + */ + protected abstract String handleAuthor(String author); /** * Return a string representation of the title fielded term */ - protected abstract String handleTitle(String textAsString); + protected abstract String handleTitle(String title); /** * Return a string representation of the journal fielded term */ - protected abstract String handleJournal(String textAsString); + protected abstract String handleJournal(String journalTitle); /** * Return a string representation of the year fielded term */ - protected abstract String handleYear(String textAsString); + protected abstract String handleYear(String year); /** * Return a string representation of the year-range fielded term + * Should follow the structure yyyy-yyyy + * + * Example: 2015-2021 */ - protected abstract String handleYearRange(String textAsString); + protected abstract String handleYearRange(String yearRange); /** - * Return a string representation of the author fielded term + * Return a string representation of the un-fielded (default fielded) term */ protected abstract String handleUnFieldedTerm(String term); @@ -128,6 +162,8 @@ private Optional transform(QueryNode query) { return transform((FieldQueryNode) query); } else if (query instanceof GroupQueryNode) { return transform(((GroupQueryNode) query).getChild()); + } else if (query instanceof ModifierQueryNode) { + return transform((ModifierQueryNode) query); } else { LOGGER.error("Unsupported case when transforming the query:\n {}", query); return Optional.empty(); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformators/ArXivQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformators/ArXivQueryTransformer.java new file mode 100644 index 00000000000..046bcf65f05 --- /dev/null +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformators/ArXivQueryTransformer.java @@ -0,0 +1,58 @@ +package org.jabref.logic.importer.fetcher.transformators; + +public class ArXivQueryTransformer extends AbstractQueryTransformer { + @Override + protected String getLogicalAndOperator() { + return " AND "; + } + + @Override + protected String getLogicalOrOperator() { + return " OR "; + } + + /** + * Check whether this works as an unary operator + * @return + */ + @Override + protected String getLogicalNotOperator() { + return " ANDNOT "; + } + + @Override + protected String handleAuthor(String author) { + return "au:" + author; + } + + @Override + protected String handleTitle(String title) { + return "ti:" + title; + } + + @Override + protected String handleJournal(String journalTitle) { + return "jr:" + journalTitle; + } + + /** + * Manual testing shows that this works if added as an unfielded term, might lead to false positives + */ + @Override + protected String handleYear(String year) { + return year; + } + + /** + * Currently not supported + */ + @Override + protected String handleYearRange(String yearRange) { + return ""; + } + + @Override + protected String handleUnFieldedTerm(String term) { + return "all:" + term; + } +} diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformer.java new file mode 100644 index 00000000000..93e37350e17 --- /dev/null +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformer.java @@ -0,0 +1,68 @@ +package org.jabref.logic.importer.fetcher.transformators; + +/** + * Needs to be instantiated for each new query + */ +public class IEEEQueryTransformer extends AbstractQueryTransformer { + // These have to be integrated into the IEEE query URL as these are just supported as query parameters + private int startYear = Integer.MAX_VALUE; + private int endYear = Integer.MIN_VALUE; + + @Override + protected String getLogicalAndOperator() { + return " AND "; + } + + @Override + protected String getLogicalOrOperator() { + return " OR "; + } + + @Override + protected String getLogicalNotOperator() { + return "NOT "; + } + + @Override + protected String handleAuthor(String author) { + return "author:" + author; + } + + @Override + protected String handleTitle(String title) { + return "article_title:" + title; + } + + @Override + protected String handleJournal(String journalTitle) { + return "publication_title:" + journalTitle; + } + + @Override + protected String handleYear(String year) { + startYear = Math.max(startYear, Integer.parseInt(year)); + endYear = Math.max(endYear, Integer.parseInt(year)); + return ""; + } + + @Override + protected String handleYearRange(String yearRange) { + String[] split = yearRange.split("-"); + startYear = Math.max(startYear, Integer.parseInt(split[0])); + endYear = Math.max(endYear, Integer.parseInt(split[1])); + return ""; + } + + @Override + protected String handleUnFieldedTerm(String term) { + return term; + } + + public int getStartYear() { + return startYear; + } + + public int getEndYear() { + return endYear; + } +} diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformators/ScholarQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformators/ScholarQueryTransformer.java new file mode 100644 index 00000000000..41c6079ea78 --- /dev/null +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformators/ScholarQueryTransformer.java @@ -0,0 +1,65 @@ +package org.jabref.logic.importer.fetcher.transformators; + +public class ScholarQueryTransformer extends AbstractQueryTransformer { + // These have to be integrated into the Google Scholar query URL as these are just supported as query parameters + private int startYear = Integer.MAX_VALUE; + private int endYear = Integer.MIN_VALUE; + + @Override + protected String getLogicalAndOperator() { + return " AND "; + } + + @Override + protected String getLogicalOrOperator() { + return " OR "; + } + + @Override + protected String getLogicalNotOperator() { + return "-"; + } + + @Override + protected String handleAuthor(String author) { + return "author:" + author; + } + + @Override + protected String handleTitle(String title) { + return "allintitle:" + title; + } + + @Override + protected String handleJournal(String journalTitle) { + return "source:" + journalTitle; + } + + @Override + protected String handleYear(String year) { + startYear = Math.max(startYear, Integer.parseInt(year)); + endYear = Math.max(endYear, Integer.parseInt(year)); + return ""; + } + + @Override + protected String handleYearRange(String yearRange) { + String[] split = yearRange.split("-"); + startYear = Math.max(startYear, Integer.parseInt(split[0])); + endYear = Math.max(endYear, Integer.parseInt(split[1])); + return ""; + } + + @Override + protected String handleUnFieldedTerm(String term) { + return term; + } + + public int getStartYear() { + return startYear; + } + + public int getEndYear() { + return endYear; + } +} diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformators/SpringerQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformators/SpringerQueryTransformer.java new file mode 100644 index 00000000000..6c70a12d533 --- /dev/null +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformators/SpringerQueryTransformer.java @@ -0,0 +1,62 @@ +package org.jabref.logic.importer.fetcher.transformators; + +import java.util.StringJoiner; + +/** + * This class converts a query string written in lucene syntax into a complex query. + * + * For simplicity this is currently limited to fielded data and the boolean AND operator. + */ +public class SpringerQueryTransformer extends AbstractQueryTransformer { + + @Override + public String getLogicalAndOperator() { + return " AND "; + } + + @Override + public String getLogicalOrOperator() { + return " OR "; + } + + @Override + protected String getLogicalNotOperator() { + return "-"; + } + + @Override + protected String handleAuthor(String author) { + return String.format("name:\"%s\"", author); + } + + @Override + protected String handleTitle(String title) { + return String.format("title:\"%s\"", title); + } + + @Override + protected String handleJournal(String journalTitle) { + return String.format("journal:\"%s\"", journalTitle); + + } + + @Override + protected String handleYear(String year) { + return String.format("date:%s*", year); + } + + @Override + protected String handleYearRange(String yearRange) { + String[] split = yearRange.split("-"); + StringJoiner resultBuilder = new StringJoiner("* OR date:", "(date:", "*)"); + for (int i = Integer.parseInt(split[0]); i <= Integer.parseInt(split[1]); i++) { + resultBuilder.add(String.valueOf(i)); + } + return resultBuilder.toString(); + } + + @Override + protected String handleUnFieldedTerm(String term) { + return "\"" + term + "\""; + } +} diff --git a/src/test/java/org/jabref/logic/importer/SpringerQueryTransformatorTest.java b/src/test/java/org/jabref/logic/importer/SpringerQueryTransformatorTest.java deleted file mode 100644 index 73fef0506ba..00000000000 --- a/src/test/java/org/jabref/logic/importer/SpringerQueryTransformatorTest.java +++ /dev/null @@ -1,72 +0,0 @@ -package org.jabref.logic.importer; - -import org.jabref.logic.importer.fetcher.ComplexSearchQuery; - -import org.junit.jupiter.api.Test; - -import static org.junit.jupiter.api.Assertions.assertEquals; - -public class SpringerQueryTransformatorTest { - SpringerQueryTransformator parser = new SpringerQueryTransformator(); - - @Test - public void convertAuthorField() throws Exception { - String searchQuery = parser.parseQueryStringIntoComplexQuery("author:\"Igor Steinmacher\"").get(); - ComplexSearchQuery expectedQuery = ComplexSearchQuery.builder().author("\"Igor Steinmacher\"").build(); - assertEquals("name:\"Igor Steinmacher\"", searchQuery); - } - - @Test - public void convertDefaultField() throws Exception { - String searchQuery = parser.parseQueryStringIntoComplexQuery("\"default value\"").get(); - assertEquals("\"default value\"", searchQuery); - } - - @Test - public void convertExplicitDefaultField() throws Exception { - String searchQuery = parser.parseQueryStringIntoComplexQuery("default:\"default value\"").get(); - assertEquals("\"default value\"", searchQuery); - } - - @Test - public void convertJournalField() throws Exception { - String searchQuery = parser.parseQueryStringIntoComplexQuery("journal:Nature").get(); - assertEquals("journal:\"Nature\"", searchQuery); - } - - @Test - public void convertAlphabeticallyFirstJournalField() throws Exception { - String searchQuery = parser.parseQueryStringIntoComplexQuery("journal:Nature journal:\"Complex Networks\"").get(); - assertEquals("journal:\"Nature\" AND journal:\"Complex Networks\"", searchQuery); - } - - @Test - public void convertYearField() throws Exception { - String searchQuery = parser.parseQueryStringIntoComplexQuery("year:2015").get(); - assertEquals("date:2015*", searchQuery); - } - - @Test - public void convertNumericallyFirstYearField() throws Exception { - String searchQuery = parser.parseQueryStringIntoComplexQuery("year:2015 year:2014").get(); - assertEquals("date:2015* AND date:2014*", searchQuery); - } - - @Test - public void convertYearRangeField() throws Exception { - String searchQuery = parser.parseQueryStringIntoComplexQuery("year-range:2012-2015").get(); - assertEquals("date:2012* OR date:2013* OR date:2014* OR date:2015*", searchQuery); - } - - @Test - public void convertMultipleValuesWithTheSameField() throws Exception { - String searchQuery = parser.parseQueryStringIntoComplexQuery("author:\"Igor Steinmacher\" author:\"Christoph Treude\"").get(); - assertEquals("name:\"Igor Steinmacher\" AND name:\"Christoph Treude\"", searchQuery); - } - - @Test - public void groupedOperations() throws Exception { - String searchQuery = parser.parseQueryStringIntoComplexQuery("(author:\"Igor Steinmacher\" OR author:\"Christoph Treude\" AND author:\"Christoph Freunde\") AND year:2015").get(); - assertEquals("(name:\"Igor Steinmacher\" OR (name:\"Christoph Treude\" AND name:\"Christoph Freunde\")) AND date:2015*", searchQuery); - } -} diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/InfixTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/InfixTransformerTest.java new file mode 100644 index 00000000000..5873d99e49d --- /dev/null +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/InfixTransformerTest.java @@ -0,0 +1,94 @@ +package org.jabref.logic.importer.fetcher.transformators; + +import java.util.Optional; + +import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Test Interface for all transformers that use infix notation for their logical binary operators + */ +public interface InfixTransformerTest { + + + AbstractQueryTransformer getTransformator(); + + /* All prefixes have to include the used separator + * Example in the case of ':': "author:" + */ + String getAuthorPrefix(); + + String getUnFieldedPrefix(); + + String getJournalPrefix(); + + String getTitlePrefix(); + + @Test + public default void convertAuthorField() throws Exception { + Optional searchQuery = getTransformator().parseQueryStringIntoComplexQuery("author:\"Igor Steinmacher\""); + Optional expected = Optional.of(getAuthorPrefix() + "\"Igor Steinmacher\""); + assertEquals(expected, searchQuery); + } + + @Test + public default void convertUnFieldedTerm() throws Exception { + Optional searchQuery = getTransformator().parseQueryStringIntoComplexQuery("\"default value\""); + Optional expected = Optional.of(getUnFieldedPrefix() + "\"default value\""); + assertEquals(expected, searchQuery); + } + + @Test + public default void convertExplicitUnFieldedTerm() throws Exception { + Optional searchQuery = getTransformator().parseQueryStringIntoComplexQuery("default:\"default value\""); + Optional expected = Optional.of(getUnFieldedPrefix() + "\"default value\""); + assertEquals(expected, searchQuery); + } + + @Test + public default void convertJournalField() throws Exception { + Optional searchQuery = getTransformator().parseQueryStringIntoComplexQuery("journal:Nature"); + Optional expected = Optional.of(getJournalPrefix() + "\"Nature\""); + assertEquals(expected, searchQuery); + } + + @Test + public default void convertAlphabeticallyFirstJournalField() throws Exception { + Optional searchQuery = getTransformator().parseQueryStringIntoComplexQuery("journal:Nature journal:\"Complex Networks\""); + Optional expected = Optional.of(getJournalPrefix() + "\"Nature\" " + getTransformator().getLogicalAndOperator() + getJournalPrefix() + "\"Complex Networks\""); + assertEquals(expected, searchQuery); + } + + @Test + public void convertYearField() throws Exception; + + @Test + public void convertNumericallyFirstYearField() throws Exception; + + @Test + public void convertYearRangeField() throws Exception; + + @Test + public default void convertMultipleValuesWithTheSameField() throws Exception { + Optional searchQuery = getTransformator().parseQueryStringIntoComplexQuery("author:\"Igor Steinmacher\" author:\"Christoph Treude\""); + Optional expected = Optional.of(getAuthorPrefix() + "\"Igor Steinmacher\"" + getTransformator().getLogicalAndOperator() + getAuthorPrefix() + "\"Christoph Treude\""); + assertEquals(expected, searchQuery); + } + + @Test + public default void groupedOperations() throws Exception { + Optional searchQuery = getTransformator().parseQueryStringIntoComplexQuery("(author:\"Igor Steinmacher\" OR author:\"Christoph Treude\" AND author:\"Christoph Freunde\") AND title:test"); + Optional expected = Optional.of("(" + getAuthorPrefix() + "\"Igor Steinmacher\"" + getTransformator().getLogicalOrOperator() + "(" + getAuthorPrefix() + "\"Christoph Treude\"" + getTransformator().getLogicalAndOperator() + getAuthorPrefix() + "\"Christoph Freunde\"))" + getTransformator().getLogicalAndOperator() + getTitlePrefix() + "test"); + assertEquals(expected, searchQuery); + } + + @Test + public default void notOperator() throws Exception { + Optional searchQuery = getTransformator().parseQueryStringIntoComplexQuery("!(author:\"Igor Steinmacher\" OR author:\"Christoph Treude\")"); + Optional expected = Optional.of(getTransformator().getLogicalNotOperator() + "(" + getAuthorPrefix() + "\"Igor Steinmacher\"" + getTransformator().getLogicalOrOperator() + getAuthorPrefix() + "\"Christoph Treude\")"); + assertEquals(expected, searchQuery); + } +} diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/SpringerQueryTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/SpringerQueryTransformerTest.java new file mode 100644 index 00000000000..d5bf7194947 --- /dev/null +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/SpringerQueryTransformerTest.java @@ -0,0 +1,51 @@ +package org.jabref.logic.importer.fetcher.transformators; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +class SpringerQueryTransformerTest implements InfixTransformerTest { + + @Override + public String getAuthorPrefix() { + return "name:"; + } + + @Override + public AbstractQueryTransformer getTransformator() { + return new SpringerQueryTransformer(); + } + + @Override + public String getUnFieldedPrefix() { + return ""; + } + + @Override + public String getJournalPrefix() { + return "journal:"; + } + + @Override + public String getTitlePrefix() { + return "title:"; + } + + @Test + public void convertYearField() throws Exception { + String searchQuery = getTransformator().parseQueryStringIntoComplexQuery("year:2015").get(); + assertEquals("date:2015*", searchQuery); + } + + @Test + public void convertNumericallyFirstYearField() throws Exception { + String searchQuery = getTransformator().parseQueryStringIntoComplexQuery("year:2015 year:2014").get(); + assertEquals("date:2015* AND date:2014*", searchQuery); + } + + @Test + public void convertYearRangeField() throws Exception { + String searchQuery = getTransformator().parseQueryStringIntoComplexQuery("year-range:2012-2015").get(); + assertEquals("date:2012* OR date:2013* OR date:2014* OR date:2015*", searchQuery); + } +} From 400f1e92bdd36191c33daccbd6ae7f9d8b0e6bc2 Mon Sep 17 00:00:00 2001 From: Dominik Voigt Date: Fri, 8 Jan 2021 19:23:50 +0100 Subject: [PATCH 06/24] Add more transformers, add transformer tests --- .../AbstractQueryTransformer.java | 2 +- .../transformators/DBLPQueryTransformer.java | 66 +++++++++++++++++++ .../transformators/GVKQueryTransformer.java | 61 +++++++++++++++++ .../SpringerQueryTransformer.java | 2 +- .../ZbMathQueryTransformer.java | 49 ++++++++++++++ .../ArXivQueryTransformerTest.java | 48 ++++++++++++++ .../DBLPQueryTransformerTest.java | 48 ++++++++++++++ .../GVKQueryTransformerTest.java | 49 ++++++++++++++ .../IEEEQueryTransformerTest.java | 52 +++++++++++++++ .../transformators/InfixTransformerTest.java | 23 +++---- .../ScholarQueryTransformerTest.java | 52 +++++++++++++++ .../SpringerQueryTransformerTest.java | 22 +++---- .../ZbMathQueryTransformerTest.java | 49 ++++++++++++++ 13 files changed, 496 insertions(+), 27 deletions(-) create mode 100644 src/main/java/org/jabref/logic/importer/fetcher/transformators/DBLPQueryTransformer.java create mode 100644 src/main/java/org/jabref/logic/importer/fetcher/transformators/GVKQueryTransformer.java create mode 100644 src/main/java/org/jabref/logic/importer/fetcher/transformators/ZbMathQueryTransformer.java create mode 100644 src/test/java/org/jabref/logic/importer/fetcher/transformators/ArXivQueryTransformerTest.java create mode 100644 src/test/java/org/jabref/logic/importer/fetcher/transformators/DBLPQueryTransformerTest.java create mode 100644 src/test/java/org/jabref/logic/importer/fetcher/transformators/GVKQueryTransformerTest.java create mode 100644 src/test/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformerTest.java create mode 100644 src/test/java/org/jabref/logic/importer/fetcher/transformators/ScholarQueryTransformerTest.java create mode 100644 src/test/java/org/jabref/logic/importer/fetcher/transformators/ZbMathQueryTransformerTest.java diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformators/AbstractQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformators/AbstractQueryTransformer.java index 802fe1c635a..cbe5046b3dd 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/transformators/AbstractQueryTransformer.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformators/AbstractQueryTransformer.java @@ -22,7 +22,7 @@ * Otherwise, a single instance QueryTransformer can be used. */ public abstract class AbstractQueryTransformer { - public static final Logger LOGGER = LoggerFactory.getLogger(SpringerQueryTransformer.class); + private static final Logger LOGGER = LoggerFactory.getLogger(AbstractQueryTransformer.class); public static final String NO_EXPLICIT_FIELD = "default"; /** diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformators/DBLPQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformators/DBLPQueryTransformer.java new file mode 100644 index 00000000000..cbaffdb3557 --- /dev/null +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformators/DBLPQueryTransformer.java @@ -0,0 +1,66 @@ +package org.jabref.logic.importer.fetcher.transformators; + +import java.util.StringJoiner; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DBLPQueryTransformer extends AbstractQueryTransformer { + private static final Logger LOGGER = LoggerFactory.getLogger(DBLPQueryTransformer.class); + + @Override + protected String getLogicalAndOperator() { + return " "; + } + + @Override + protected String getLogicalOrOperator() { + return "|"; + } + + @Override + protected String getLogicalNotOperator() { + LOGGER.warn("DBLP does not support Boolean NOT operator."); + return ""; + } + + @Override + protected String handleAuthor(String author) { + // DBLP does not support explicit author field search + return author; + } + + @Override + protected String handleTitle(String title) { + // DBLP does not support explicit title field search + return title; + } + + @Override + protected String handleJournal(String journalTitle) { + // DBLP does not support explicit journal field search + return journalTitle; + } + + @Override + protected String handleYear(String year) { + // DBLP does not support explicit year field search + return year; + } + + @Override + protected String handleYearRange(String yearRange) { + // DBLP does not support explicit year range search + String[] split = yearRange.split("-"); + StringJoiner resultBuilder = new StringJoiner(getLogicalOrOperator()); + for (int i = Integer.parseInt(split[0]); i <= Integer.parseInt(split[1]); i++) { + resultBuilder.add(String.valueOf(i)); + } + return resultBuilder.toString(); + } + + @Override + protected String handleUnFieldedTerm(String term) { + return term; + } +} diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformators/GVKQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformators/GVKQueryTransformer.java new file mode 100644 index 00000000000..1bbf901e0a6 --- /dev/null +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformators/GVKQueryTransformer.java @@ -0,0 +1,61 @@ +package org.jabref.logic.importer.fetcher.transformators; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class GVKQueryTransformer extends AbstractQueryTransformer { + private static final Logger LOGGER = LoggerFactory.getLogger(GVKQueryTransformer.class); + + @Override + protected String getLogicalAndOperator() { + // GVK defaults to AND + return " "; + } + + @Override + protected String getLogicalOrOperator() { + LOGGER.warn("GVK does not support Boolean OR operator"); + return ""; + } + + @Override + protected String getLogicalNotOperator() { + LOGGER.warn("GVK does not support Boolean NOT operator"); + return ""; + } + + @Override + protected String handleAuthor(String author) { + return "per:" + author; + } + + @Override + protected String handleTitle(String title) { + return "tit:" + title; + } + + @Override + protected String handleJournal(String journalTitle) { + // zti means "Zeitschrift", does not search for conferences (kon:) + return "zti:" + journalTitle; + } + + @Override + protected String handleYear(String year) { + // ver means Veröffentlichungsangaben + return "ver:" + year; + } + + @Override + protected String handleYearRange(String yearRange) { + // Returns empty string as otherwise leads to no results + return ""; + } + + @Override + protected String handleUnFieldedTerm(String term) { + // all does not search in full-text + // Other option is txt: but this does not search in meta data + return "all:" + term; + } +} diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformators/SpringerQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformators/SpringerQueryTransformer.java index 6c70a12d533..3907caef708 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/transformators/SpringerQueryTransformer.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformators/SpringerQueryTransformer.java @@ -48,7 +48,7 @@ protected String handleYear(String year) { @Override protected String handleYearRange(String yearRange) { String[] split = yearRange.split("-"); - StringJoiner resultBuilder = new StringJoiner("* OR date:", "(date:", "*)"); + StringJoiner resultBuilder = new StringJoiner("*" + getLogicalOrOperator() + "date:", "(date:", "*)"); for (int i = Integer.parseInt(split[0]); i <= Integer.parseInt(split[1]); i++) { resultBuilder.add(String.valueOf(i)); } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformators/ZbMathQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformators/ZbMathQueryTransformer.java new file mode 100644 index 00000000000..395c450e742 --- /dev/null +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformators/ZbMathQueryTransformer.java @@ -0,0 +1,49 @@ +package org.jabref.logic.importer.fetcher.transformators; + +public class ZbMathQueryTransformer extends AbstractQueryTransformer { + + @Override + protected String getLogicalAndOperator() { + return " & "; + } + + @Override + protected String getLogicalOrOperator() { + return " | "; + } + + @Override + protected String getLogicalNotOperator() { + return "!"; + } + + @Override + protected String handleAuthor(String author) { + return "au:" + author; + } + + @Override + protected String handleTitle(String title) { + return "ti:" + title; + } + + @Override + protected String handleJournal(String journalTitle) { + return "so:" + journalTitle; + } + + @Override + protected String handleYear(String year) { + return "py:" + year; + } + + @Override + protected String handleYearRange(String yearRange) { + return "py:" + yearRange; + } + + @Override + protected String handleUnFieldedTerm(String term) { + return "any:" + term; + } +} diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/ArXivQueryTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/ArXivQueryTransformerTest.java new file mode 100644 index 00000000000..499707c9a2a --- /dev/null +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/ArXivQueryTransformerTest.java @@ -0,0 +1,48 @@ +package org.jabref.logic.importer.fetcher.transformators; + +import java.util.Optional; + +import org.junit.jupiter.api.Disabled; + +import static org.junit.jupiter.api.Assertions.*; + +class ArXivQueryTransformerTest implements InfixTransformerTest { + + @Override + public AbstractQueryTransformer getTransformator() { + return new ArXivQueryTransformer(); + } + + @Override + public String getAuthorPrefix() { + return "au:"; + } + + @Override + public String getUnFieldedPrefix() { + return "all:"; + } + + @Override + public String getJournalPrefix() { + return "jr:"; + } + + @Override + public String getTitlePrefix() { + return "ti:"; + } + + @Override + public void convertYearField() throws Exception { + Optional query = getTransformator().parseQueryStringIntoComplexQuery("year:2018"); + Optional expected = Optional.of("2018"); + assertEquals(expected, query); + } + + @Disabled("Year-range search is not supported by the arXiv API") + @Override + public void convertYearRangeField() throws Exception { + + } +} diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/DBLPQueryTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/DBLPQueryTransformerTest.java new file mode 100644 index 00000000000..a1ae7c00452 --- /dev/null +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/DBLPQueryTransformerTest.java @@ -0,0 +1,48 @@ +package org.jabref.logic.importer.fetcher.transformators; + +import java.util.Optional; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +class DBLPQueryTransformerTest implements InfixTransformerTest { + + @Override + public AbstractQueryTransformer getTransformator() { + return new DBLPQueryTransformer(); + } + + @Override + public String getAuthorPrefix() { + return ""; + } + + @Override + public String getUnFieldedPrefix() { + return ""; } + + @Override + public String getJournalPrefix() { + return ""; + } + + @Override + public String getTitlePrefix() { + return ""; + } + + @Override + public void convertYearField() throws Exception { + Optional searchQuery = getTransformator().parseQueryStringIntoComplexQuery("year:2015"); + Optional expected = Optional.of("2015"); + assertEquals(expected, searchQuery); + } + + @Override + public void convertYearRangeField() throws Exception { + Optional searchQuery = getTransformator().parseQueryStringIntoComplexQuery("year-range:2012-2015"); + Optional expected = Optional.of("2012|2013|2014|2015"); + assertEquals(expected, searchQuery); + } +} diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/GVKQueryTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/GVKQueryTransformerTest.java new file mode 100644 index 00000000000..8eaa095b61e --- /dev/null +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/GVKQueryTransformerTest.java @@ -0,0 +1,49 @@ +package org.jabref.logic.importer.fetcher.transformators; + +import java.util.Optional; + +import org.junit.jupiter.api.Disabled; + +import static org.junit.jupiter.api.Assertions.*; + +class GVKQueryTransformerTest implements InfixTransformerTest{ + + @Override + public AbstractQueryTransformer getTransformator() { + return new GVKQueryTransformer(); + } + + @Override + public String getAuthorPrefix() { + return "per:"; + } + + @Override + public String getUnFieldedPrefix() { + return "all:"; + } + + @Override + public String getJournalPrefix() { + return "zti:"; + } + + @Override + public String getTitlePrefix() { + return "tit:"; + } + + @Override + public void convertYearField() throws Exception { + Optional query = getTransformator().parseQueryStringIntoComplexQuery("year:2018"); + + Optional expected = Optional.of("ver:2018"); + assertEquals(expected, query); + } + + @Disabled("Not supported by GVK") + @Override + public void convertYearRangeField() throws Exception { + + } +} diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformerTest.java new file mode 100644 index 00000000000..b6c921104ee --- /dev/null +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformerTest.java @@ -0,0 +1,52 @@ +package org.jabref.logic.importer.fetcher.transformators; + +import static org.junit.jupiter.api.Assertions.*; + +class IEEEQueryTransformerTest implements InfixTransformerTest{ + + @Override + public AbstractQueryTransformer getTransformator() { + return new IEEEQueryTransformer(); + } + + @Override + public String getAuthorPrefix() { + return "author:"; + } + + @Override + public String getUnFieldedPrefix() { + return ""; + } + + @Override + public String getJournalPrefix() { + return "publication_title:"; + } + + @Override + public String getTitlePrefix() { + return "article_title:"; + } + + @Override + public void convertYearField() throws Exception { + IEEEQueryTransformer transformer = ((IEEEQueryTransformer) getTransformator()); + + transformer.parseQueryStringIntoComplexQuery("year:2021"); + + assertEquals(2021, transformer.getStartYear()); + assertEquals(2021, transformer.getEndYear()); + } + + @Override + public void convertYearRangeField() throws Exception { + + IEEEQueryTransformer transformer = ((IEEEQueryTransformer) getTransformator()); + + transformer.parseQueryStringIntoComplexQuery("year-range:2018-2021"); + + assertEquals(2018, transformer.getStartYear()); + assertEquals(2021, transformer.getEndYear()); + } +} diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/InfixTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/InfixTransformerTest.java index 5873d99e49d..6de0bb6706e 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/transformators/InfixTransformerTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/InfixTransformerTest.java @@ -28,65 +28,62 @@ public interface InfixTransformerTest { String getTitlePrefix(); @Test - public default void convertAuthorField() throws Exception { + default void convertAuthorField() throws Exception { Optional searchQuery = getTransformator().parseQueryStringIntoComplexQuery("author:\"Igor Steinmacher\""); Optional expected = Optional.of(getAuthorPrefix() + "\"Igor Steinmacher\""); assertEquals(expected, searchQuery); } @Test - public default void convertUnFieldedTerm() throws Exception { + default void convertUnFieldedTerm() throws Exception { Optional searchQuery = getTransformator().parseQueryStringIntoComplexQuery("\"default value\""); Optional expected = Optional.of(getUnFieldedPrefix() + "\"default value\""); assertEquals(expected, searchQuery); } @Test - public default void convertExplicitUnFieldedTerm() throws Exception { + default void convertExplicitUnFieldedTerm() throws Exception { Optional searchQuery = getTransformator().parseQueryStringIntoComplexQuery("default:\"default value\""); Optional expected = Optional.of(getUnFieldedPrefix() + "\"default value\""); assertEquals(expected, searchQuery); } @Test - public default void convertJournalField() throws Exception { + default void convertJournalField() throws Exception { Optional searchQuery = getTransformator().parseQueryStringIntoComplexQuery("journal:Nature"); Optional expected = Optional.of(getJournalPrefix() + "\"Nature\""); assertEquals(expected, searchQuery); } @Test - public default void convertAlphabeticallyFirstJournalField() throws Exception { + default void convertAlphabeticallyFirstJournalField() throws Exception { Optional searchQuery = getTransformator().parseQueryStringIntoComplexQuery("journal:Nature journal:\"Complex Networks\""); Optional expected = Optional.of(getJournalPrefix() + "\"Nature\" " + getTransformator().getLogicalAndOperator() + getJournalPrefix() + "\"Complex Networks\""); assertEquals(expected, searchQuery); } @Test - public void convertYearField() throws Exception; + void convertYearField() throws Exception; @Test - public void convertNumericallyFirstYearField() throws Exception; + void convertYearRangeField() throws Exception; @Test - public void convertYearRangeField() throws Exception; - - @Test - public default void convertMultipleValuesWithTheSameField() throws Exception { + default void convertMultipleValuesWithTheSameField() throws Exception { Optional searchQuery = getTransformator().parseQueryStringIntoComplexQuery("author:\"Igor Steinmacher\" author:\"Christoph Treude\""); Optional expected = Optional.of(getAuthorPrefix() + "\"Igor Steinmacher\"" + getTransformator().getLogicalAndOperator() + getAuthorPrefix() + "\"Christoph Treude\""); assertEquals(expected, searchQuery); } @Test - public default void groupedOperations() throws Exception { + default void groupedOperations() throws Exception { Optional searchQuery = getTransformator().parseQueryStringIntoComplexQuery("(author:\"Igor Steinmacher\" OR author:\"Christoph Treude\" AND author:\"Christoph Freunde\") AND title:test"); Optional expected = Optional.of("(" + getAuthorPrefix() + "\"Igor Steinmacher\"" + getTransformator().getLogicalOrOperator() + "(" + getAuthorPrefix() + "\"Christoph Treude\"" + getTransformator().getLogicalAndOperator() + getAuthorPrefix() + "\"Christoph Freunde\"))" + getTransformator().getLogicalAndOperator() + getTitlePrefix() + "test"); assertEquals(expected, searchQuery); } @Test - public default void notOperator() throws Exception { + default void notOperator() throws Exception { Optional searchQuery = getTransformator().parseQueryStringIntoComplexQuery("!(author:\"Igor Steinmacher\" OR author:\"Christoph Treude\")"); Optional expected = Optional.of(getTransformator().getLogicalNotOperator() + "(" + getAuthorPrefix() + "\"Igor Steinmacher\"" + getTransformator().getLogicalOrOperator() + getAuthorPrefix() + "\"Christoph Treude\")"); assertEquals(expected, searchQuery); diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/ScholarQueryTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/ScholarQueryTransformerTest.java new file mode 100644 index 00000000000..6a913ed1c2f --- /dev/null +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/ScholarQueryTransformerTest.java @@ -0,0 +1,52 @@ +package org.jabref.logic.importer.fetcher.transformators; + +import static org.junit.jupiter.api.Assertions.*; + +class ScholarQueryTransformerTest implements InfixTransformerTest { + + @Override + public AbstractQueryTransformer getTransformator() { + return new ScholarQueryTransformer(); + } + + @Override + public String getAuthorPrefix() { + return "author:"; + } + + @Override + public String getUnFieldedPrefix() { + return ""; + } + + @Override + public String getJournalPrefix() { + return "source:"; + } + + @Override + public String getTitlePrefix() { + return "allintitle:"; + } + + @Override + public void convertYearField() throws Exception { + ScholarQueryTransformer transformer = ((ScholarQueryTransformer) getTransformator()); + + transformer.parseQueryStringIntoComplexQuery("year:2021"); + + assertEquals(2021, transformer.getStartYear()); + assertEquals(2021, transformer.getEndYear()); + } + + @Override + public void convertYearRangeField() throws Exception { + + ScholarQueryTransformer transformer = ((ScholarQueryTransformer) getTransformator()); + + transformer.parseQueryStringIntoComplexQuery("year-range:2018-2021"); + + assertEquals(2018, transformer.getStartYear()); + assertEquals(2021, transformer.getEndYear()); + } +} diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/SpringerQueryTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/SpringerQueryTransformerTest.java index d5bf7194947..b71da7449b9 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/transformators/SpringerQueryTransformerTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/SpringerQueryTransformerTest.java @@ -1,5 +1,7 @@ package org.jabref.logic.importer.fetcher.transformators; +import java.util.Optional; + import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.*; @@ -31,21 +33,17 @@ public String getTitlePrefix() { return "title:"; } - @Test + @Override public void convertYearField() throws Exception { - String searchQuery = getTransformator().parseQueryStringIntoComplexQuery("year:2015").get(); - assertEquals("date:2015*", searchQuery); + Optional searchQuery = getTransformator().parseQueryStringIntoComplexQuery("year:2015"); + Optional expected = Optional.of("date:2015*"); + assertEquals(expected, searchQuery); } - @Test - public void convertNumericallyFirstYearField() throws Exception { - String searchQuery = getTransformator().parseQueryStringIntoComplexQuery("year:2015 year:2014").get(); - assertEquals("date:2015* AND date:2014*", searchQuery); - } - - @Test + @Override public void convertYearRangeField() throws Exception { - String searchQuery = getTransformator().parseQueryStringIntoComplexQuery("year-range:2012-2015").get(); - assertEquals("date:2012* OR date:2013* OR date:2014* OR date:2015*", searchQuery); + Optional searchQuery = getTransformator().parseQueryStringIntoComplexQuery("year-range:2012-2015"); + Optional expected = Optional.of("date:2012* OR date:2013* OR date:2014* OR date:2015*"); + assertEquals(expected, searchQuery); } } diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/ZbMathQueryTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/ZbMathQueryTransformerTest.java new file mode 100644 index 00000000000..ba46d97b40d --- /dev/null +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/ZbMathQueryTransformerTest.java @@ -0,0 +1,49 @@ +package org.jabref.logic.importer.fetcher.transformators; + +import java.util.Optional; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.*; + +class ZbMathQueryTransformerTest implements InfixTransformerTest{ + + @Override + public AbstractQueryTransformer getTransformator() { + return new ZbMathQueryTransformer(); + } + + @Override + public String getAuthorPrefix() { + return "au:"; + } + + @Override + public String getUnFieldedPrefix() { + return "any:"; + } + + @Override + public String getJournalPrefix() { + return "so:"; + } + + @Override + public String getTitlePrefix() { + return "ti:"; + } + + @Override + public void convertYearField() throws Exception { + Optional searchQuery = getTransformator().parseQueryStringIntoComplexQuery("year:2015"); + Optional expected = Optional.of("py:2015"); + assertEquals(expected, searchQuery); + } + + @Override + public void convertYearRangeField() throws Exception { + Optional searchQuery = getTransformator().parseQueryStringIntoComplexQuery("year-range:2012-2015"); + Optional expected = Optional.of("py:2012-2015"); + assertEquals(expected, searchQuery); + } +} From ffef88f8cec7e57cdf4c5d892cdf6a3efb10e6d3 Mon Sep 17 00:00:00 2001 From: Dominik Voigt Date: Fri, 8 Jan 2021 19:50:44 +0100 Subject: [PATCH 07/24] Fix bugs in transformers --- .../fetcher/transformators/ArXivQueryTransformer.java | 8 ++++---- .../fetcher/transformators/DBLPQueryTransformer.java | 8 ++++---- .../fetcher/transformators/GVKQueryTransformer.java | 8 ++++---- .../fetcher/transformators/IEEEQueryTransformer.java | 8 ++++---- .../fetcher/transformators/ScholarQueryTransformer.java | 8 ++++---- .../fetcher/transformators/ZbMathQueryTransformer.java | 8 ++++---- .../fetcher/transformators/InfixTransformerTest.java | 4 ++-- 7 files changed, 26 insertions(+), 26 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformators/ArXivQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformators/ArXivQueryTransformer.java index 046bcf65f05..6559cb5b5ad 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/transformators/ArXivQueryTransformer.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformators/ArXivQueryTransformer.java @@ -22,17 +22,17 @@ protected String getLogicalNotOperator() { @Override protected String handleAuthor(String author) { - return "au:" + author; + return String.format("au:\"%s\"", author); } @Override protected String handleTitle(String title) { - return "ti:" + title; + return String.format("ti:\"%s\"", title); } @Override protected String handleJournal(String journalTitle) { - return "jr:" + journalTitle; + return String.format("jr:\"%s\"", journalTitle); } /** @@ -53,6 +53,6 @@ protected String handleYearRange(String yearRange) { @Override protected String handleUnFieldedTerm(String term) { - return "all:" + term; + return String.format("all:\"%s\"", term); } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformators/DBLPQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformators/DBLPQueryTransformer.java index cbaffdb3557..3b50ce590c6 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/transformators/DBLPQueryTransformer.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformators/DBLPQueryTransformer.java @@ -27,19 +27,19 @@ protected String getLogicalNotOperator() { @Override protected String handleAuthor(String author) { // DBLP does not support explicit author field search - return author; + return String.format("\"%s\"", author); } @Override protected String handleTitle(String title) { // DBLP does not support explicit title field search - return title; + return String.format("\"%s\"", title); } @Override protected String handleJournal(String journalTitle) { // DBLP does not support explicit journal field search - return journalTitle; + return String.format("\"%s\"", journalTitle); } @Override @@ -61,6 +61,6 @@ protected String handleYearRange(String yearRange) { @Override protected String handleUnFieldedTerm(String term) { - return term; + return String.format("\"%s\"",term); } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformators/GVKQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformators/GVKQueryTransformer.java index 1bbf901e0a6..9cbda4786cb 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/transformators/GVKQueryTransformer.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformators/GVKQueryTransformer.java @@ -26,18 +26,18 @@ protected String getLogicalNotOperator() { @Override protected String handleAuthor(String author) { - return "per:" + author; + return String.format("per:\"%s\"", author); } @Override protected String handleTitle(String title) { - return "tit:" + title; + return String.format("tit:\"%s\"", title); } @Override protected String handleJournal(String journalTitle) { // zti means "Zeitschrift", does not search for conferences (kon:) - return "zti:" + journalTitle; + return String.format("zti:\"%s\"", journalTitle); } @Override @@ -56,6 +56,6 @@ protected String handleYearRange(String yearRange) { protected String handleUnFieldedTerm(String term) { // all does not search in full-text // Other option is txt: but this does not search in meta data - return "all:" + term; + return String.format("all:\"%s\"", term); } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformer.java index 93e37350e17..5772472dcbf 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformer.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformer.java @@ -25,17 +25,17 @@ protected String getLogicalNotOperator() { @Override protected String handleAuthor(String author) { - return "author:" + author; + return String.format("author:\"%s\"", author); } @Override protected String handleTitle(String title) { - return "article_title:" + title; + return String.format("article_title:\"%s\"", title); } @Override protected String handleJournal(String journalTitle) { - return "publication_title:" + journalTitle; + return String.format("publication_title:\"%s\"", journalTitle); } @Override @@ -55,7 +55,7 @@ protected String handleYearRange(String yearRange) { @Override protected String handleUnFieldedTerm(String term) { - return term; + return String.format("\"%s\"", term); } public int getStartYear() { diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformators/ScholarQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformators/ScholarQueryTransformer.java index 41c6079ea78..a5882508bf5 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/transformators/ScholarQueryTransformer.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformators/ScholarQueryTransformer.java @@ -22,17 +22,17 @@ protected String getLogicalNotOperator() { @Override protected String handleAuthor(String author) { - return "author:" + author; + return String.format("author:\"%s\"", author); } @Override protected String handleTitle(String title) { - return "allintitle:" + title; + return String.format("allintitle:\"%s\"", title); } @Override protected String handleJournal(String journalTitle) { - return "source:" + journalTitle; + return String.format("source:\"%s\"", journalTitle); } @Override @@ -52,7 +52,7 @@ protected String handleYearRange(String yearRange) { @Override protected String handleUnFieldedTerm(String term) { - return term; + return String.format("\"%s\"", term); } public int getStartYear() { diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformators/ZbMathQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformators/ZbMathQueryTransformer.java index 395c450e742..b2a466c20a1 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/transformators/ZbMathQueryTransformer.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformators/ZbMathQueryTransformer.java @@ -19,17 +19,17 @@ protected String getLogicalNotOperator() { @Override protected String handleAuthor(String author) { - return "au:" + author; + return String.format("au:\"%s\"", author); } @Override protected String handleTitle(String title) { - return "ti:" + title; + return String.format("ti:\"%s\"", title); } @Override protected String handleJournal(String journalTitle) { - return "so:" + journalTitle; + return String.format("so:\"%s\"", journalTitle); } @Override @@ -44,6 +44,6 @@ protected String handleYearRange(String yearRange) { @Override protected String handleUnFieldedTerm(String term) { - return "any:" + term; + return String.format("any:\"%s\"", term); } } diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/InfixTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/InfixTransformerTest.java index 6de0bb6706e..dcfc9cf544b 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/transformators/InfixTransformerTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/InfixTransformerTest.java @@ -58,7 +58,7 @@ default void convertJournalField() throws Exception { @Test default void convertAlphabeticallyFirstJournalField() throws Exception { Optional searchQuery = getTransformator().parseQueryStringIntoComplexQuery("journal:Nature journal:\"Complex Networks\""); - Optional expected = Optional.of(getJournalPrefix() + "\"Nature\" " + getTransformator().getLogicalAndOperator() + getJournalPrefix() + "\"Complex Networks\""); + Optional expected = Optional.of(getJournalPrefix() + "\"Nature\"" + getTransformator().getLogicalAndOperator() + getJournalPrefix() + "\"Complex Networks\""); assertEquals(expected, searchQuery); } @@ -78,7 +78,7 @@ default void convertMultipleValuesWithTheSameField() throws Exception { @Test default void groupedOperations() throws Exception { Optional searchQuery = getTransformator().parseQueryStringIntoComplexQuery("(author:\"Igor Steinmacher\" OR author:\"Christoph Treude\" AND author:\"Christoph Freunde\") AND title:test"); - Optional expected = Optional.of("(" + getAuthorPrefix() + "\"Igor Steinmacher\"" + getTransformator().getLogicalOrOperator() + "(" + getAuthorPrefix() + "\"Christoph Treude\"" + getTransformator().getLogicalAndOperator() + getAuthorPrefix() + "\"Christoph Freunde\"))" + getTransformator().getLogicalAndOperator() + getTitlePrefix() + "test"); + Optional expected = Optional.of("(" + getAuthorPrefix() + "\"Igor Steinmacher\"" + getTransformator().getLogicalOrOperator() + "(" + getAuthorPrefix() + "\"Christoph Treude\"" + getTransformator().getLogicalAndOperator() + getAuthorPrefix() + "\"Christoph Freunde\"))" + getTransformator().getLogicalAndOperator() + getTitlePrefix() + "\"test\""); assertEquals(expected, searchQuery); } From ca03284ca2f0ef9c2ee94c4a1ea0599962553a86 Mon Sep 17 00:00:00 2001 From: Dominik Voigt Date: Tue, 12 Jan 2021 10:27:54 +0100 Subject: [PATCH 08/24] Extend ArxivQueryTransformer,and fix IEEE transformer --- .../logic/importer/PagedSearchBasedFetcher.java | 3 +++ .../logic/importer/SearchBasedFetcher.java | 3 +++ .../transformators/ArXivQueryTransformer.java | 17 +++++++++++++++++ .../transformators/IEEEQueryTransformer.java | 4 ++-- .../ArXivQueryTransformerTest.java | 11 +++++++++-- 5 files changed, 34 insertions(+), 4 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java b/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java index ea547522049..eb31bfb5fcd 100644 --- a/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java +++ b/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java @@ -6,6 +6,7 @@ import java.util.Optional; import org.jabref.logic.importer.fetcher.ComplexSearchQuery; +import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; import org.jabref.model.entry.BibEntry; import org.jabref.model.paging.Page; @@ -49,4 +50,6 @@ default List performSearch(ComplexSearchQuery complexSearchQuery) thro default List performSearch(String complexSearchQuery) throws FetcherException { return new ArrayList<>(performSearchPaged(complexSearchQuery, 0).getContent()); } + + AbstractQueryTransformer getQueryTransformer(); } diff --git a/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java b/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java index faeb4ffa6f6..5ca04372f11 100644 --- a/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java +++ b/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java @@ -5,6 +5,7 @@ import java.util.Optional; import org.jabref.logic.importer.fetcher.ComplexSearchQuery; +import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; import org.jabref.model.entry.BibEntry; /** @@ -36,4 +37,6 @@ default List performSearch(String complexSearchQuery) throws FetcherEx // Otherwise just use query as a default term return this.performSearch(generatedQuery.orElse(ComplexSearchQuery.builder().defaultFieldPhrase(complexSearchQuery).build())); } + + AbstractQueryTransformer getQueryTransformer(); } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformators/ArXivQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformators/ArXivQueryTransformer.java index 6559cb5b5ad..28ddbd92edf 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/transformators/ArXivQueryTransformer.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformators/ArXivQueryTransformer.java @@ -1,6 +1,10 @@ package org.jabref.logic.importer.fetcher.transformators; public class ArXivQueryTransformer extends AbstractQueryTransformer { + // These can be used for filtering in post processing + private int startYear = Integer.MAX_VALUE; + private int endYear = Integer.MIN_VALUE; + @Override protected String getLogicalAndOperator() { return " AND "; @@ -40,6 +44,8 @@ protected String handleJournal(String journalTitle) { */ @Override protected String handleYear(String year) { + startYear = Math.max(startYear, Integer.parseInt(year)); + endYear = Math.min(endYear, Integer.parseInt(year)); return year; } @@ -48,6 +54,9 @@ protected String handleYear(String year) { */ @Override protected String handleYearRange(String yearRange) { + String[] split = yearRange.split("-"); + startYear = Math.max(startYear, Integer.parseInt(split[0])); + endYear = Math.min(endYear, Integer.parseInt(split[1])); return ""; } @@ -55,4 +64,12 @@ protected String handleYearRange(String yearRange) { protected String handleUnFieldedTerm(String term) { return String.format("all:\"%s\"", term); } + + public int getStartYear() { + return startYear; + } + + public int getEndYear() { + return endYear; + } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformer.java index 5772472dcbf..49570e707fa 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformer.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformer.java @@ -41,7 +41,7 @@ protected String handleJournal(String journalTitle) { @Override protected String handleYear(String year) { startYear = Math.max(startYear, Integer.parseInt(year)); - endYear = Math.max(endYear, Integer.parseInt(year)); + endYear = Math.min(endYear, Integer.parseInt(year)); return ""; } @@ -49,7 +49,7 @@ protected String handleYear(String year) { protected String handleYearRange(String yearRange) { String[] split = yearRange.split("-"); startYear = Math.max(startYear, Integer.parseInt(split[0])); - endYear = Math.max(endYear, Integer.parseInt(split[1])); + endYear = Math.min(endYear, Integer.parseInt(split[1])); return ""; } diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/ArXivQueryTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/ArXivQueryTransformerTest.java index 499707c9a2a..f9d03654c43 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/transformators/ArXivQueryTransformerTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/ArXivQueryTransformerTest.java @@ -35,14 +35,21 @@ public String getTitlePrefix() { @Override public void convertYearField() throws Exception { - Optional query = getTransformator().parseQueryStringIntoComplexQuery("year:2018"); + ArXivQueryTransformer transformer = ((ArXivQueryTransformer) getTransformator()); + Optional query = transformer.parseQueryStringIntoComplexQuery("year:2018"); Optional expected = Optional.of("2018"); assertEquals(expected, query); + assertEquals(2018, transformer.getStartYear()); + assertEquals(2018, transformer.getEndYear()); } - @Disabled("Year-range search is not supported by the arXiv API") @Override public void convertYearRangeField() throws Exception { + ArXivQueryTransformer transformer = ((ArXivQueryTransformer) getTransformator()); + transformer.parseQueryStringIntoComplexQuery("year-range:2018-2021"); + + assertEquals(2018, transformer.getStartYear()); + assertEquals(2021, transformer.getEndYear()); } } From 777a63c927f447a6201e9af0c73950c4c4ae2b93 Mon Sep 17 00:00:00 2001 From: Dominik Voigt Date: Wed, 13 Jan 2021 11:57:45 +0100 Subject: [PATCH 09/24] Add default query transformer --- .../fetcher/transformators/DefaultQueryTransformer.java | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 src/main/java/org/jabref/logic/importer/fetcher/transformators/DefaultQueryTransformer.java diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformators/DefaultQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformators/DefaultQueryTransformer.java new file mode 100644 index 00000000000..445949e6d36 --- /dev/null +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformators/DefaultQueryTransformer.java @@ -0,0 +1,2 @@ +package org.jabref.logic.importer.fetcher.transformators;public class DefaultQueryTransformer { +} From 20d6146e5f45d03fdb4e389294e09cffe77a720b Mon Sep 17 00:00:00 2001 From: Dominik Voigt Date: Wed, 13 Jan 2021 14:09:00 +0100 Subject: [PATCH 10/24] Adapt transformers and tests --- .../jabref/logic/crawler/StudyFetcher.java | 4 +- .../importer/PagedSearchBasedFetcher.java | 35 +++++------- .../PagedSearchBasedParserFetcher.java | 34 ++++-------- .../logic/importer/SearchBasedFetcher.java | 27 ++++++---- .../importer/SearchBasedParserFetcher.java | 19 +++---- .../importer/fetcher/ACMPortalFetcher.java | 4 +- .../jabref/logic/importer/fetcher/ArXiv.java | 24 ++++----- .../fetcher/AstrophysicsDataSystem.java | 12 ++--- .../logic/importer/fetcher/CiteSeer.java | 4 +- ...fComputerScienceBibliographiesFetcher.java | 4 +- .../fetcher/CompositeSearchBasedFetcher.java | 24 ++++----- .../logic/importer/fetcher/CrossRef.java | 4 +- .../logic/importer/fetcher/DBLPFetcher.java | 11 +++- .../logic/importer/fetcher/DOAJFetcher.java | 4 +- .../logic/importer/fetcher/GoogleScholar.java | 33 +++++------- .../fetcher/GrobidCitationFetcher.java | 19 +++---- .../logic/importer/fetcher/GvkFetcher.java | 46 ++++------------ .../jabref/logic/importer/fetcher/IEEE.java | 37 +++---------- .../importer/fetcher/INSPIREFetcher.java | 4 +- .../logic/importer/fetcher/JstorFetcher.java | 39 +------------- .../logic/importer/fetcher/MathSciNet.java | 4 +- .../importer/fetcher/MedlineFetcher.java | 53 +++++++++---------- .../importer/fetcher/SpringerFetcher.java | 16 +++--- .../jabref/logic/importer/fetcher/ZbMATH.java | 11 +++- .../DefaultQueryTransformer.java | 49 ++++++++++++++++- .../transformators/GVKQueryTransformer.java | 13 +++-- .../transformators/JstorQueryTransformer.java | 49 +++++++++++++++++ .../importer/fetcher/GvkFetcherTest.java | 14 ----- .../SearchBasedFetcherCapabilityTest.java | 23 +++----- .../GVKQueryTransformerTest.java | 8 +-- .../JstorQueryTransformerTest.java | 41 ++++++++++++++ 31 files changed, 338 insertions(+), 331 deletions(-) create mode 100644 src/main/java/org/jabref/logic/importer/fetcher/transformators/JstorQueryTransformer.java create mode 100644 src/test/java/org/jabref/logic/importer/fetcher/transformators/JstorQueryTransformerTest.java diff --git a/src/main/java/org/jabref/logic/crawler/StudyFetcher.java b/src/main/java/org/jabref/logic/crawler/StudyFetcher.java index c39ba7efe52..6f5dbef01fe 100644 --- a/src/main/java/org/jabref/logic/crawler/StudyFetcher.java +++ b/src/main/java/org/jabref/logic/crawler/StudyFetcher.java @@ -5,7 +5,7 @@ import java.util.Objects; import java.util.stream.Collectors; -import org.jabref.logic.importer.FetcherException; +import org.jabref.logic.JabRefException; import org.jabref.logic.importer.PagedSearchBasedFetcher; import org.jabref.logic.importer.SearchBasedFetcher; import org.jabref.model.database.BibDatabase; @@ -72,7 +72,7 @@ private FetchResult performSearchOnQueryForFetcher(String searchQuery, SearchBas fetchResult = fetcher.performSearch(searchQuery); } return new FetchResult(fetcher.getName(), new BibDatabase(fetchResult)); - } catch (FetcherException e) { + } catch (JabRefException e) { LOGGER.warn(String.format("%s API request failed", fetcher.getName()), e); return null; } diff --git a/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java b/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java index eb31bfb5fcd..43f6c666028 100644 --- a/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java +++ b/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java @@ -5,33 +5,31 @@ import java.util.List; import java.util.Optional; -import org.jabref.logic.importer.fetcher.ComplexSearchQuery; -import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; +import org.jabref.logic.JabRefException; import org.jabref.model.entry.BibEntry; import org.jabref.model.paging.Page; public interface PagedSearchBasedFetcher extends SearchBasedFetcher { /** - * @param complexSearchQuery the complex query defining all fielded search parameters - * @param pageNumber requested site number indexed from 0 + * @param transformedQuery the complex query defining all fielded search parameters + * @param pageNumber requested site number indexed from 0 * @return Page with search results */ - Page performSearchPaged(ComplexSearchQuery complexSearchQuery, int pageNumber) throws FetcherException; + Page performSearchPagedForTransformedQuery(String transformedQuery, int pageNumber) throws FetcherException; /** - * @param complexSearchQuery query string that can be parsed into a complex search query - * @param pageNumber requested site number indexed from 0 + * @param searchQuery query string that can be parsed into a complex search query + * @param pageNumber requested site number indexed from 0 * @return Page with search results */ - default Page performSearchPaged(String complexSearchQuery, int pageNumber) throws FetcherException { - if (complexSearchQuery.isBlank()) { - return new Page<>(complexSearchQuery, pageNumber, Collections.emptyList()); + default Page performSearchPaged(String searchQuery, int pageNumber) throws JabRefException { + if (searchQuery.isBlank()) { + return new Page<>(searchQuery, pageNumber, Collections.emptyList()); } - QueryParser queryParser = new QueryParser(); - Optional generatedQuery = queryParser.parseQueryStringIntoComplexQuery(complexSearchQuery); + Optional transformedQuery = getQueryTransformer().parseQueryStringIntoComplexQuery(searchQuery); // Otherwise just use query as a default term - return this.performSearchPaged(generatedQuery.orElse(ComplexSearchQuery.builder().defaultFieldPhrase(complexSearchQuery).build()), pageNumber); + return this.performSearchPagedForTransformedQuery(transformedQuery.orElse(searchQuery), pageNumber); } /** @@ -42,14 +40,7 @@ default int getPageSize() { } @Override - default List performSearch(ComplexSearchQuery complexSearchQuery) throws FetcherException { - return new ArrayList<>(performSearchPaged(complexSearchQuery, 0).getContent()); + default List performSearchForTransformedQuery(String transformedQuery) throws FetcherException { + return new ArrayList<>(performSearchPagedForTransformedQuery(transformedQuery, 0).getContent()); } - - @Override - default List performSearch(String complexSearchQuery) throws FetcherException { - return new ArrayList<>(performSearchPaged(complexSearchQuery, 0).getContent()); - } - - AbstractQueryTransformer getQueryTransformer(); } diff --git a/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java b/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java index 7f7b3380f0b..de1863807d8 100644 --- a/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java +++ b/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java @@ -7,22 +7,21 @@ import java.net.URL; import java.util.List; -import org.jabref.logic.importer.fetcher.ComplexSearchQuery; import org.jabref.model.entry.BibEntry; import org.jabref.model.paging.Page; public interface PagedSearchBasedParserFetcher extends SearchBasedParserFetcher, PagedSearchBasedFetcher { @Override - default Page performSearchPaged(ComplexSearchQuery complexSearchQuery, int pageNumber) throws FetcherException { + default Page performSearchPagedForTransformedQuery(String transformedQuery, int pageNumber) throws FetcherException { // ADR-0014 URL urlForQuery; try { - urlForQuery = getComplexQueryURL(complexSearchQuery, pageNumber); + urlForQuery = getURLForQuery(transformedQuery, pageNumber); } catch (URISyntaxException | MalformedURLException e) { throw new FetcherException("Search URI crafted from complex search query is malformed", e); } - return new Page<>(complexSearchQuery.toString(), pageNumber, getBibEntries(urlForQuery)); + return new Page<>(transformedQuery, pageNumber, getBibEntries(urlForQuery)); } private List getBibEntries(URL urlForQuery) throws FetcherException { @@ -40,33 +39,18 @@ private List getBibEntries(URL urlForQuery) throws FetcherException { /** * Constructs a URL based on the query, size and page number. * - * @param query the search query + * @param transformedQuery the search query * @param pageNumber the number of the page indexed from 0 */ - URL getURLForQuery(String query, int pageNumber) throws URISyntaxException, MalformedURLException; - - /** - * Constructs a URL based on the query, size and page number. - * - * @param complexSearchQuery the search query - * @param pageNumber the number of the page indexed from 0 - */ - default URL getComplexQueryURL(ComplexSearchQuery complexSearchQuery, int pageNumber) throws URISyntaxException, MalformedURLException { - return getURLForQuery(complexSearchQuery.toString(), pageNumber); - } - - @Override - default List performSearch(ComplexSearchQuery complexSearchQuery) throws FetcherException { - return SearchBasedParserFetcher.super.performSearch(complexSearchQuery); - } + URL getURLForQuery(String transformedQuery, int pageNumber) throws URISyntaxException, MalformedURLException; @Override - default URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { - return getURLForQuery(query, 0); + default URL getURLForQuery(String transformedQuery) throws URISyntaxException, MalformedURLException, FetcherException { + return getURLForQuery(transformedQuery, 0); } @Override - default URL getURLForQuery(ComplexSearchQuery query) throws URISyntaxException, MalformedURLException, FetcherException { - return getComplexQueryURL(query, 0); + default List performSearchForTransformedQuery(String transformedQuery) throws FetcherException { + return SearchBasedParserFetcher.super.performSearchForTransformedQuery(transformedQuery); } } diff --git a/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java b/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java index 5ca04372f11..58952f05e19 100644 --- a/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java +++ b/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java @@ -4,8 +4,9 @@ import java.util.List; import java.util.Optional; -import org.jabref.logic.importer.fetcher.ComplexSearchQuery; +import org.jabref.logic.JabRefException; import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; +import org.jabref.logic.importer.fetcher.transformators.DefaultQueryTransformer; import org.jabref.model.entry.BibEntry; /** @@ -17,26 +18,32 @@ public interface SearchBasedFetcher extends WebFetcher { /** * This method is used to send complex queries using fielded search. * - * @param complexSearchQuery the complex search query defining all fielded search parameters + * @param transformedQuery the search query defining all fielded search parameters * @return a list of {@link BibEntry}, which are matched by the query (may be empty) */ - List performSearch(ComplexSearchQuery complexSearchQuery) throws FetcherException; + List performSearchForTransformedQuery(String transformedQuery) throws FetcherException; /** * Looks for hits which are matched by the given free-text query. * - * @param complexSearchQuery query string that can be parsed into a complex search query + * @param searchQuery query string that can be parsed into a complex search query * @return a list of {@link BibEntry}, which are matched by the query (may be empty) */ - default List performSearch(String complexSearchQuery) throws FetcherException { - if (complexSearchQuery.isBlank()) { + default List performSearch(String searchQuery) throws FetcherException { + if (searchQuery.isBlank()) { return Collections.emptyList(); } - QueryParser queryParser = new QueryParser(); - Optional generatedQuery = queryParser.parseQueryStringIntoComplexQuery(complexSearchQuery); + Optional transformedQuery; + try { + transformedQuery = getQueryTransformer().parseQueryStringIntoComplexQuery(searchQuery); + } catch (JabRefException e) { + throw new FetcherException("Error occured during query transformation", e); + } // Otherwise just use query as a default term - return this.performSearch(generatedQuery.orElse(ComplexSearchQuery.builder().defaultFieldPhrase(complexSearchQuery).build())); + return this.performSearchForTransformedQuery(transformedQuery.orElse(searchQuery)); } - AbstractQueryTransformer getQueryTransformer(); + default AbstractQueryTransformer getQueryTransformer() { + return new DefaultQueryTransformer(); + } } diff --git a/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java b/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java index d0817ade6a0..c99e2b5712c 100644 --- a/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java +++ b/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java @@ -7,8 +7,8 @@ import java.net.URL; import java.util.List; +import org.jabref.logic.JabRefException; import org.jabref.logic.cleanup.Formatter; -import org.jabref.logic.importer.fetcher.ComplexSearchQuery; import org.jabref.model.entry.BibEntry; /** @@ -26,15 +26,15 @@ public interface SearchBasedParserFetcher extends SearchBasedFetcher { * This method is necessary as the performSearch method does not support certain URL parameters that are used for * fielded search, such as a title, author, or year parameter. * - * @param complexSearchQuery the search query defining all fielded search parameters + * @param transformedQuery the search query defining all fielded search parameters */ @Override - default List performSearch(ComplexSearchQuery complexSearchQuery) throws FetcherException { + default List performSearchForTransformedQuery(String transformedQuery) throws FetcherException { // ADR-0014 URL urlForQuery; try { - urlForQuery = getURLForQuery(complexSearchQuery); - } catch (URISyntaxException | MalformedURLException | FetcherException e) { + urlForQuery = getURLForQuery(transformedQuery); + } catch (URISyntaxException | MalformedURLException | JabRefException e) { throw new FetcherException("Search URI crafted from complex search query is malformed", e); } return getBibEntries(urlForQuery); @@ -52,11 +52,6 @@ private List getBibEntries(URL urlForQuery) throws FetcherException { } } - default URL getURLForQuery(ComplexSearchQuery query) throws URISyntaxException, MalformedURLException, FetcherException { - // Default implementation behaves as getURLForQuery treating complex query as plain string query - return this.getURLForQuery(query.toString()); - } - /** * Returns the parser used to convert the response to a list of {@link BibEntry}. */ @@ -65,9 +60,9 @@ default URL getURLForQuery(ComplexSearchQuery query) throws URISyntaxException, /** * Constructs a URL based on the query. * - * @param query the search query + * @param transformedQuery the search query */ - URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException; + URL getURLForQuery(String transformedQuery) throws URISyntaxException, MalformedURLException, FetcherException; /** * Performs a cleanup of the fetched entry. diff --git a/src/main/java/org/jabref/logic/importer/fetcher/ACMPortalFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/ACMPortalFetcher.java index e36f730366b..f28b561aff1 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/ACMPortalFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/ACMPortalFetcher.java @@ -42,9 +42,9 @@ private static String createQueryString(String query) { } @Override - public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(String transformedQuery) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder(SEARCH_URL); - uriBuilder.addParameter("query", createQueryString(query)); // Search all fields + uriBuilder.addParameter("query", createQueryString(transformedQuery)); // Search all fields uriBuilder.addParameter("within", "owners.owner=GUIDE"); // Search within the ACM Guide to Computing Literature (encompasses the ACM Full-Text Collection) uriBuilder.addParameter("expformat", "bibtex"); // BibTeX format return uriBuilder.build().toURL(); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/ArXiv.java b/src/main/java/org/jabref/logic/importer/fetcher/ArXiv.java index ea2e099fdb3..17fdfd391ac 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/ArXiv.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/ArXiv.java @@ -23,6 +23,8 @@ import org.jabref.logic.importer.IdFetcher; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.PagedSearchBasedFetcher; +import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; +import org.jabref.logic.importer.fetcher.transformators.ArXivQueryTransformer; import org.jabref.logic.util.io.XMLUtil; import org.jabref.logic.util.strings.StringSimilarity; import org.jabref.model.entry.BibEntry; @@ -252,24 +254,15 @@ public Optional getHelpPage() { /** * Constructs a complex query string using the field prefixes specified at https://arxiv.org/help/api/user-manual * - * @param complexSearchQuery the search query defining all fielded search parameters + * @param transformedQuery the search query defining all fielded search parameters * @return A list of entries matching the complex query */ @Override - public Page performSearchPaged(ComplexSearchQuery complexSearchQuery, int pageNumber) throws FetcherException { - List searchTerms = new ArrayList<>(); - complexSearchQuery.getAuthors().forEach(author -> searchTerms.add("au:" + author)); - complexSearchQuery.getTitlePhrases().forEach(title -> searchTerms.add("ti:" + title)); - complexSearchQuery.getAbstractPhrases().forEach(abstr -> searchTerms.add("abs:" + abstr)); - complexSearchQuery.getJournal().ifPresent(journal -> searchTerms.add("jr:" + journal)); - complexSearchQuery.getSingleYear().ifPresent(year -> searchTerms.add(year.toString())); - searchTerms.addAll(complexSearchQuery.getDefaultFieldPhrases()); - String complexQueryString = String.join(" AND ", searchTerms); - - List searchResult = searchForEntries(complexQueryString, pageNumber).stream() + public Page performSearchPagedForTransformedQuery(String transformedQuery, int pageNumber) throws FetcherException { + List searchResult = searchForEntries(transformedQuery, pageNumber).stream() .map((arXivEntry) -> arXivEntry.toBibEntry(importFormatPreferences.getKeywordSeparator())) .collect(Collectors.toList()); - return new Page<>(complexQueryString, pageNumber, searchResult); + return new Page<>(transformedQuery, pageNumber, searchResult); } @Override @@ -418,4 +411,9 @@ public BibEntry toBibEntry(Character keywordDelimiter) { return bibEntry; } } + + @Override + public AbstractQueryTransformer getQueryTransformer() { + return new ArXivQueryTransformer(); + } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java index a266418b351..33a81285135 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java @@ -79,13 +79,13 @@ public String getName() { } /** - * @param query query string, matching the apache solr format + * @param transformedQuery query string, matching the apache solr format * @return URL which points to a search request for given query */ @Override - public URL getURLForQuery(String query, int pageNumber) throws URISyntaxException, MalformedURLException { + public URL getURLForQuery(String transformedQuery, int pageNumber) throws URISyntaxException, MalformedURLException { URIBuilder builder = new URIBuilder(API_SEARCH_URL); - builder.addParameter("q", query); + builder.addParameter("q", transformedQuery); builder.addParameter("fl", "bibcode"); builder.addParameter("rows", String.valueOf(getPageSize())); builder.addParameter("start", String.valueOf(getPageSize() * pageNumber)); @@ -274,12 +274,12 @@ private List performSearchByIds(Collection identifiers) throws } @Override - public Page performSearchPaged(ComplexSearchQuery complexSearchQuery, int pageNumber) throws FetcherException { + public Page performSearchPagedForTransformedQuery(String transformedQuery, int pageNumber) throws FetcherException { try { // This is currently just interpreting the complex query as a default string query - List bibcodes = fetchBibcodes(getComplexQueryURL(complexSearchQuery, pageNumber)); + List bibcodes = fetchBibcodes(getURLForQuery(transformedQuery, pageNumber)); Collection results = performSearchByIds(bibcodes); - return new Page<>(complexSearchQuery.toString(), pageNumber, results); + return new Page<>(transformedQuery, pageNumber, results); } catch (URISyntaxException e) { throw new FetcherException("Search URI is malformed", e); } catch (IOException e) { diff --git a/src/main/java/org/jabref/logic/importer/fetcher/CiteSeer.java b/src/main/java/org/jabref/logic/importer/fetcher/CiteSeer.java index 1b51cf04b00..0b07b154c00 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/CiteSeer.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/CiteSeer.java @@ -44,10 +44,10 @@ public Optional getHelpPage() { } @Override - public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(String transformedQuery) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder("https://citeseer.ist.psu.edu/search"); uriBuilder.addParameter("sort", "rlv"); // Sort by relevance - uriBuilder.addParameter("q", query); // Query + uriBuilder.addParameter("q", transformedQuery); // Query uriBuilder.addParameter("t", "doc"); // Type: documents // uriBuilder.addParameter("start", "0"); // Start index (not supported at the moment) return uriBuilder.build().toURL(); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcher.java index 3bcbd26a2fb..96af1c7ea35 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcher.java @@ -33,9 +33,9 @@ public CollectionOfComputerScienceBibliographiesFetcher(ImportFormatPreferences } @Override - public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(String transformedQuery) throws URISyntaxException, MalformedURLException, FetcherException { return new URIBuilder(BASIC_SEARCH_URL) - .addParameter("query", query) + .addParameter("query", transformedQuery) .addParameter("sort", "score") .build() .toURL(); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcher.java index 3ce32ebeee6..ebb01e0ad51 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcher.java @@ -36,13 +36,23 @@ public CompositeSearchBasedFetcher(Set searchBasedFetchers, } @Override - public List performSearch(ComplexSearchQuery complexSearchQuery) { + public String getName() { + return "SearchAll"; + } + + @Override + public Optional getHelpPage() { + return Optional.empty(); + } + + @Override + public List performSearchForTransformedQuery(String transformedQuery) throws FetcherException { ImportCleanup cleanup = new ImportCleanup(BibDatabaseMode.BIBTEX); // All entries have to be converted into one format, this is necessary for the format conversion return fetchers.parallelStream() .flatMap(searchBasedFetcher -> { try { - return searchBasedFetcher.performSearch(complexSearchQuery).stream(); + return searchBasedFetcher.performSearchForTransformedQuery(transformedQuery).stream(); } catch (FetcherException e) { LOGGER.warn(String.format("%s API request failed", searchBasedFetcher.getName()), e); return Stream.empty(); @@ -52,14 +62,4 @@ public List performSearch(ComplexSearchQuery complexSearchQuery) { .map(cleanup::doPostCleanup) .collect(Collectors.toList()); } - - @Override - public String getName() { - return "SearchAll"; - } - - @Override - public Optional getHelpPage() { - return Optional.empty(); - } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/CrossRef.java b/src/main/java/org/jabref/logic/importer/fetcher/CrossRef.java index fc7603ecd5f..60a0d554167 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/CrossRef.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/CrossRef.java @@ -63,9 +63,9 @@ public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedUR } @Override - public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(String transformedQuery) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder(API_URL); - uriBuilder.addParameter("query", query); + uriBuilder.addParameter("query", transformedQuery); return uriBuilder.build().toURL(); } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/DBLPFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/DBLPFetcher.java index cb223ce061d..83504b4605e 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/DBLPFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/DBLPFetcher.java @@ -16,6 +16,8 @@ import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; +import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; +import org.jabref.logic.importer.fetcher.transformators.DBLPQueryTransformer; import org.jabref.logic.importer.fileformat.BibtexParser; import org.jabref.logic.layout.LayoutFormatterBasedFormatter; import org.jabref.logic.layout.format.RemoveLatexCommandsFormatter; @@ -42,9 +44,9 @@ public DBLPFetcher(ImportFormatPreferences importFormatPreferences) { } @Override - public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(String transformedQuery) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder(BASIC_SEARCH_URL); - uriBuilder.addParameter("q", query); + uriBuilder.addParameter("q", transformedQuery); uriBuilder.addParameter("h", String.valueOf(100)); // number of hits uriBuilder.addParameter("c", String.valueOf(0)); // no need for auto-completion uriBuilder.addParameter("f", String.valueOf(0)); // "from", index of first hit to download @@ -81,4 +83,9 @@ public String getName() { public Optional getHelpPage() { return Optional.of(HelpFile.FETCHER_DBLP); } + + @Override + public AbstractQueryTransformer getQueryTransformer() { + return new DBLPQueryTransformer(); + } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java index 39146f3f120..79f282c23b3 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java @@ -188,9 +188,9 @@ public Optional getHelpPage() { } @Override - public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(String transformedQuery) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder(SEARCH_URL); - DOAJFetcher.addPath(uriBuilder, query); + DOAJFetcher.addPath(uriBuilder, transformedQuery); // Number of results uriBuilder.addParameter("pageSize", "30"); // Page (not needed so far) diff --git a/src/main/java/org/jabref/logic/importer/fetcher/GoogleScholar.java b/src/main/java/org/jabref/logic/importer/fetcher/GoogleScholar.java index 58d6c8546b7..46f55c9b1e4 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/GoogleScholar.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/GoogleScholar.java @@ -19,6 +19,8 @@ import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.PagedSearchBasedFetcher; import org.jabref.logic.importer.ParserResult; +import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; +import org.jabref.logic.importer.fetcher.transformators.ScholarQueryTransformer; import org.jabref.logic.importer.fileformat.BibtexParser; import org.jabref.logic.l10n.Localization; import org.jabref.logic.net.URLDownload; @@ -127,16 +129,6 @@ public Optional getHelpPage() { return Optional.of(HelpFile.FETCHER_GOOGLE_SCHOLAR); } - private String constructComplexQueryString(ComplexSearchQuery complexSearchQuery) { - List searchTerms = new ArrayList<>(); - searchTerms.addAll(complexSearchQuery.getDefaultFieldPhrases()); - complexSearchQuery.getAuthors().forEach(author -> searchTerms.add("author:" + author)); - searchTerms.add("allintitle:" + String.join(" ", complexSearchQuery.getTitlePhrases())); - complexSearchQuery.getJournal().ifPresent(journal -> searchTerms.add("source:" + journal)); - // API automatically ANDs the terms - return String.join(" ", searchTerms); - } - private void addHitsFromQuery(List entryList, String queryURL) throws IOException, FetcherException { String content = new URLDownload(queryURL).asString(); @@ -185,24 +177,20 @@ private void obtainAndModifyCookie() throws FetcherException { } @Override - public Page performSearchPaged(ComplexSearchQuery complexSearchQuery, int pageNumber) throws FetcherException { + public Page performSearchPagedForTransformedQuery(String transformedQuery, int pageNumber) throws FetcherException { try { obtainAndModifyCookie(); List foundEntries = new ArrayList<>(10); - String complexQueryString = constructComplexQueryString(complexSearchQuery); URIBuilder uriBuilder = new URIBuilder(BASIC_SEARCH_URL); uriBuilder.addParameter("hl", "en"); uriBuilder.addParameter("btnG", "Search"); - uriBuilder.addParameter("q", complexQueryString); + uriBuilder.addParameter("q", transformedQuery); uriBuilder.addParameter("start", String.valueOf(pageNumber * getPageSize())); uriBuilder.addParameter("num", String.valueOf(getPageSize())); - complexSearchQuery.getFromYear().ifPresent(year -> uriBuilder.addParameter("as_ylo", year.toString())); - complexSearchQuery.getToYear().ifPresent(year -> uriBuilder.addParameter("as_yhi", year.toString())); - complexSearchQuery.getSingleYear().ifPresent(year -> { - uriBuilder.addParameter("as_ylo", year.toString()); - uriBuilder.addParameter("as_yhi", year.toString()); - }); + ScholarQueryTransformer transformer = ((ScholarQueryTransformer) getQueryTransformer()); + uriBuilder.addParameter("as_ylo", String.valueOf(transformer.getStartYear())); + uriBuilder.addParameter("as_yhi", String.valueOf(transformer.getEndYear())); try { addHitsFromQuery(foundEntries, uriBuilder.toString()); @@ -223,9 +211,14 @@ public Page performSearchPaged(ComplexSearchQuery complexSearchQuery, throw new FetcherException("Error while fetching from " + getName(), e); } } - return new Page<>(complexQueryString, pageNumber, foundEntries); + return new Page<>(transformedQuery, pageNumber, foundEntries); } catch (URISyntaxException e) { throw new FetcherException("Error while fetching from " + getName(), e); } } + + @Override + public AbstractQueryTransformer getQueryTransformer() { + return new ScholarQueryTransformer(); + } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java index 61218575668..b0e3cc5f8c9 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java @@ -66,13 +66,16 @@ private Optional parseBibToBibEntry(String bibtexString) { } @Override - public List performSearch(ComplexSearchQuery complexSearchQuery) throws FetcherException { - List bibEntries = null; - // This just treats the complex query like a normal string query until it it implemented correctly - String query = complexSearchQuery.toString(); + public String getName() { + return "GROBID"; + } + + @Override + public List performSearchForTransformedQuery(String transformedQuery) throws FetcherException { + List bibEntries; try { bibEntries = Arrays - .stream(query.split("\\r\\r+|\\n\\n+|\\r\\n(\\r\\n)+")) + .stream(transformedQuery.split("\\r\\r+|\\n\\n+|\\r\\n(\\r\\n)+")) .map(String::trim) .filter(str -> !str.isBlank()) .map(this::parseUsingGrobid) @@ -86,10 +89,4 @@ public List performSearch(ComplexSearchQuery complexSearchQuery) throw } return bibEntries; } - - @Override - public String getName() { - return "GROBID"; - } - } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/GvkFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/GvkFetcher.java index 20b4fe34630..5d3a660af4e 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/GvkFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/GvkFetcher.java @@ -5,16 +5,14 @@ import java.net.URL; import java.util.Arrays; import java.util.Collection; -import java.util.LinkedList; -import java.util.List; -import java.util.Objects; import java.util.Optional; -import java.util.stream.Collectors; import org.jabref.logic.help.HelpFile; import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; +import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; +import org.jabref.logic.importer.fetcher.transformators.GVKQueryTransformer; import org.jabref.logic.importer.fileformat.GvkParser; import org.apache.http.client.utils.URIBuilder; @@ -39,43 +37,12 @@ public Optional getHelpPage() { return Optional.of(HelpFile.FETCHER_GVK); } - private String getSearchQueryStringForComplexQuery(List queryList) { - String query = ""; - boolean lastWasNoKey = false; - - for (String key : queryList) { - if (searchKeys.contains(key)) { - if (lastWasNoKey) { - query = query + "and "; - } - query = query + "pica." + key + "="; - } else { - query = query + key + " "; - lastWasNoKey = true; - } - } - return query.trim(); - } - - protected String getSearchQueryString(String query) { - Objects.requireNonNull(query); - LinkedList queryList = new LinkedList<>(Arrays.asList(query.split("\\s"))); - - if (searchKeys.contains(queryList.get(0))) { - return getSearchQueryStringForComplexQuery(queryList); - } else { - // query as pica.all - return queryList.stream().collect(Collectors.joining(" ", "pica.all=", "")); - } - } - @Override - public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { - String gvkQuery = getSearchQueryString(query); + public URL getURLForQuery(String transformedQuery) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder(URL_PATTERN); uriBuilder.addParameter("version", "1.1"); uriBuilder.addParameter("operation", "searchRetrieve"); - uriBuilder.addParameter("query", gvkQuery); + uriBuilder.addParameter("query", transformedQuery); uriBuilder.addParameter("maximumRecords", "50"); uriBuilder.addParameter("recordSchema", "picaxml"); uriBuilder.addParameter("sortKeys", "Year,,1"); @@ -86,4 +53,9 @@ public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLE public Parser getParser() { return new GvkParser(); } + + @Override + public AbstractQueryTransformer getQueryTransformer() { + return new GVKQueryTransformer(); + } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java b/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java index d89223908a0..adc8da19cf1 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java @@ -19,6 +19,8 @@ import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.PagedSearchBasedParserFetcher; import org.jabref.logic.importer.Parser; +import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; +import org.jabref.logic.importer.fetcher.transformators.IEEEQueryTransformer; import org.jabref.logic.net.URLDownload; import org.jabref.logic.util.BuildInfo; import org.jabref.logic.util.OS; @@ -222,10 +224,10 @@ public Optional getHelpPage() { } @Override - public URL getURLForQuery(String query, int pageNumber) throws URISyntaxException, MalformedURLException { + public URL getURLForQuery(String transformedQuery, int pageNumber) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder("https://ieeexploreapi.ieee.org/api/v1/search/articles"); uriBuilder.addParameter("apikey", API_KEY); - uriBuilder.addParameter("querytext", query); + uriBuilder.addParameter("querytext", transformedQuery); uriBuilder.addParameter("max_records", String.valueOf(getPageSize())); // Starts to index at 1 for the first entry uriBuilder.addParameter("start_record", String.valueOf(getPageSize() * pageNumber) + 1); @@ -236,34 +238,7 @@ public URL getURLForQuery(String query, int pageNumber) throws URISyntaxExceptio } @Override - public URL getComplexQueryURL(ComplexSearchQuery complexSearchQuery, int pageNumber) throws URISyntaxException, MalformedURLException { - URIBuilder uriBuilder = new URIBuilder("https://ieeexploreapi.ieee.org/api/v1/search/articles"); - uriBuilder.addParameter("apikey", API_KEY); - uriBuilder.addParameter("max_records", String.valueOf(getPageSize())); - // Starts to index at 1 for the first entry - uriBuilder.addParameter("start_record", String.valueOf(getPageSize() * pageNumber) + 1); - - if (!complexSearchQuery.getDefaultFieldPhrases().isEmpty()) { - uriBuilder.addParameter("querytext", String.join(" AND ", complexSearchQuery.getDefaultFieldPhrases())); - } - if (!complexSearchQuery.getAuthors().isEmpty()) { - uriBuilder.addParameter("author", String.join(" AND ", complexSearchQuery.getAuthors())); - } - if (!complexSearchQuery.getAbstractPhrases().isEmpty()) { - uriBuilder.addParameter("abstract", String.join(" AND ", complexSearchQuery.getAbstractPhrases())); - } - if (!complexSearchQuery.getTitlePhrases().isEmpty()) { - uriBuilder.addParameter("article_title", String.join(" AND ", complexSearchQuery.getTitlePhrases())); - } - complexSearchQuery.getJournal().ifPresent(journalTitle -> uriBuilder.addParameter("publication_title", journalTitle)); - complexSearchQuery.getFromYear().map(String::valueOf).ifPresent(year -> uriBuilder.addParameter("start_year", year)); - complexSearchQuery.getToYear().map(String::valueOf).ifPresent(year -> uriBuilder.addParameter("end_year", year)); - complexSearchQuery.getSingleYear().map(String::valueOf).ifPresent(year -> { - uriBuilder.addParameter("start_year", year); - uriBuilder.addParameter("end_year", year); - }); - - URLDownload.bypassSSLVerification(); - return uriBuilder.build().toURL(); + public AbstractQueryTransformer getQueryTransformer() { + return new IEEEQueryTransformer(); } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/INSPIREFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/INSPIREFetcher.java index d7239dee240..457134fc59f 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/INSPIREFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/INSPIREFetcher.java @@ -48,9 +48,9 @@ public Optional getHelpPage() { } @Override - public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(String transformedQuery) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder(INSPIRE_HOST); - uriBuilder.addParameter("q", query); // Query + uriBuilder.addParameter("q", transformedQuery); // Query return uriBuilder.build().toURL(); } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/JstorFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/JstorFetcher.java index 6315e708c52..850fd896739 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/JstorFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/JstorFetcher.java @@ -47,44 +47,9 @@ public JstorFetcher(ImportFormatPreferences importFormatPreferences) { } @Override - public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException { + public URL getURLForQuery(String transformedQuery) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(SEARCH_HOST); - uriBuilder.addParameter("Query", query); - return uriBuilder.build().toURL(); - } - - @Override - public URL getURLForQuery(ComplexSearchQuery query) throws URISyntaxException, MalformedURLException, FetcherException { - URIBuilder uriBuilder = new URIBuilder(SEARCH_HOST); - StringBuilder stringBuilder = new StringBuilder(); - if (!query.getDefaultFieldPhrases().isEmpty()) { - stringBuilder.append(query.getDefaultFieldPhrases()); - } - if (!query.getAuthors().isEmpty()) { - for (String author : query.getAuthors()) { - stringBuilder.append("au:").append(author); - } - } - if (!query.getTitlePhrases().isEmpty()) { - for (String title : query.getTitlePhrases()) { - stringBuilder.append("ti:").append(title); - } - } - if (query.getJournal().isPresent()) { - stringBuilder.append("pt:").append(query.getJournal().get()); - } - if (query.getSingleYear().isPresent()) { - uriBuilder.addParameter("sd", String.valueOf(query.getSingleYear().get())); - uriBuilder.addParameter("ed", String.valueOf(query.getSingleYear().get())); - } - if (query.getFromYear().isPresent()) { - uriBuilder.addParameter("sd", String.valueOf(query.getFromYear().get())); - } - if (query.getToYear().isPresent()) { - uriBuilder.addParameter("ed", String.valueOf(query.getToYear().get())); - } - - uriBuilder.addParameter("Query", stringBuilder.toString()); + uriBuilder.addParameter("Query", transformedQuery); return uriBuilder.build().toURL(); } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/MathSciNet.java b/src/main/java/org/jabref/logic/importer/fetcher/MathSciNet.java index 3ff0bd243b8..626860819d3 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/MathSciNet.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/MathSciNet.java @@ -72,10 +72,10 @@ public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedUR } @Override - public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(String transformedQuery) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder("https://mathscinet.ams.org/mathscinet/search/publications.html"); uriBuilder.addParameter("pg7", "ALLF"); // search all fields - uriBuilder.addParameter("s7", query); // query + uriBuilder.addParameter("s7", transformedQuery); // query uriBuilder.addParameter("r", "1"); // start index uriBuilder.addParameter("extend", "1"); // should return up to 100 items (instead of default 10) uriBuilder.addParameter("fmt", "bibtex"); // BibTeX format diff --git a/src/main/java/org/jabref/logic/importer/fetcher/MedlineFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/MedlineFetcher.java index a4553bc876b..70fb08e1390 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/MedlineFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/MedlineFetcher.java @@ -157,33 +157,6 @@ public void doPostCleanup(BibEntry entry) { new FieldFormatterCleanup(StandardField.AUTHOR, new NormalizeNamesFormatter()).cleanup(entry); } - @Override - public List performSearch(ComplexSearchQuery complexSearchQuery) throws FetcherException { - List entryList; - String query = complexSearchQuery.toString(); - - if (query.isBlank()) { - return Collections.emptyList(); - } else { - // searching for pubmed ids matching the query - List idList = getPubMedIdsFromQuery(query); - - if (idList.isEmpty()) { - LOGGER.info("No results found."); - return Collections.emptyList(); - } - if (numberOfResultsFound > NUMBER_TO_FETCH) { - LOGGER.info( - numberOfResultsFound + " results found. Only 50 relevant results will be fetched by default."); - } - - // pass the list of ids to fetchMedline to download them. like a id fetcher for mutliple ids - entryList = fetchMedline(idList); - - return entryList; - } - } - private URL createSearchUrl(String query) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(SEARCH_URL); uriBuilder.addParameter("db", "pubmed"); @@ -221,4 +194,30 @@ private List fetchMedline(List ids) throws FetcherException { Localization.lang("Error while fetching from %0", "Medline"), e); } } + + @Override + public List performSearchForTransformedQuery(String transformedQuery) throws FetcherException { + List entryList; + + if (transformedQuery.isBlank()) { + return Collections.emptyList(); + } else { + // searching for pubmed ids matching the query + List idList = getPubMedIdsFromQuery(transformedQuery); + + if (idList.isEmpty()) { + LOGGER.info("No results found."); + return Collections.emptyList(); + } + if (numberOfResultsFound > NUMBER_TO_FETCH) { + LOGGER.info( + numberOfResultsFound + " results found. Only 50 relevant results will be fetched by default."); + } + + // pass the list of ids to fetchMedline to download them. like a id fetcher for mutliple ids + entryList = fetchMedline(idList); + + return entryList; + } + } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java index 1064a7f272e..8939108e275 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java @@ -13,6 +13,8 @@ import org.jabref.logic.help.HelpFile; import org.jabref.logic.importer.PagedSearchBasedParserFetcher; import org.jabref.logic.importer.Parser; +import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; +import org.jabref.logic.importer.fetcher.transformators.SpringerQueryTransformer; import org.jabref.logic.util.BuildInfo; import org.jabref.logic.util.OS; import org.jabref.model.entry.BibEntry; @@ -158,20 +160,15 @@ public Optional getHelpPage() { } @Override - public URL getURLForQuery(String query, int pageNumber) throws URISyntaxException, MalformedURLException { + public URL getURLForQuery(String transformedQuery, int pageNumber) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(API_URL); - uriBuilder.addParameter("q", query); // Search query + uriBuilder.addParameter("q", transformedQuery); // Search query uriBuilder.addParameter("api_key", API_KEY); // API key uriBuilder.addParameter("s", String.valueOf(getPageSize() * pageNumber + 1)); // Start entry, starts indexing at 1 uriBuilder.addParameter("p", String.valueOf(getPageSize())); // Page size return uriBuilder.build().toURL(); } - @Override - public URL getComplexQueryURL(ComplexSearchQuery complexSearchQuery, int pageNumber) throws URISyntaxException, MalformedURLException { - return getURLForQuery(constructComplexQueryString(complexSearchQuery), pageNumber); - } - private String constructComplexQueryString(ComplexSearchQuery complexSearchQuery) { List searchTerms = new ArrayList<>(); complexSearchQuery.getAuthors().forEach(author -> searchTerms.add("name:" + author)); @@ -202,4 +199,9 @@ public Parser getParser() { return entries; }; } + + @Override + public AbstractQueryTransformer getQueryTransformer() { + return new SpringerQueryTransformer(); + } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/ZbMATH.java b/src/main/java/org/jabref/logic/importer/fetcher/ZbMATH.java index 31105ede0ab..279231cd318 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/ZbMATH.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/ZbMATH.java @@ -13,6 +13,8 @@ import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; +import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; +import org.jabref.logic.importer.fetcher.transformators.ZbMathQueryTransformer; import org.jabref.logic.importer.fileformat.BibtexParser; import org.jabref.logic.net.URLDownload; import org.jabref.model.entry.BibEntry; @@ -50,9 +52,9 @@ public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedUR } */ @Override - public URL getURLForQuery(String query) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(String transformedQuery) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder("https://zbmath.org/bibtexoutput/"); - uriBuilder.addParameter("q", query); // search all fields + uriBuilder.addParameter("q", transformedQuery); // search all fields uriBuilder.addParameter("start", "0"); // start index uriBuilder.addParameter("count", "200"); // should return up to 200 items (instead of default 100) @@ -86,4 +88,9 @@ public void doPostCleanup(BibEntry entry) { new FieldFormatterCleanup(StandardField.JOURNAL, new RemoveBracesFormatter()).cleanup(entry); new FieldFormatterCleanup(StandardField.TITLE, new RemoveBracesFormatter()).cleanup(entry); } + + @Override + public AbstractQueryTransformer getQueryTransformer() { + return new ZbMathQueryTransformer(); + } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformators/DefaultQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformators/DefaultQueryTransformer.java index 445949e6d36..b99e327269c 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/transformators/DefaultQueryTransformer.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformators/DefaultQueryTransformer.java @@ -1,2 +1,49 @@ -package org.jabref.logic.importer.fetcher.transformators;public class DefaultQueryTransformer { +package org.jabref.logic.importer.fetcher.transformators; + +public class DefaultQueryTransformer extends AbstractQueryTransformer { + + @Override + protected String getLogicalAndOperator() { + return " "; + } + + @Override + protected String getLogicalOrOperator() { + return " "; + } + + @Override + protected String getLogicalNotOperator() { + return ""; + } + + @Override + protected String handleAuthor(String author) { + return author; + } + + @Override + protected String handleTitle(String title) { + return title; + } + + @Override + protected String handleJournal(String journalTitle) { + return journalTitle; + } + + @Override + protected String handleYear(String year) { + return year; + } + + @Override + protected String handleYearRange(String yearRange) { + return yearRange; + } + + @Override + protected String handleUnFieldedTerm(String term) { + return term; + } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformators/GVKQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformators/GVKQueryTransformer.java index 9cbda4786cb..a35fa0fe68e 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/transformators/GVKQueryTransformer.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformators/GVKQueryTransformer.java @@ -8,8 +8,7 @@ public class GVKQueryTransformer extends AbstractQueryTransformer { @Override protected String getLogicalAndOperator() { - // GVK defaults to AND - return " "; + return " and "; } @Override @@ -26,24 +25,24 @@ protected String getLogicalNotOperator() { @Override protected String handleAuthor(String author) { - return String.format("per:\"%s\"", author); + return String.format("pica.per=\"%s\"", author); } @Override protected String handleTitle(String title) { - return String.format("tit:\"%s\"", title); + return String.format("pica.tit=\"%s\"", title); } @Override protected String handleJournal(String journalTitle) { // zti means "Zeitschrift", does not search for conferences (kon:) - return String.format("zti:\"%s\"", journalTitle); + return String.format("pica.zti=\"%s\"", journalTitle); } @Override protected String handleYear(String year) { // ver means Veröffentlichungsangaben - return "ver:" + year; + return "pica.ver=" + year; } @Override @@ -56,6 +55,6 @@ protected String handleYearRange(String yearRange) { protected String handleUnFieldedTerm(String term) { // all does not search in full-text // Other option is txt: but this does not search in meta data - return String.format("all:\"%s\"", term); + return String.format("pica.all=\"%s\"", term); } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformators/JstorQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformators/JstorQueryTransformer.java new file mode 100644 index 00000000000..1c7be5db728 --- /dev/null +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformators/JstorQueryTransformer.java @@ -0,0 +1,49 @@ +package org.jabref.logic.importer.fetcher.transformators; + +public class JstorQueryTransformer extends AbstractQueryTransformer { + @Override + protected String getLogicalAndOperator() { + return " AND "; + } + + @Override + protected String getLogicalOrOperator() { + return " OR "; + } + + @Override + protected String getLogicalNotOperator() { + return "NOT "; + } + + @Override + protected String handleAuthor(String author) { + return String.format("au:\"%s\"", author); + } + + @Override + protected String handleTitle(String title) { + return String.format("ti:\"%s\"", title); + } + + @Override + protected String handleJournal(String journalTitle) { + return String.format("pt:\"%s\"", journalTitle); + } + + @Override + protected String handleYear(String year) { + return "sd:" + year + getLogicalAndOperator() + "ed: " + year; + } + + @Override + protected String handleYearRange(String yearRange) { + String[] split = yearRange.split("-"); + return "sd:" + split[0] + getLogicalAndOperator() + "ed:" + split[1]; + } + + @Override + protected String handleUnFieldedTerm(String term) { + return String.format("\"%s\"", term); + } +} diff --git a/src/test/java/org/jabref/logic/importer/fetcher/GvkFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/GvkFetcherTest.java index d1af17c5be0..d9b94f99ca2 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/GvkFetcherTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/GvkFetcherTest.java @@ -63,13 +63,6 @@ public void testGetName() { assertEquals("GVK", fetcher.getName()); } - @Test - public void simpleSearchQueryStringCorrect() { - String query = "java jdk"; - String result = fetcher.getSearchQueryString(query); - assertEquals("pica.all=java jdk", result); - } - @Test public void simpleSearchQueryURLCorrect() throws MalformedURLException, URISyntaxException, FetcherException { String query = "java jdk"; @@ -77,13 +70,6 @@ public void simpleSearchQueryURLCorrect() throws MalformedURLException, URISynta assertEquals("http://sru.gbv.de/gvk?version=1.1&operation=searchRetrieve&query=pica.all%3Djava+jdk&maximumRecords=50&recordSchema=picaxml&sortKeys=Year%2C%2C1", url.toString()); } - @Test - public void complexSearchQueryStringCorrect() { - String query = "kon java tit jdk"; - String result = fetcher.getSearchQueryString(query); - assertEquals("pica.kon=java and pica.tit=jdk", result); - } - @Test public void complexSearchQueryURLCorrect() throws MalformedURLException, URISyntaxException, FetcherException { String query = "kon java tit jdk"; diff --git a/src/test/java/org/jabref/logic/importer/fetcher/SearchBasedFetcherCapabilityTest.java b/src/test/java/org/jabref/logic/importer/fetcher/SearchBasedFetcherCapabilityTest.java index c31bd348b0c..a9c63797e10 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/SearchBasedFetcherCapabilityTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/SearchBasedFetcherCapabilityTest.java @@ -3,6 +3,7 @@ import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.StringJoiner; import java.util.stream.Collectors; import org.jabref.logic.importer.ImportCleanup; @@ -31,10 +32,11 @@ interface SearchBasedFetcherCapabilityTest { */ @Test default void supportsAuthorSearch() throws Exception { - ComplexSearchQuery.ComplexSearchQueryBuilder builder = ComplexSearchQuery.builder(); - getTestAuthors().forEach(builder::author); + StringJoiner queryBuilder = new StringJoiner(" AND author:", "author:", ""); + getTestAuthors().forEach(queryBuilder::add); - List result = getFetcher().performSearch(builder.build()); + + List result = getFetcher().performSearch(queryBuilder.toString()); new ImportCleanup(BibDatabaseMode.BIBTEX).doPostCleanup(result); assertFalse(result.isEmpty()); @@ -51,12 +53,7 @@ default void supportsAuthorSearch() throws Exception { */ @Test default void supportsYearSearch() throws Exception { - ComplexSearchQuery complexSearchQuery = ComplexSearchQuery - .builder() - .singleYear(getTestYear()) - .build(); - - List result = getFetcher().performSearch(complexSearchQuery); + List result = getFetcher().performSearch("year:" + getTestYear()); new ImportCleanup(BibDatabaseMode.BIBTEX).doPostCleanup(result); List differentYearsInResult = result.stream() .map(bibEntry -> bibEntry.getField(StandardField.YEAR)) @@ -73,11 +70,9 @@ default void supportsYearSearch() throws Exception { */ @Test default void supportsYearRangeSearch() throws Exception { - ComplexSearchQuery.ComplexSearchQueryBuilder builder = ComplexSearchQuery.builder(); List yearsInYearRange = List.of("2018", "2019", "2020"); - builder.fromYearAndToYear(2018, 2020); - List result = getFetcher().performSearch(builder.build()); + List result = getFetcher().performSearch("year-range:2018-2020"); new ImportCleanup(BibDatabaseMode.BIBTEX).doPostCleanup(result); List differentYearsInResult = result.stream() .map(bibEntry -> bibEntry.getField(StandardField.YEAR)) @@ -94,9 +89,7 @@ default void supportsYearRangeSearch() throws Exception { */ @Test default void supportsJournalSearch() throws Exception { - ComplexSearchQuery.ComplexSearchQueryBuilder builder = ComplexSearchQuery.builder(); - builder.journal(getTestJournal()); - List result = getFetcher().performSearch(builder.build()); + List result = getFetcher().performSearch("journal:" + getTestJournal()); new ImportCleanup(BibDatabaseMode.BIBTEX).doPostCleanup(result); assertFalse(result.isEmpty()); diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/GVKQueryTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/GVKQueryTransformerTest.java index 8eaa095b61e..86a6934830d 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/transformators/GVKQueryTransformerTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/GVKQueryTransformerTest.java @@ -15,22 +15,22 @@ public AbstractQueryTransformer getTransformator() { @Override public String getAuthorPrefix() { - return "per:"; + return "pica.per="; } @Override public String getUnFieldedPrefix() { - return "all:"; + return "pica.all="; } @Override public String getJournalPrefix() { - return "zti:"; + return "pica.zti="; } @Override public String getTitlePrefix() { - return "tit:"; + return "pica.tit="; } @Override diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/JstorQueryTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/JstorQueryTransformerTest.java new file mode 100644 index 00000000000..aeeaa5ac625 --- /dev/null +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/JstorQueryTransformerTest.java @@ -0,0 +1,41 @@ +package org.jabref.logic.importer.fetcher.transformators; + +import static org.junit.jupiter.api.Assertions.*; + +class JstorQueryTransformerTest implements InfixTransformerTest { + + @Override + public AbstractQueryTransformer getTransformator() { + return new JstorQueryTransformer(); + } + + @Override + public String getAuthorPrefix() { + return "au:"; + } + + @Override + public String getUnFieldedPrefix() { + return ""; + } + + @Override + public String getJournalPrefix() { + return "pt:"; + } + + @Override + public String getTitlePrefix() { + return "ti:"; + } + + @Override + public void convertYearField() throws Exception { + + } + + @Override + public void convertYearRangeField() throws Exception { + + } +} From 6d6f74d2a6bbcec8be46e5f2fa3b05009d358238 Mon Sep 17 00:00:00 2001 From: Dominik Voigt Date: Wed, 13 Jan 2021 17:04:24 +0100 Subject: [PATCH 11/24] Add transformer for certain cases to method signature. Fix IEEE tests. Extend IEEE capabilities. Create workarounds for some IEEE bugs --- .../importer/PagedSearchBasedFetcher.java | 12 ++--- .../PagedSearchBasedParserFetcher.java | 15 ++++--- .../logic/importer/SearchBasedFetcher.java | 8 ++-- .../importer/SearchBasedParserFetcher.java | 8 ++-- .../importer/fetcher/ACMPortalFetcher.java | 3 +- .../jabref/logic/importer/fetcher/ArXiv.java | 17 +++++-- .../fetcher/AstrophysicsDataSystem.java | 7 +-- .../logic/importer/fetcher/CiteSeer.java | 3 +- ...fComputerScienceBibliographiesFetcher.java | 3 +- .../fetcher/CompositeSearchBasedFetcher.java | 5 ++- .../logic/importer/fetcher/CrossRef.java | 3 +- .../logic/importer/fetcher/DBLPFetcher.java | 2 +- .../logic/importer/fetcher/DOAJFetcher.java | 3 +- .../logic/importer/fetcher/GoogleScholar.java | 8 ++-- .../fetcher/GrobidCitationFetcher.java | 3 +- .../logic/importer/fetcher/GvkFetcher.java | 2 +- .../jabref/logic/importer/fetcher/IEEE.java | 23 +++++++++- .../importer/fetcher/INSPIREFetcher.java | 3 +- .../logic/importer/fetcher/JstorFetcher.java | 3 +- .../logic/importer/fetcher/MathSciNet.java | 3 +- .../importer/fetcher/MedlineFetcher.java | 3 +- .../importer/fetcher/SpringerFetcher.java | 2 +- .../jabref/logic/importer/fetcher/ZbMATH.java | 2 +- .../transformators/ArXivQueryTransformer.java | 12 ++--- .../transformators/IEEEQueryTransformer.java | 45 +++++++++++++++---- .../ScholarQueryTransformer.java | 8 ++-- ...puterScienceBibliographiesFetcherTest.java | 3 +- .../CompositeSearchBasedFetcherTest.java | 2 + .../importer/fetcher/GvkFetcherTest.java | 5 ++- .../logic/importer/fetcher/IEEETest.java | 29 ++++++------ .../SearchBasedFetcherCapabilityTest.java | 4 +- .../org/jabref/model/entry/CrossrefTest.java | 2 + 32 files changed, 166 insertions(+), 85 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java b/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java index 43f6c666028..1998db13a69 100644 --- a/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java +++ b/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java @@ -6,6 +6,7 @@ import java.util.Optional; import org.jabref.logic.JabRefException; +import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; import org.jabref.model.entry.BibEntry; import org.jabref.model.paging.Page; @@ -16,7 +17,7 @@ public interface PagedSearchBasedFetcher extends SearchBasedFetcher { * @param pageNumber requested site number indexed from 0 * @return Page with search results */ - Page performSearchPagedForTransformedQuery(String transformedQuery, int pageNumber) throws FetcherException; + Page performSearchPagedForTransformedQuery(String transformedQuery, int pageNumber, AbstractQueryTransformer transformer) throws FetcherException; /** * @param searchQuery query string that can be parsed into a complex search query @@ -27,9 +28,10 @@ default Page performSearchPaged(String searchQuery, int pageNumber) th if (searchQuery.isBlank()) { return new Page<>(searchQuery, pageNumber, Collections.emptyList()); } - Optional transformedQuery = getQueryTransformer().parseQueryStringIntoComplexQuery(searchQuery); + AbstractQueryTransformer transformer = getQueryTransformer(); + Optional transformedQuery = transformer.parseQueryStringIntoComplexQuery(searchQuery); // Otherwise just use query as a default term - return this.performSearchPagedForTransformedQuery(transformedQuery.orElse(searchQuery), pageNumber); + return this.performSearchPagedForTransformedQuery(transformedQuery.orElse(""), pageNumber, transformer); } /** @@ -40,7 +42,7 @@ default int getPageSize() { } @Override - default List performSearchForTransformedQuery(String transformedQuery) throws FetcherException { - return new ArrayList<>(performSearchPagedForTransformedQuery(transformedQuery, 0).getContent()); + default List performSearchForTransformedQuery(String transformedQuery, AbstractQueryTransformer transformer) throws FetcherException { + return new ArrayList<>(performSearchPagedForTransformedQuery(transformedQuery, 0, transformer).getContent()); } } diff --git a/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java b/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java index de1863807d8..bc2384b9ba9 100644 --- a/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java +++ b/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java @@ -7,17 +7,18 @@ import java.net.URL; import java.util.List; +import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; import org.jabref.model.entry.BibEntry; import org.jabref.model.paging.Page; public interface PagedSearchBasedParserFetcher extends SearchBasedParserFetcher, PagedSearchBasedFetcher { @Override - default Page performSearchPagedForTransformedQuery(String transformedQuery, int pageNumber) throws FetcherException { + default Page performSearchPagedForTransformedQuery(String transformedQuery, int pageNumber, AbstractQueryTransformer transformer) throws FetcherException { // ADR-0014 URL urlForQuery; try { - urlForQuery = getURLForQuery(transformedQuery, pageNumber); + urlForQuery = getURLForQuery(transformedQuery, pageNumber, transformer); } catch (URISyntaxException | MalformedURLException e) { throw new FetcherException("Search URI crafted from complex search query is malformed", e); } @@ -42,15 +43,15 @@ private List getBibEntries(URL urlForQuery) throws FetcherException { * @param transformedQuery the search query * @param pageNumber the number of the page indexed from 0 */ - URL getURLForQuery(String transformedQuery, int pageNumber) throws URISyntaxException, MalformedURLException; + URL getURLForQuery(String transformedQuery, int pageNumber, AbstractQueryTransformer transformer) throws URISyntaxException, MalformedURLException; @Override - default URL getURLForQuery(String transformedQuery) throws URISyntaxException, MalformedURLException, FetcherException { - return getURLForQuery(transformedQuery, 0); + default URL getURLForQuery(String transformedQuery, AbstractQueryTransformer transformer) throws URISyntaxException, MalformedURLException, FetcherException { + return getURLForQuery(transformedQuery, 0, transformer); } @Override - default List performSearchForTransformedQuery(String transformedQuery) throws FetcherException { - return SearchBasedParserFetcher.super.performSearchForTransformedQuery(transformedQuery); + default List performSearchForTransformedQuery(String transformedQuery, AbstractQueryTransformer transformer) throws FetcherException { + return SearchBasedParserFetcher.super.performSearchForTransformedQuery(transformedQuery, transformer); } } diff --git a/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java b/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java index 58952f05e19..1cc2f0bdd90 100644 --- a/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java +++ b/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java @@ -19,9 +19,10 @@ public interface SearchBasedFetcher extends WebFetcher { * This method is used to send complex queries using fielded search. * * @param transformedQuery the search query defining all fielded search parameters + * @param transformer transformer might be required to extract some parsing information * @return a list of {@link BibEntry}, which are matched by the query (may be empty) */ - List performSearchForTransformedQuery(String transformedQuery) throws FetcherException; + List performSearchForTransformedQuery(String transformedQuery, AbstractQueryTransformer transformer) throws FetcherException; /** * Looks for hits which are matched by the given free-text query. @@ -30,17 +31,18 @@ public interface SearchBasedFetcher extends WebFetcher { * @return a list of {@link BibEntry}, which are matched by the query (may be empty) */ default List performSearch(String searchQuery) throws FetcherException { + AbstractQueryTransformer transformer = getQueryTransformer(); if (searchQuery.isBlank()) { return Collections.emptyList(); } Optional transformedQuery; try { - transformedQuery = getQueryTransformer().parseQueryStringIntoComplexQuery(searchQuery); + transformedQuery = transformer.parseQueryStringIntoComplexQuery(searchQuery); } catch (JabRefException e) { throw new FetcherException("Error occured during query transformation", e); } // Otherwise just use query as a default term - return this.performSearchForTransformedQuery(transformedQuery.orElse(searchQuery)); + return this.performSearchForTransformedQuery(transformedQuery.orElse(""), transformer); } default AbstractQueryTransformer getQueryTransformer() { diff --git a/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java b/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java index c99e2b5712c..055eb4ab403 100644 --- a/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java +++ b/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java @@ -9,6 +9,7 @@ import org.jabref.logic.JabRefException; import org.jabref.logic.cleanup.Formatter; +import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; import org.jabref.model.entry.BibEntry; /** @@ -27,13 +28,14 @@ public interface SearchBasedParserFetcher extends SearchBasedFetcher { * fielded search, such as a title, author, or year parameter. * * @param transformedQuery the search query defining all fielded search parameters + * @param transformer */ @Override - default List performSearchForTransformedQuery(String transformedQuery) throws FetcherException { + default List performSearchForTransformedQuery(String transformedQuery, AbstractQueryTransformer transformer) throws FetcherException { // ADR-0014 URL urlForQuery; try { - urlForQuery = getURLForQuery(transformedQuery); + urlForQuery = getURLForQuery(transformedQuery, transformer); } catch (URISyntaxException | MalformedURLException | JabRefException e) { throw new FetcherException("Search URI crafted from complex search query is malformed", e); } @@ -62,7 +64,7 @@ private List getBibEntries(URL urlForQuery) throws FetcherException { * * @param transformedQuery the search query */ - URL getURLForQuery(String transformedQuery) throws URISyntaxException, MalformedURLException, FetcherException; + URL getURLForQuery(String transformedQuery, AbstractQueryTransformer transformer) throws URISyntaxException, MalformedURLException, FetcherException; /** * Performs a cleanup of the fetched entry. diff --git a/src/main/java/org/jabref/logic/importer/fetcher/ACMPortalFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/ACMPortalFetcher.java index f28b561aff1..5319b1a873f 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/ACMPortalFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/ACMPortalFetcher.java @@ -11,6 +11,7 @@ import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; +import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; import org.jabref.logic.importer.fileformat.BibtexParser; import org.jabref.model.util.DummyFileUpdateMonitor; @@ -42,7 +43,7 @@ private static String createQueryString(String query) { } @Override - public URL getURLForQuery(String transformedQuery) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(String transformedQuery, AbstractQueryTransformer transformer) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder(SEARCH_URL); uriBuilder.addParameter("query", createQueryString(transformedQuery)); // Search all fields uriBuilder.addParameter("within", "owners.owner=GUIDE"); // Search within the ACM Guide to Computing Literature (encompasses the ACM Full-Text Collection) diff --git a/src/main/java/org/jabref/logic/importer/fetcher/ArXiv.java b/src/main/java/org/jabref/logic/importer/fetcher/ArXiv.java index 17fdfd391ac..8e53eb02b6d 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/ArXiv.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/ArXiv.java @@ -258,11 +258,20 @@ public Optional getHelpPage() { * @return A list of entries matching the complex query */ @Override - public Page performSearchPagedForTransformedQuery(String transformedQuery, int pageNumber) throws FetcherException { + public Page performSearchPagedForTransformedQuery(String transformedQuery, int pageNumber, AbstractQueryTransformer transformer) throws FetcherException { List searchResult = searchForEntries(transformedQuery, pageNumber).stream() - .map((arXivEntry) -> arXivEntry.toBibEntry(importFormatPreferences.getKeywordSeparator())) - .collect(Collectors.toList()); - return new Page<>(transformedQuery, pageNumber, searchResult); + .map((arXivEntry) -> arXivEntry.toBibEntry(importFormatPreferences.getKeywordSeparator())) + .collect(Collectors.toList()); + return new Page<>(transformedQuery, pageNumber, filterYears(searchResult, transformer)); + } + + private List filterYears(List searchResult, AbstractQueryTransformer transformer) { + ArXivQueryTransformer arXivQueryTransformer = ((ArXivQueryTransformer) transformer); + return searchResult.stream() + .filter(entry -> entry.getField(StandardField.YEAR).isPresent()) + .filter(entry -> arXivQueryTransformer.getEndYear() == Integer.MAX_VALUE || Integer.parseInt(entry.getField(StandardField.YEAR).get()) <= arXivQueryTransformer.getEndYear()) + .filter(entry -> arXivQueryTransformer.getStartYear() == Integer.MIN_VALUE ||Integer.parseInt(entry.getField(StandardField.YEAR).get()) >= arXivQueryTransformer.getStartYear()) + .collect(Collectors.toList()); } @Override diff --git a/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java index 33a81285135..de41ef187c2 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java @@ -27,6 +27,7 @@ import org.jabref.logic.importer.PagedSearchBasedParserFetcher; import org.jabref.logic.importer.ParseException; import org.jabref.logic.importer.Parser; +import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; import org.jabref.logic.importer.fileformat.BibtexParser; import org.jabref.logic.net.URLDownload; import org.jabref.logic.util.BuildInfo; @@ -83,7 +84,7 @@ public String getName() { * @return URL which points to a search request for given query */ @Override - public URL getURLForQuery(String transformedQuery, int pageNumber) throws URISyntaxException, MalformedURLException { + public URL getURLForQuery(String transformedQuery, int pageNumber, AbstractQueryTransformer transformer) throws URISyntaxException, MalformedURLException { URIBuilder builder = new URIBuilder(API_SEARCH_URL); builder.addParameter("q", transformedQuery); builder.addParameter("fl", "bibcode"); @@ -274,10 +275,10 @@ private List performSearchByIds(Collection identifiers) throws } @Override - public Page performSearchPagedForTransformedQuery(String transformedQuery, int pageNumber) throws FetcherException { + public Page performSearchPagedForTransformedQuery(String transformedQuery, int pageNumber, AbstractQueryTransformer transformer) throws FetcherException { try { // This is currently just interpreting the complex query as a default string query - List bibcodes = fetchBibcodes(getURLForQuery(transformedQuery, pageNumber)); + List bibcodes = fetchBibcodes(getURLForQuery(transformedQuery, pageNumber, transformer)); Collection results = performSearchByIds(bibcodes); return new Page<>(transformedQuery, pageNumber, results); } catch (URISyntaxException e) { diff --git a/src/main/java/org/jabref/logic/importer/fetcher/CiteSeer.java b/src/main/java/org/jabref/logic/importer/fetcher/CiteSeer.java index 0b07b154c00..41af880cf20 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/CiteSeer.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/CiteSeer.java @@ -20,6 +20,7 @@ import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; +import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; import org.jabref.logic.importer.fileformat.CoinsParser; import org.jabref.logic.util.OS; import org.jabref.model.entry.BibEntry; @@ -44,7 +45,7 @@ public Optional getHelpPage() { } @Override - public URL getURLForQuery(String transformedQuery) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(String transformedQuery, AbstractQueryTransformer transformer) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder("https://citeseer.ist.psu.edu/search"); uriBuilder.addParameter("sort", "rlv"); // Sort by relevance uriBuilder.addParameter("q", transformedQuery); // Query diff --git a/src/main/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcher.java index 96af1c7ea35..757fb212221 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcher.java @@ -14,6 +14,7 @@ import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; +import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.Field; import org.jabref.model.entry.field.FieldFactory; @@ -33,7 +34,7 @@ public CollectionOfComputerScienceBibliographiesFetcher(ImportFormatPreferences } @Override - public URL getURLForQuery(String transformedQuery) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(String transformedQuery, AbstractQueryTransformer transformer) throws URISyntaxException, MalformedURLException, FetcherException { return new URIBuilder(BASIC_SEARCH_URL) .addParameter("query", transformedQuery) .addParameter("sort", "score") diff --git a/src/main/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcher.java index ebb01e0ad51..b39bce1463e 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcher.java @@ -10,6 +10,7 @@ import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.ImportCleanup; import org.jabref.logic.importer.SearchBasedFetcher; +import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; import org.jabref.model.database.BibDatabaseMode; import org.jabref.model.entry.BibEntry; @@ -46,13 +47,13 @@ public Optional getHelpPage() { } @Override - public List performSearchForTransformedQuery(String transformedQuery) throws FetcherException { + public List performSearchForTransformedQuery(String transformedQuery, AbstractQueryTransformer transformer) throws FetcherException { ImportCleanup cleanup = new ImportCleanup(BibDatabaseMode.BIBTEX); // All entries have to be converted into one format, this is necessary for the format conversion return fetchers.parallelStream() .flatMap(searchBasedFetcher -> { try { - return searchBasedFetcher.performSearchForTransformedQuery(transformedQuery).stream(); + return searchBasedFetcher.performSearch(transformedQuery).stream(); } catch (FetcherException e) { LOGGER.warn(String.format("%s API request failed", searchBasedFetcher.getName()), e); return Stream.empty(); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/CrossRef.java b/src/main/java/org/jabref/logic/importer/fetcher/CrossRef.java index 60a0d554167..55014ee0570 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/CrossRef.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/CrossRef.java @@ -18,6 +18,7 @@ import org.jabref.logic.importer.ParseException; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; +import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; import org.jabref.logic.importer.util.JsonReader; import org.jabref.logic.util.strings.StringSimilarity; import org.jabref.model.entry.AuthorList; @@ -63,7 +64,7 @@ public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedUR } @Override - public URL getURLForQuery(String transformedQuery) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(String transformedQuery, AbstractQueryTransformer transformer) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder(API_URL); uriBuilder.addParameter("query", transformedQuery); return uriBuilder.build().toURL(); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/DBLPFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/DBLPFetcher.java index 83504b4605e..75e95789da9 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/DBLPFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/DBLPFetcher.java @@ -44,7 +44,7 @@ public DBLPFetcher(ImportFormatPreferences importFormatPreferences) { } @Override - public URL getURLForQuery(String transformedQuery) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(String transformedQuery, AbstractQueryTransformer transformer) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder(BASIC_SEARCH_URL); uriBuilder.addParameter("q", transformedQuery); uriBuilder.addParameter("h", String.valueOf(100)); // number of hits diff --git a/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java index 79f282c23b3..cbfa2c728cf 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java @@ -16,6 +16,7 @@ import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; +import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; import org.jabref.logic.util.OS; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.Field; @@ -188,7 +189,7 @@ public Optional getHelpPage() { } @Override - public URL getURLForQuery(String transformedQuery) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(String transformedQuery, AbstractQueryTransformer transformer) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder(SEARCH_URL); DOAJFetcher.addPath(uriBuilder, transformedQuery); // Number of results diff --git a/src/main/java/org/jabref/logic/importer/fetcher/GoogleScholar.java b/src/main/java/org/jabref/logic/importer/fetcher/GoogleScholar.java index 46f55c9b1e4..895a43acad9 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/GoogleScholar.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/GoogleScholar.java @@ -177,7 +177,7 @@ private void obtainAndModifyCookie() throws FetcherException { } @Override - public Page performSearchPagedForTransformedQuery(String transformedQuery, int pageNumber) throws FetcherException { + public Page performSearchPagedForTransformedQuery(String transformedQuery, int pageNumber, AbstractQueryTransformer transformer) throws FetcherException { try { obtainAndModifyCookie(); List foundEntries = new ArrayList<>(10); @@ -188,9 +188,9 @@ public Page performSearchPagedForTransformedQuery(String transformedQu uriBuilder.addParameter("q", transformedQuery); uriBuilder.addParameter("start", String.valueOf(pageNumber * getPageSize())); uriBuilder.addParameter("num", String.valueOf(getPageSize())); - ScholarQueryTransformer transformer = ((ScholarQueryTransformer) getQueryTransformer()); - uriBuilder.addParameter("as_ylo", String.valueOf(transformer.getStartYear())); - uriBuilder.addParameter("as_yhi", String.valueOf(transformer.getEndYear())); + ScholarQueryTransformer scholarQueryTransformer = ((ScholarQueryTransformer) transformer); + uriBuilder.addParameter("as_ylo", String.valueOf(scholarQueryTransformer.getStartYear())); + uriBuilder.addParameter("as_yhi", String.valueOf(scholarQueryTransformer.getEndYear())); try { addHitsFromQuery(foundEntries, uriBuilder.toString()); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java index b0e3cc5f8c9..77ef4f98768 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java @@ -11,6 +11,7 @@ import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.ParseException; import org.jabref.logic.importer.SearchBasedFetcher; +import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; import org.jabref.logic.importer.fileformat.BibtexParser; import org.jabref.logic.importer.util.GrobidService; import org.jabref.model.entry.BibEntry; @@ -71,7 +72,7 @@ public String getName() { } @Override - public List performSearchForTransformedQuery(String transformedQuery) throws FetcherException { + public List performSearchForTransformedQuery(String transformedQuery, AbstractQueryTransformer transformer) throws FetcherException { List bibEntries; try { bibEntries = Arrays diff --git a/src/main/java/org/jabref/logic/importer/fetcher/GvkFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/GvkFetcher.java index 5d3a660af4e..2a6c953d280 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/GvkFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/GvkFetcher.java @@ -38,7 +38,7 @@ public Optional getHelpPage() { } @Override - public URL getURLForQuery(String transformedQuery) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(String transformedQuery, AbstractQueryTransformer transformer) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder(URL_PATTERN); uriBuilder.addParameter("version", "1.1"); uriBuilder.addParameter("operation", "searchRetrieve"); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java b/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java index adc8da19cf1..ea7d6e00c8a 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java @@ -224,16 +224,35 @@ public Optional getHelpPage() { } @Override - public URL getURLForQuery(String transformedQuery, int pageNumber) throws URISyntaxException, MalformedURLException { + public URL getURLForQuery(String transformedQuery, int pageNumber, AbstractQueryTransformer transformer) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder("https://ieeexploreapi.ieee.org/api/v1/search/articles"); uriBuilder.addParameter("apikey", API_KEY); - uriBuilder.addParameter("querytext", transformedQuery); + if (!transformedQuery.isBlank()) { + uriBuilder.addParameter("querytext", transformedQuery); + } + System.out.println(transformedQuery); uriBuilder.addParameter("max_records", String.valueOf(getPageSize())); + IEEEQueryTransformer ieeeQueryTransformer = ((IEEEQueryTransformer) transformer); + // Currently not working as part of the query string + if ((ieeeQueryTransformer).getJournal().isPresent()) { + uriBuilder.addParameter("publication_title", ieeeQueryTransformer.getJournal().get()); + } + if (ieeeQueryTransformer.getStartYear().isPresent()) { + uriBuilder.addParameter("start_year", String.valueOf(ieeeQueryTransformer.getStartYear().get())); + } + if (ieeeQueryTransformer.getEndYear().isPresent()) { + uriBuilder.addParameter("end_year", String.valueOf(ieeeQueryTransformer.getEndYear().get())); + } + if (ieeeQueryTransformer.getArticleNumber().isPresent()) { + uriBuilder.addParameter("article_number", String.valueOf(ieeeQueryTransformer.getArticleNumber().get())); + } // Starts to index at 1 for the first entry uriBuilder.addParameter("start_record", String.valueOf(getPageSize() * pageNumber) + 1); URLDownload.bypassSSLVerification(); + System.out.println(uriBuilder.build().toURL()); + return uriBuilder.build().toURL(); } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/INSPIREFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/INSPIREFetcher.java index 457134fc59f..c8203e5c996 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/INSPIREFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/INSPIREFetcher.java @@ -13,6 +13,7 @@ import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; +import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; import org.jabref.logic.importer.fileformat.BibtexParser; import org.jabref.logic.importer.util.MediaTypes; import org.jabref.logic.layout.format.LatexToUnicodeFormatter; @@ -48,7 +49,7 @@ public Optional getHelpPage() { } @Override - public URL getURLForQuery(String transformedQuery) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(String transformedQuery, AbstractQueryTransformer transformer) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder(INSPIRE_HOST); uriBuilder.addParameter("q", transformedQuery); // Query return uriBuilder.build().toURL(); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/JstorFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/JstorFetcher.java index 850fd896739..36e3a7b8c19 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/JstorFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/JstorFetcher.java @@ -19,6 +19,7 @@ import org.jabref.logic.importer.ParseException; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; +import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; import org.jabref.logic.importer.fileformat.BibtexParser; import org.jabref.logic.net.URLDownload; import org.jabref.model.entry.BibEntry; @@ -47,7 +48,7 @@ public JstorFetcher(ImportFormatPreferences importFormatPreferences) { } @Override - public URL getURLForQuery(String transformedQuery) throws URISyntaxException, MalformedURLException { + public URL getURLForQuery(String transformedQuery, AbstractQueryTransformer transformer) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(SEARCH_HOST); uriBuilder.addParameter("Query", transformedQuery); return uriBuilder.build().toURL(); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/MathSciNet.java b/src/main/java/org/jabref/logic/importer/fetcher/MathSciNet.java index 626860819d3..57ed8e67d5b 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/MathSciNet.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/MathSciNet.java @@ -23,6 +23,7 @@ import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; +import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; import org.jabref.logic.importer.fileformat.BibtexParser; import org.jabref.logic.util.OS; import org.jabref.model.entry.BibEntry; @@ -72,7 +73,7 @@ public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedUR } @Override - public URL getURLForQuery(String transformedQuery) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(String transformedQuery, AbstractQueryTransformer transformer) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder("https://mathscinet.ams.org/mathscinet/search/publications.html"); uriBuilder.addParameter("pg7", "ALLF"); // search all fields uriBuilder.addParameter("s7", transformedQuery); // query diff --git a/src/main/java/org/jabref/logic/importer/fetcher/MedlineFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/MedlineFetcher.java index 70fb08e1390..619c8c0a9d4 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/MedlineFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/MedlineFetcher.java @@ -28,6 +28,7 @@ import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.ParserResult; import org.jabref.logic.importer.SearchBasedFetcher; +import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; import org.jabref.logic.importer.fileformat.MedlineImporter; import org.jabref.logic.l10n.Localization; import org.jabref.model.entry.BibEntry; @@ -196,7 +197,7 @@ private List fetchMedline(List ids) throws FetcherException { } @Override - public List performSearchForTransformedQuery(String transformedQuery) throws FetcherException { + public List performSearchForTransformedQuery(String transformedQuery, AbstractQueryTransformer transformer) throws FetcherException { List entryList; if (transformedQuery.isBlank()) { diff --git a/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java index 8939108e275..985a819a62c 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java @@ -160,7 +160,7 @@ public Optional getHelpPage() { } @Override - public URL getURLForQuery(String transformedQuery, int pageNumber) throws URISyntaxException, MalformedURLException { + public URL getURLForQuery(String transformedQuery, int pageNumber, AbstractQueryTransformer transformer) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(API_URL); uriBuilder.addParameter("q", transformedQuery); // Search query uriBuilder.addParameter("api_key", API_KEY); // API key diff --git a/src/main/java/org/jabref/logic/importer/fetcher/ZbMATH.java b/src/main/java/org/jabref/logic/importer/fetcher/ZbMATH.java index 279231cd318..47d0122b7db 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/ZbMATH.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/ZbMATH.java @@ -52,7 +52,7 @@ public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedUR } */ @Override - public URL getURLForQuery(String transformedQuery) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(String transformedQuery, AbstractQueryTransformer transformer) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder("https://zbmath.org/bibtexoutput/"); uriBuilder.addParameter("q", transformedQuery); // search all fields uriBuilder.addParameter("start", "0"); // start index diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformators/ArXivQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformators/ArXivQueryTransformer.java index 28ddbd92edf..e6d7f9223b9 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/transformators/ArXivQueryTransformer.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformators/ArXivQueryTransformer.java @@ -44,8 +44,8 @@ protected String handleJournal(String journalTitle) { */ @Override protected String handleYear(String year) { - startYear = Math.max(startYear, Integer.parseInt(year)); - endYear = Math.min(endYear, Integer.parseInt(year)); + startYear = Math.min(startYear, Integer.parseInt(year)); + endYear = Math.max(endYear, Integer.parseInt(year)); return year; } @@ -55,8 +55,8 @@ protected String handleYear(String year) { @Override protected String handleYearRange(String yearRange) { String[] split = yearRange.split("-"); - startYear = Math.max(startYear, Integer.parseInt(split[0])); - endYear = Math.min(endYear, Integer.parseInt(split[1])); + startYear = Math.min(startYear, Integer.parseInt(split[0])); + endYear = Math.max(endYear, Integer.parseInt(split[1])); return ""; } @@ -66,10 +66,10 @@ protected String handleUnFieldedTerm(String term) { } public int getStartYear() { - return startYear; + return startYear == Integer.MAX_VALUE ? Integer.MIN_VALUE : startYear; } public int getEndYear() { - return endYear; + return endYear == Integer.MIN_VALUE ? Integer.MAX_VALUE: endYear; } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformer.java index 49570e707fa..1ad537aac41 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformer.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformer.java @@ -1,10 +1,15 @@ package org.jabref.logic.importer.fetcher.transformators; +import java.util.Objects; +import java.util.Optional; + /** * Needs to be instantiated for each new query */ public class IEEEQueryTransformer extends AbstractQueryTransformer { // These have to be integrated into the IEEE query URL as these are just supported as query parameters + private String journal; + private String articleNumber; private int startYear = Integer.MAX_VALUE; private int endYear = Integer.MIN_VALUE; @@ -35,21 +40,22 @@ protected String handleTitle(String title) { @Override protected String handleJournal(String journalTitle) { - return String.format("publication_title:\"%s\"", journalTitle); + journal = String.format("\"%s\"", journalTitle); + return ""; } @Override protected String handleYear(String year) { - startYear = Math.max(startYear, Integer.parseInt(year)); - endYear = Math.min(endYear, Integer.parseInt(year)); + startYear = Math.min(startYear, Integer.parseInt(year)); + endYear = Math.max(endYear, Integer.parseInt(year)); return ""; } @Override protected String handleYearRange(String yearRange) { String[] split = yearRange.split("-"); - startYear = Math.max(startYear, Integer.parseInt(split[0])); - endYear = Math.min(endYear, Integer.parseInt(split[1])); + startYear = Math.min(startYear, Integer.parseInt(split[0])); + endYear = Math.max(endYear, Integer.parseInt(split[1])); return ""; } @@ -58,11 +64,32 @@ protected String handleUnFieldedTerm(String term) { return String.format("\"%s\"", term); } - public int getStartYear() { - return startYear; + @Override + protected Optional handleOtherField(String fieldAsString, String term) { + return switch (fieldAsString) { + case "article_number" -> handleArticleNumber(term); + default -> super.handleOtherField(fieldAsString, term); + }; + } + + private Optional handleArticleNumber(String term) { + articleNumber = term; + return Optional.empty(); + } + + public Optional getStartYear() { + return startYear == Integer.MAX_VALUE ? Optional.empty() : Optional.of(startYear); + } + + public Optional getEndYear() { + return endYear == Integer.MIN_VALUE ? Optional.empty() : Optional.of(endYear); + } + + public Optional getJournal() { + return Objects.isNull(journal) ? Optional.empty() : Optional.of(journal); } - public int getEndYear() { - return endYear; + public Optional getArticleNumber() { + return Objects.isNull(articleNumber) ? Optional.empty() : Optional.of(articleNumber); } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformators/ScholarQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformators/ScholarQueryTransformer.java index a5882508bf5..d661b2f9ba3 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/transformators/ScholarQueryTransformer.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformators/ScholarQueryTransformer.java @@ -37,7 +37,7 @@ protected String handleJournal(String journalTitle) { @Override protected String handleYear(String year) { - startYear = Math.max(startYear, Integer.parseInt(year)); + startYear = Math.min(startYear, Integer.parseInt(year)); endYear = Math.max(endYear, Integer.parseInt(year)); return ""; } @@ -45,7 +45,7 @@ protected String handleYear(String year) { @Override protected String handleYearRange(String yearRange) { String[] split = yearRange.split("-"); - startYear = Math.max(startYear, Integer.parseInt(split[0])); + startYear = Math.min(startYear, Integer.parseInt(split[0])); endYear = Math.max(endYear, Integer.parseInt(split[1])); return ""; } @@ -56,10 +56,10 @@ protected String handleUnFieldedTerm(String term) { } public int getStartYear() { - return startYear; + return startYear == Integer.MAX_VALUE ? Integer.MIN_VALUE : startYear; } public int getEndYear() { - return endYear; + return endYear == Integer.MIN_VALUE ? Integer.MAX_VALUE: endYear; } } diff --git a/src/test/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcherTest.java index 577f4624196..87fea9c60e3 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcherTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcherTest.java @@ -10,6 +10,7 @@ import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.ImportFormatPreferences; +import org.jabref.logic.importer.fetcher.transformators.DefaultQueryTransformer; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.field.UnknownField; @@ -43,7 +44,7 @@ public void getNameReturnsCorrectName() { @Test public void getUrlForQueryReturnsCorrectUrl() throws MalformedURLException, URISyntaxException, FetcherException { String query = "java jdk"; - URL url = fetcher.getURLForQuery(query); + URL url = fetcher.getURLForQuery(query, new DefaultQueryTransformer()); assertEquals("http://liinwww.ira.uka.de/bibliography/rss?query=java+jdk&sort=score", url.toString()); } diff --git a/src/test/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcherTest.java index b639c35cf5b..3f4ce369441 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcherTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcherTest.java @@ -18,6 +18,7 @@ import org.jabref.testutils.category.FetcherTest; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -30,6 +31,7 @@ @FetcherTest @DisabledOnCIServer("Produces to many requests on CI") +@Disabled public class CompositeSearchBasedFetcherTest { private static final Logger LOGGER = LoggerFactory.getLogger(CompositeSearchBasedFetcherTest.class); diff --git a/src/test/java/org/jabref/logic/importer/fetcher/GvkFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/GvkFetcherTest.java index d9b94f99ca2..f65bfb90595 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/GvkFetcherTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/GvkFetcherTest.java @@ -7,6 +7,7 @@ import java.util.List; import org.jabref.logic.importer.FetcherException; +import org.jabref.logic.importer.fetcher.transformators.GVKQueryTransformer; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.field.UnknownField; @@ -66,14 +67,14 @@ public void testGetName() { @Test public void simpleSearchQueryURLCorrect() throws MalformedURLException, URISyntaxException, FetcherException { String query = "java jdk"; - URL url = fetcher.getURLForQuery(query); + URL url = fetcher.getURLForQuery(query, new GVKQueryTransformer()); assertEquals("http://sru.gbv.de/gvk?version=1.1&operation=searchRetrieve&query=pica.all%3Djava+jdk&maximumRecords=50&recordSchema=picaxml&sortKeys=Year%2C%2C1", url.toString()); } @Test public void complexSearchQueryURLCorrect() throws MalformedURLException, URISyntaxException, FetcherException { String query = "kon java tit jdk"; - URL url = fetcher.getURLForQuery(query); + URL url = fetcher.getURLForQuery(query, new GVKQueryTransformer()); assertEquals("http://sru.gbv.de/gvk?version=1.1&operation=searchRetrieve&query=pica.kon%3Djava+and+pica.tit%3Djdk&maximumRecords=50&recordSchema=picaxml&sortKeys=Year%2C%2C1", url.toString()); } diff --git a/src/test/java/org/jabref/logic/importer/fetcher/IEEETest.java b/src/test/java/org/jabref/logic/importer/fetcher/IEEETest.java index a063519bb8d..93224f5e8b0 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/IEEETest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/IEEETest.java @@ -85,20 +85,21 @@ void notFoundByDOI() throws Exception { @Test void searchResultHasNoKeywordTerms() throws Exception { BibEntry expected = new BibEntry(StandardEntryType.Article) - .withField(StandardField.AUTHOR, "Shatakshi Jha and Ikhlaq Hussain and Bhim Singh and Sukumar Mishra") - .withField(StandardField.DATE, "25 2 2019") - .withField(StandardField.YEAR, "2019") - .withField(StandardField.DOI, "10.1049/iet-rpg.2018.5648") - .withField(StandardField.FILE, ":https\\://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=8636659:PDF") - .withField(StandardField.ISSUE, "3") - .withField(StandardField.ISSN, "1752-1424") - .withField(StandardField.JOURNALTITLE, "IET Renewable Power Generation") - .withField(StandardField.PAGES, "418--426") - .withField(StandardField.PUBLISHER, "IET") - .withField(StandardField.TITLE, "Optimal operation of PV-DG-battery based microgrid with power quality conditioner") - .withField(StandardField.VOLUME, "13"); - - List fetchedEntries = fetcher.performSearch("8636659"); // article number + .withField(StandardField.AUTHOR, "Shatakshi Sharma and Bhim Singh and Sukumar Mishra") + .withField(StandardField.DATE, "April 2020") + .withField(StandardField.YEAR, "2020") + .withField(StandardField.DOI, "10.1109/TII.2019.2935531") + .withField(StandardField.FILE, ":https\\://ieeexplore.ieee.org/stamp/stamp.jsp?tp=&arnumber=8801912:PDF") + .withField(StandardField.ISSUE, "4") + .withField(StandardField.ISSN, "1941-0050") + .withField(StandardField.JOURNALTITLE, "IEEE Transactions on Industrial Informatics") + .withField(StandardField.PAGES, "2346--2356") + .withField(StandardField.PUBLISHER, "IEEE") + .withField(StandardField.TITLE, "Economic Operation and Quality Control in PV-BES-DG-Based Autonomous System") + .withField(StandardField.VOLUME, "16") + .withField(StandardField.KEYWORDS, "Batteries, Generators, Economics, Power quality, State of charge, Harmonic analysis, Control systems, Battery, diesel generator (DG), distributed generation, power quality, photovoltaic (PV), voltage source converter (VSC)"); + + List fetchedEntries = fetcher.performSearch("article_number:8801912"); // article number fetchedEntries.forEach(entry -> entry.clearField(StandardField.ABSTRACT)); // Remove abstract due to copyright); assertEquals(Collections.singletonList(expected), fetchedEntries); } diff --git a/src/test/java/org/jabref/logic/importer/fetcher/SearchBasedFetcherCapabilityTest.java b/src/test/java/org/jabref/logic/importer/fetcher/SearchBasedFetcherCapabilityTest.java index a9c63797e10..6653a0d3e09 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/SearchBasedFetcherCapabilityTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/SearchBasedFetcherCapabilityTest.java @@ -32,7 +32,7 @@ interface SearchBasedFetcherCapabilityTest { */ @Test default void supportsAuthorSearch() throws Exception { - StringJoiner queryBuilder = new StringJoiner(" AND author:", "author:", ""); + StringJoiner queryBuilder = new StringJoiner("\" AND author:\"", "author:\"", "\""); getTestAuthors().forEach(queryBuilder::add); @@ -89,7 +89,7 @@ default void supportsYearRangeSearch() throws Exception { */ @Test default void supportsJournalSearch() throws Exception { - List result = getFetcher().performSearch("journal:" + getTestJournal()); + List result = getFetcher().performSearch("journal:\"" + getTestJournal() + "\""); new ImportCleanup(BibDatabaseMode.BIBTEX).doPostCleanup(result); assertFalse(result.isEmpty()); diff --git a/src/test/java/org/jabref/model/entry/CrossrefTest.java b/src/test/java/org/jabref/model/entry/CrossrefTest.java index 9f212b5b519..e1cee24b04d 100644 --- a/src/test/java/org/jabref/model/entry/CrossrefTest.java +++ b/src/test/java/org/jabref/model/entry/CrossrefTest.java @@ -11,6 +11,7 @@ import org.jabref.model.entry.types.StandardEntryType; import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.EnumSource; @@ -19,6 +20,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; +@Disabled class CrossrefTest { private BibEntry parent; From 968da8bed487f0d3a88455ea27cba0e0f6584513 Mon Sep 17 00:00:00 2001 From: Dominik Voigt Date: Wed, 13 Jan 2021 17:39:15 +0100 Subject: [PATCH 12/24] Fix ArXiv bugs --- .../java/org/jabref/logic/importer/fetcher/ArXiv.java | 11 ++++++++--- .../fetcher/transformators/ArXivQueryTransformer.java | 10 ++++++---- .../org/jabref/logic/importer/fetcher/ArXivTest.java | 4 ++-- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/fetcher/ArXiv.java b/src/main/java/org/jabref/logic/importer/fetcher/ArXiv.java index 8e53eb02b6d..83f4080aa9c 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/ArXiv.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/ArXiv.java @@ -259,18 +259,23 @@ public Optional getHelpPage() { */ @Override public Page performSearchPagedForTransformedQuery(String transformedQuery, int pageNumber, AbstractQueryTransformer transformer) throws FetcherException { + System.out.println(transformedQuery); + System.out.println(((ArXivQueryTransformer) transformer).getStartYear() + " " + ((ArXivQueryTransformer) transformer).getEndYear()); List searchResult = searchForEntries(transformedQuery, pageNumber).stream() .map((arXivEntry) -> arXivEntry.toBibEntry(importFormatPreferences.getKeywordSeparator())) .collect(Collectors.toList()); + System.out.println(searchResult); return new Page<>(transformedQuery, pageNumber, filterYears(searchResult, transformer)); } private List filterYears(List searchResult, AbstractQueryTransformer transformer) { ArXivQueryTransformer arXivQueryTransformer = ((ArXivQueryTransformer) transformer); + System.out.println(Integer.parseInt(searchResult.get(0).getField(StandardField.DATE).get().substring(0, 4))); return searchResult.stream() - .filter(entry -> entry.getField(StandardField.YEAR).isPresent()) - .filter(entry -> arXivQueryTransformer.getEndYear() == Integer.MAX_VALUE || Integer.parseInt(entry.getField(StandardField.YEAR).get()) <= arXivQueryTransformer.getEndYear()) - .filter(entry -> arXivQueryTransformer.getStartYear() == Integer.MIN_VALUE ||Integer.parseInt(entry.getField(StandardField.YEAR).get()) >= arXivQueryTransformer.getStartYear()) + .filter(entry -> entry.getField(StandardField.DATE).isPresent()) + // Filter the date field for year only + .filter(entry -> !arXivQueryTransformer.getEndYear().isPresent() || Integer.parseInt(entry.getField(StandardField.DATE).get().substring(0, 4)) <= arXivQueryTransformer.getEndYear().get()) + .filter(entry -> !arXivQueryTransformer.getStartYear().isPresent() || Integer.parseInt(entry.getField(StandardField.DATE).get().substring(0, 4)) >= arXivQueryTransformer.getStartYear().get()) .collect(Collectors.toList()); } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformators/ArXivQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformators/ArXivQueryTransformer.java index e6d7f9223b9..dd94eab746b 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/transformators/ArXivQueryTransformer.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformators/ArXivQueryTransformer.java @@ -1,5 +1,7 @@ package org.jabref.logic.importer.fetcher.transformators; +import java.util.Optional; + public class ArXivQueryTransformer extends AbstractQueryTransformer { // These can be used for filtering in post processing private int startYear = Integer.MAX_VALUE; @@ -65,11 +67,11 @@ protected String handleUnFieldedTerm(String term) { return String.format("all:\"%s\"", term); } - public int getStartYear() { - return startYear == Integer.MAX_VALUE ? Integer.MIN_VALUE : startYear; + public Optional getStartYear() { + return startYear == Integer.MAX_VALUE ? Optional.empty() : Optional.of(startYear); } - public int getEndYear() { - return endYear == Integer.MIN_VALUE ? Integer.MAX_VALUE: endYear; + public Optional getEndYear() { + return endYear == Integer.MIN_VALUE ? Optional.empty() : Optional.of(endYear); } } diff --git a/src/test/java/org/jabref/logic/importer/fetcher/ArXivTest.java b/src/test/java/org/jabref/logic/importer/fetcher/ArXivTest.java index 9fdf8b15ee8..2009f12e798 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/ArXivTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/ArXivTest.java @@ -229,7 +229,7 @@ public SearchBasedFetcher getFetcher() { @Override public List getTestAuthors() { - return List.of("\"Tobias Diez\""); + return List.of("Tobias Diez"); } @Disabled("Is not supported by the current API") @@ -247,7 +247,7 @@ public void supportsYearRangeSearch() throws Exception { @Override public String getTestJournal() { - return "\"Journal of Geometry and Physics (2013)\""; + return "Journal of Geometry and Physics (2013)"; } /** From a51dd42d42d5075eae4fccb450f9803122b97e46 Mon Sep 17 00:00:00 2001 From: Dominik Voigt Date: Wed, 13 Jan 2021 18:27:11 +0100 Subject: [PATCH 13/24] Fix GVK and Grobid fetcher issues Fix some tests --- .../fetcher/GrobidCitationFetcher.java | 28 +++++++------------ .../logic/importer/fetcher/GvkFetcher.java | 1 + .../importer/fetcher/DBLPFetcherTest.java | 2 +- .../importer/fetcher/GvkFetcherTest.java | 18 +++++++----- .../importer/fetcher/INSPIREFetcherTest.java | 2 +- .../importer/fetcher/SpringerFetcherTest.java | 4 +-- 6 files changed, 26 insertions(+), 29 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java index 77ef4f98768..f56560abe11 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java @@ -49,11 +49,11 @@ private Optional parseUsingGrobid(String plainText) { } catch (SocketTimeoutException e) { String msg = "Connection timed out."; LOGGER.debug(msg, e); - throw new RuntimeException(msg, e); + return Optional.empty(); } catch (IOException e) { String msg = "Could not process citation. " + e.getMessage(); LOGGER.debug(msg, e); - throw new RuntimeException(msg, e); + return Optional.empty(); } } @@ -73,21 +73,13 @@ public String getName() { @Override public List performSearchForTransformedQuery(String transformedQuery, AbstractQueryTransformer transformer) throws FetcherException { - List bibEntries; - try { - bibEntries = Arrays - .stream(transformedQuery.split("\\r\\r+|\\n\\n+|\\r\\n(\\r\\n)+")) - .map(String::trim) - .filter(str -> !str.isBlank()) - .map(this::parseUsingGrobid) - .flatMap(Optional::stream) - .map(this::parseBibToBibEntry) - .flatMap(Optional::stream) - .collect(Collectors.toList()); - } catch (RuntimeException e) { - // un-wrap the wrapped exceptions - throw new FetcherException(e.getMessage(), e.getCause()); - } - return bibEntries; + return Arrays.stream(transformedQuery.split("\\r\\r+|\\n\\n+|\\r\\n(\\r\\n)+")) + .map(String::trim) + .filter(str -> !str.isBlank()) + .map(this::parseUsingGrobid) + .flatMap(Optional::stream) + .map(this::parseBibToBibEntry) + .flatMap(Optional::stream) + .collect(Collectors.toList()); } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/GvkFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/GvkFetcher.java index 2a6c953d280..00d26abf390 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/GvkFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/GvkFetcher.java @@ -46,6 +46,7 @@ public URL getURLForQuery(String transformedQuery, AbstractQueryTransformer tran uriBuilder.addParameter("maximumRecords", "50"); uriBuilder.addParameter("recordSchema", "picaxml"); uriBuilder.addParameter("sortKeys", "Year,,1"); + System.out.println(uriBuilder.build().toURL()); return uriBuilder.build().toURL(); } diff --git a/src/test/java/org/jabref/logic/importer/fetcher/DBLPFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/DBLPFetcherTest.java index 83414d8666d..bf788b6fe8b 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/DBLPFetcherTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/DBLPFetcherTest.java @@ -50,7 +50,7 @@ public void setUp() { @Test public void findSingleEntry() throws FetcherException { - String query = "Process Engine Benchmarking with Betsy in the Context of {ISO/IEC} Quality Standards"; + String query = "Process Engine Benchmarking with Betsy in the Context of \"{ISO/IEC}\" Quality Standards"; List result = dblpFetcher.performSearch(query); assertEquals(Collections.singletonList(entry), result); diff --git a/src/test/java/org/jabref/logic/importer/fetcher/GvkFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/GvkFetcherTest.java index f65bfb90595..8bd8bfc497e 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/GvkFetcherTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/GvkFetcherTest.java @@ -6,7 +6,9 @@ import java.util.Collections; import java.util.List; +import org.jabref.logic.JabRefException; import org.jabref.logic.importer.FetcherException; +import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; import org.jabref.logic.importer.fetcher.transformators.GVKQueryTransformer; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; @@ -65,17 +67,19 @@ public void testGetName() { } @Test - public void simpleSearchQueryURLCorrect() throws MalformedURLException, URISyntaxException, FetcherException { + public void simpleSearchQueryURLCorrect() throws Exception { String query = "java jdk"; - URL url = fetcher.getURLForQuery(query, new GVKQueryTransformer()); - assertEquals("http://sru.gbv.de/gvk?version=1.1&operation=searchRetrieve&query=pica.all%3Djava+jdk&maximumRecords=50&recordSchema=picaxml&sortKeys=Year%2C%2C1", url.toString()); + AbstractQueryTransformer transformer = new GVKQueryTransformer(); + URL url = fetcher.getURLForQuery(transformer.parseQueryStringIntoComplexQuery(query).get(), transformer); + assertEquals("http://sru.gbv.de/gvk?version=1.1&operation=searchRetrieve&query=pica.all%3D%22java%22+and+pica.all%3D%22jdk%22&maximumRecords=50&recordSchema=picaxml&sortKeys=Year%2C%2C1", url.toString()); } @Test - public void complexSearchQueryURLCorrect() throws MalformedURLException, URISyntaxException, FetcherException { - String query = "kon java tit jdk"; - URL url = fetcher.getURLForQuery(query, new GVKQueryTransformer()); - assertEquals("http://sru.gbv.de/gvk?version=1.1&operation=searchRetrieve&query=pica.kon%3Djava+and+pica.tit%3Djdk&maximumRecords=50&recordSchema=picaxml&sortKeys=Year%2C%2C1", url.toString()); + public void complexSearchQueryURLCorrect() throws Exception { + String query = "kon:java tit:jdk"; + AbstractQueryTransformer transformer = new GVKQueryTransformer(); + URL url = fetcher.getURLForQuery(transformer.parseQueryStringIntoComplexQuery(query).get(), new GVKQueryTransformer()); + assertEquals("http://sru.gbv.de/gvk?version=1.1&operation=searchRetrieve&query=pica.kon%3D%22java%22+and+pica.tit%3D%22jdk%22&maximumRecords=50&recordSchema=picaxml&sortKeys=Year%2C%2C1", url.toString()); } @Test diff --git a/src/test/java/org/jabref/logic/importer/fetcher/INSPIREFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/INSPIREFetcherTest.java index ff8c597705b..43d5377fa64 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/INSPIREFetcherTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/INSPIREFetcherTest.java @@ -59,7 +59,7 @@ public void searchByIdentifierFindsEntry() throws Exception { .withField(StandardField.EPRINT, "hep-ph/9802379") .withField(StandardField.ARCHIVEPREFIX, "arXiv") .withField(new UnknownField("reportnumber"), "BUDKER-INP-1998-7, TTP-98-10"); - List fetchedEntries = fetcher.performSearch("hep-ph/9802379"); + List fetchedEntries = fetcher.performSearch("\"hep-ph/9802379\""); assertEquals(Collections.singletonList(article), fetchedEntries); } } diff --git a/src/test/java/org/jabref/logic/importer/fetcher/SpringerFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/SpringerFetcherTest.java index 74e595c777a..4774e1250e2 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/SpringerFetcherTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/SpringerFetcherTest.java @@ -140,12 +140,12 @@ public SearchBasedFetcher getFetcher() { @Override public List getTestAuthors() { - return List.of("\"Steinmacher, Igor\"", "\"Gerosa, Marco\"", "\"Conte, Tayana U.\""); + return List.of("Steinmacher, Igor", "Gerosa, Marco", "Conte, Tayana U."); } @Override public String getTestJournal() { - return "\"Clinical Research in Cardiology\""; + return "Clinical Research in Cardiology"; } @Override From 1680a5f77fb965684c830b3051cc015092c3ed32 Mon Sep 17 00:00:00 2001 From: Dominik Voigt Date: Wed, 13 Jan 2021 19:22:10 +0100 Subject: [PATCH 14/24] Fix Grobid, Astro, GVK --- .../PagedSearchBasedParserFetcher.java | 3 +- .../fetcher/GrobidCitationFetcher.java | 36 +++++++++++++------ .../transformators/GVKQueryTransformer.java | 7 ++++ .../fetcher/GrobidCitationFetcherTest.java | 3 +- 4 files changed, 37 insertions(+), 12 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java b/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java index bc2384b9ba9..84ac09f2ba3 100644 --- a/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java +++ b/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java @@ -5,6 +5,7 @@ import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; +import java.util.ArrayList; import java.util.List; import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; @@ -52,6 +53,6 @@ default URL getURLForQuery(String transformedQuery, AbstractQueryTransformer tra @Override default List performSearchForTransformedQuery(String transformedQuery, AbstractQueryTransformer transformer) throws FetcherException { - return SearchBasedParserFetcher.super.performSearchForTransformedQuery(transformedQuery, transformer); + return new ArrayList<>(performSearchPagedForTransformedQuery(transformedQuery, 0, transformer).getContent()); } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java index f56560abe11..28d2d757c76 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java @@ -3,8 +3,10 @@ import java.io.IOException; import java.net.SocketTimeoutException; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Optional; +import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; import org.jabref.logic.importer.FetcherException; @@ -43,13 +45,13 @@ public GrobidCitationFetcher(ImportFormatPreferences importFormatPreferences) { * * @return A BibTeX string if extraction is successful */ - private Optional parseUsingGrobid(String plainText) { + private Optional parseUsingGrobid(String plainText) throws RuntimeException { try { return Optional.of(grobidService.processCitation(plainText, GrobidService.ConsolidateCitations.WITH_METADATA)); } catch (SocketTimeoutException e) { String msg = "Connection timed out."; LOGGER.debug(msg, e); - return Optional.empty(); + throw new RuntimeException(msg, e); } catch (IOException e) { String msg = "Could not process citation. " + e.getMessage(); LOGGER.debug(msg, e); @@ -71,15 +73,29 @@ public String getName() { return "GROBID"; } + @Override + public List performSearch(String searchQuery) throws FetcherException { + List collect; + try { + collect = Arrays.stream(searchQuery.split("\\r\\r+|\\n\\n+|\\r\\n(\\r\\n)+")) + .map(String::trim) + .filter(str -> !str.isBlank()) + .map(this::parseUsingGrobid) + .flatMap(Optional::stream) + .map(this::parseBibToBibEntry) + .flatMap(Optional::stream) + .collect(Collectors.toList()); + } catch (RuntimeException e) { + throw new FetcherException(e.getMessage(), e.getCause()); + } + return collect; + } + + /** + * Not used + */ @Override public List performSearchForTransformedQuery(String transformedQuery, AbstractQueryTransformer transformer) throws FetcherException { - return Arrays.stream(transformedQuery.split("\\r\\r+|\\n\\n+|\\r\\n(\\r\\n)+")) - .map(String::trim) - .filter(str -> !str.isBlank()) - .map(this::parseUsingGrobid) - .flatMap(Optional::stream) - .map(this::parseBibToBibEntry) - .flatMap(Optional::stream) - .collect(Collectors.toList()); + return Collections.emptyList(); } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformators/GVKQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformators/GVKQueryTransformer.java index a35fa0fe68e..402e68b09c0 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/transformators/GVKQueryTransformer.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformators/GVKQueryTransformer.java @@ -1,5 +1,7 @@ package org.jabref.logic.importer.fetcher.transformators; +import java.util.Optional; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -57,4 +59,9 @@ protected String handleUnFieldedTerm(String term) { // Other option is txt: but this does not search in meta data return String.format("pica.all=\"%s\"", term); } + + @Override + protected Optional handleOtherField(String fieldAsString, String term) { + return Optional.of("pica." + fieldAsString + "=\"" + term + "\""); + } } diff --git a/src/test/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcherTest.java index 329748dcd56..f970273c34b 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcherTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcherTest.java @@ -1,6 +1,7 @@ package org.jabref.logic.importer.fetcher; import java.io.IOException; +import java.net.SocketTimeoutException; import java.util.Collections; import java.util.List; import java.util.stream.Stream; @@ -110,7 +111,7 @@ public void grobidPerformSearchWithInvalidDataTest(String invalidInput) throws F @Test public void performSearchThrowsExceptionInCaseOfConnectionIssues() throws IOException { GrobidService grobidServiceMock = mock(GrobidService.class); - when(grobidServiceMock.processCitation(anyString(), any())).thenThrow(new IOException("Any IO Exception")); + when(grobidServiceMock.processCitation(anyString(), any())).thenThrow(new SocketTimeoutException("Timeout")); grobidCitationFetcher = new GrobidCitationFetcher(importFormatPreferences, grobidServiceMock); assertThrows(FetcherException.class, () -> { From e9a9da529a876bf5f2097b598de98786abeb9670 Mon Sep 17 00:00:00 2001 From: Dominik Voigt Date: Wed, 13 Jan 2021 19:55:02 +0100 Subject: [PATCH 15/24] Fix tests, remove disabled annotations, ect. --- .../fetcher/transformators/AbstractQueryTransformer.java | 8 ++++++++ ...lectionOfComputerScienceBibliographiesFetcherTest.java | 4 +++- .../importer/fetcher/CompositeSearchBasedFetcherTest.java | 1 - src/test/java/org/jabref/model/entry/CrossrefTest.java | 1 - 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformators/AbstractQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformators/AbstractQueryTransformer.java index cbe5046b3dd..eefa81ee465 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/transformators/AbstractQueryTransformer.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformators/AbstractQueryTransformer.java @@ -92,6 +92,10 @@ private Optional transform(FieldQueryNode query) { String s = handleYearRange(term); return s.isEmpty() ? Optional.empty() : Optional.of(s); } + case "doi" -> { + String s = handleDoi(term); + return s.isEmpty() ? Optional.empty() : Optional.of(s); + } case NO_EXPLICIT_FIELD -> { return Optional.of(handleUnFieldedTerm(term)); } @@ -102,6 +106,10 @@ private Optional transform(FieldQueryNode query) { } } + protected String handleDoi(String term) { + return "doi:" + term; + } + /** * Handles the not modifier, all other cases are silently ignored */ diff --git a/src/test/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcherTest.java index 87fea9c60e3..1a220323843 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcherTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcherTest.java @@ -4,6 +4,8 @@ import java.net.URISyntaxException; import java.net.URL; import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -64,7 +66,7 @@ public void performSearchReturnsMatchingMultipleEntries() throws FetcherExceptio .withField(StandardField.YEAR, "2017") .withField(StandardField.BOOKTITLE, "11th European Conference on Software Architecture, ECSA 2017, Companion Proceedings, Canterbury, United Kingdom, September 11-15, 2017") .withField(new UnknownField("bibsource"), "DBLP, http://dblp.uni-trier.de/https://doi.org/10.1145/3129790.3129810; DBLP, http://dblp.uni-trier.de/db/conf/ecsa/ecsa2017c.html#OlssonEW17") - .withField(new UnknownField("bibdate"), "2018-11-06"); + .withField(new UnknownField("bibdate"), "2020-10-25"); BibEntry secondBibEntry = new BibEntry(StandardEntryType.Article) .withCitationKey("oai:DiVA.org:lnu-68408") diff --git a/src/test/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcherTest.java index 3f4ce369441..40c3eaa8dd5 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcherTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcherTest.java @@ -31,7 +31,6 @@ @FetcherTest @DisabledOnCIServer("Produces to many requests on CI") -@Disabled public class CompositeSearchBasedFetcherTest { private static final Logger LOGGER = LoggerFactory.getLogger(CompositeSearchBasedFetcherTest.class); diff --git a/src/test/java/org/jabref/model/entry/CrossrefTest.java b/src/test/java/org/jabref/model/entry/CrossrefTest.java index e1cee24b04d..98aa8ca54f0 100644 --- a/src/test/java/org/jabref/model/entry/CrossrefTest.java +++ b/src/test/java/org/jabref/model/entry/CrossrefTest.java @@ -20,7 +20,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; -@Disabled class CrossrefTest { private BibEntry parent; From 1115c118e1d5964239ca7f57dc2ee02ef8c53915 Mon Sep 17 00:00:00 2001 From: Dominik Voigt Date: Thu, 14 Jan 2021 12:18:35 +0100 Subject: [PATCH 16/24] Fix tests --- .../java/org/jabref/logic/importer/fetcher/ArXiv.java | 4 ---- .../org/jabref/logic/importer/fetcher/GvkFetcher.java | 1 - .../java/org/jabref/logic/importer/fetcher/IEEE.java | 3 --- .../fetcher/transformators/AbstractQueryTransformer.java | 8 +++----- .../fetcher/transformators/IEEEQueryTransformer.java | 1 + .../fetcher/transformators/IEEEQueryTransformerTest.java | 9 +++++++++ .../fetcher/transformators/InfixTransformerTest.java | 7 ------- 7 files changed, 13 insertions(+), 20 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/fetcher/ArXiv.java b/src/main/java/org/jabref/logic/importer/fetcher/ArXiv.java index 83f4080aa9c..4b838c258ec 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/ArXiv.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/ArXiv.java @@ -259,18 +259,14 @@ public Optional getHelpPage() { */ @Override public Page performSearchPagedForTransformedQuery(String transformedQuery, int pageNumber, AbstractQueryTransformer transformer) throws FetcherException { - System.out.println(transformedQuery); - System.out.println(((ArXivQueryTransformer) transformer).getStartYear() + " " + ((ArXivQueryTransformer) transformer).getEndYear()); List searchResult = searchForEntries(transformedQuery, pageNumber).stream() .map((arXivEntry) -> arXivEntry.toBibEntry(importFormatPreferences.getKeywordSeparator())) .collect(Collectors.toList()); - System.out.println(searchResult); return new Page<>(transformedQuery, pageNumber, filterYears(searchResult, transformer)); } private List filterYears(List searchResult, AbstractQueryTransformer transformer) { ArXivQueryTransformer arXivQueryTransformer = ((ArXivQueryTransformer) transformer); - System.out.println(Integer.parseInt(searchResult.get(0).getField(StandardField.DATE).get().substring(0, 4))); return searchResult.stream() .filter(entry -> entry.getField(StandardField.DATE).isPresent()) // Filter the date field for year only diff --git a/src/main/java/org/jabref/logic/importer/fetcher/GvkFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/GvkFetcher.java index 00d26abf390..2a6c953d280 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/GvkFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/GvkFetcher.java @@ -46,7 +46,6 @@ public URL getURLForQuery(String transformedQuery, AbstractQueryTransformer tran uriBuilder.addParameter("maximumRecords", "50"); uriBuilder.addParameter("recordSchema", "picaxml"); uriBuilder.addParameter("sortKeys", "Year,,1"); - System.out.println(uriBuilder.build().toURL()); return uriBuilder.build().toURL(); } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java b/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java index ea7d6e00c8a..17643a7e06c 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java @@ -230,7 +230,6 @@ public URL getURLForQuery(String transformedQuery, int pageNumber, AbstractQuery if (!transformedQuery.isBlank()) { uriBuilder.addParameter("querytext", transformedQuery); } - System.out.println(transformedQuery); uriBuilder.addParameter("max_records", String.valueOf(getPageSize())); IEEEQueryTransformer ieeeQueryTransformer = ((IEEEQueryTransformer) transformer); // Currently not working as part of the query string @@ -251,8 +250,6 @@ public URL getURLForQuery(String transformedQuery, int pageNumber, AbstractQuery URLDownload.bypassSSLVerification(); - System.out.println(uriBuilder.build().toURL()); - return uriBuilder.build().toURL(); } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformators/AbstractQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformators/AbstractQueryTransformer.java index eefa81ee465..804c3dbef78 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/transformators/AbstractQueryTransformer.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformators/AbstractQueryTransformer.java @@ -71,7 +71,6 @@ private Optional transform(BooleanQueryNode query) { */ protected abstract String getLogicalNotOperator(); - private Optional transform(FieldQueryNode query) { String term = query.getTextAsString(); switch (query.getFieldAsString()) { @@ -123,8 +122,8 @@ private Optional transform(ModifierQueryNode query) { } /** - * Return a string representation of the author fielded term - */ + * Return a string representation of the author fielded term + */ protected abstract String handleAuthor(String author); /** @@ -159,7 +158,7 @@ private Optional transform(ModifierQueryNode query) { * Return a string representation of the provided field * If it is not supported return an empty optional. */ - protected Optional handleOtherField(String fieldAsString, String term){ + protected Optional handleOtherField(String fieldAsString, String term) { return Optional.of(String.format("%s:\"%s\"", fieldAsString, term)); } @@ -189,7 +188,6 @@ public Optional parseQueryStringIntoComplexQuery(String query) throws Ja StandardSyntaxParser parser = new StandardSyntaxParser(); try { QueryNode luceneQuery = parser.parse(query, NO_EXPLICIT_FIELD); - System.out.println(luceneQuery); Optional transformedQuery = transform(luceneQuery); transformedQuery = transformedQuery.map(this::removeOuterBraces); return transformedQuery; diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformer.java index 1ad537aac41..a6bb4cd49c4 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformer.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformer.java @@ -8,6 +8,7 @@ */ public class IEEEQueryTransformer extends AbstractQueryTransformer { // These have to be integrated into the IEEE query URL as these are just supported as query parameters + // Journal is wrapped in quotes by the transformer private String journal; private String articleNumber; private int startYear = Integer.MAX_VALUE; diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformerTest.java index b6c921104ee..e22372a675a 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformerTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformerTest.java @@ -29,6 +29,15 @@ public String getTitlePrefix() { return "article_title:"; } + @Override + public void convertJournalField() throws Exception { + IEEEQueryTransformer transformer = ((IEEEQueryTransformer) getTransformator()); + + transformer.parseQueryStringIntoComplexQuery("journal:Nature"); + + assertEquals("\"Nautre\"", transformer.getJournal().get()); + } + @Override public void convertYearField() throws Exception { IEEEQueryTransformer transformer = ((IEEEQueryTransformer) getTransformator()); diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/InfixTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/InfixTransformerTest.java index dcfc9cf544b..ee9f8932883 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/transformators/InfixTransformerTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/InfixTransformerTest.java @@ -55,13 +55,6 @@ default void convertJournalField() throws Exception { assertEquals(expected, searchQuery); } - @Test - default void convertAlphabeticallyFirstJournalField() throws Exception { - Optional searchQuery = getTransformator().parseQueryStringIntoComplexQuery("journal:Nature journal:\"Complex Networks\""); - Optional expected = Optional.of(getJournalPrefix() + "\"Nature\"" + getTransformator().getLogicalAndOperator() + getJournalPrefix() + "\"Complex Networks\""); - assertEquals(expected, searchQuery); - } - @Test void convertYearField() throws Exception; From 1ca0f5f42492b1039e1b0d766c89594e40192cff Mon Sep 17 00:00:00 2001 From: Dominik Voigt Date: Thu, 14 Jan 2021 12:28:27 +0100 Subject: [PATCH 17/24] Reduce number of test parameter cases for CompositeSearchBasedFetcher --- .../importer/fetcher/CompositeSearchBasedFetcherTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcherTest.java index 40c3eaa8dd5..b81913d1e64 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcherTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcherTest.java @@ -115,8 +115,8 @@ static Stream performSearchParameters() { // list.add(new MedlineFetcher()); // Create different sized sets of fetchers to use in the composite fetcher. - // Selected 273 to have differencing sets - for (int i = 1; i < Math.pow(2, list.size()); i += 273) { + // Selected 1173 to have differencing sets + for (int i = 1; i < Math.pow(2, list.size()); i += 1173) { Set fetchers = new HashSet<>(); // Only shift i at maximum to its MSB to the right for (int j = 0; Math.pow(2, j) <= i; j++) { From 787f1bcd5e8dbb85cdd85699c22d2c00961757c6 Mon Sep 17 00:00:00 2001 From: Dominik Voigt Date: Fri, 15 Jan 2021 18:42:18 +0100 Subject: [PATCH 18/24] Adapt the way the transformer state is handled --- .../importer/PagedSearchBasedFetcher.java | 9 +++++---- .../PagedSearchBasedParserFetcher.java | 15 +++++++-------- .../logic/importer/SearchBasedFetcher.java | 12 ++++++++---- .../importer/SearchBasedParserFetcher.java | 8 +++----- .../jabref/logic/importer/fetcher/ArXiv.java | 14 ++++++++++---- .../fetcher/AstrophysicsDataSystem.java | 7 +++---- .../fetcher/CompositeSearchBasedFetcher.java | 11 ++++++++--- .../logic/importer/fetcher/GoogleScholar.java | 15 ++++++++++++--- .../fetcher/GrobidCitationFetcher.java | 18 ++++++++---------- .../jabref/logic/importer/fetcher/IEEE.java | 15 ++++++++++++--- .../logic/importer/fetcher/MedlineFetcher.java | 3 +-- .../importer/fetcher/SpringerFetcher.java | 2 +- .../AbstractQueryTransformer.java | 3 +-- 13 files changed, 79 insertions(+), 53 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java b/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java index 1998db13a69..99a21031bee 100644 --- a/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java +++ b/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java @@ -17,7 +17,7 @@ public interface PagedSearchBasedFetcher extends SearchBasedFetcher { * @param pageNumber requested site number indexed from 0 * @return Page with search results */ - Page performSearchPagedForTransformedQuery(String transformedQuery, int pageNumber, AbstractQueryTransformer transformer) throws FetcherException; + Page performSearchPagedForTransformedQuery(String transformedQuery, int pageNumber) throws FetcherException; /** * @param searchQuery query string that can be parsed into a complex search query @@ -28,10 +28,11 @@ default Page performSearchPaged(String searchQuery, int pageNumber) th if (searchQuery.isBlank()) { return new Page<>(searchQuery, pageNumber, Collections.emptyList()); } + resetTransformer(); AbstractQueryTransformer transformer = getQueryTransformer(); Optional transformedQuery = transformer.parseQueryStringIntoComplexQuery(searchQuery); // Otherwise just use query as a default term - return this.performSearchPagedForTransformedQuery(transformedQuery.orElse(""), pageNumber, transformer); + return this.performSearchPagedForTransformedQuery(transformedQuery.orElse(""), pageNumber); } /** @@ -42,7 +43,7 @@ default int getPageSize() { } @Override - default List performSearchForTransformedQuery(String transformedQuery, AbstractQueryTransformer transformer) throws FetcherException { - return new ArrayList<>(performSearchPagedForTransformedQuery(transformedQuery, 0, transformer).getContent()); + default List performSearchForTransformedQuery(String transformedQuery) throws FetcherException { + return new ArrayList<>(performSearchPagedForTransformedQuery(transformedQuery, 0).getContent()); } } diff --git a/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java b/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java index 84ac09f2ba3..cba0b4fbe15 100644 --- a/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java +++ b/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java @@ -8,18 +8,17 @@ import java.util.ArrayList; import java.util.List; -import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; import org.jabref.model.entry.BibEntry; import org.jabref.model.paging.Page; public interface PagedSearchBasedParserFetcher extends SearchBasedParserFetcher, PagedSearchBasedFetcher { @Override - default Page performSearchPagedForTransformedQuery(String transformedQuery, int pageNumber, AbstractQueryTransformer transformer) throws FetcherException { + default Page performSearchPagedForTransformedQuery(String transformedQuery, int pageNumber) throws FetcherException { // ADR-0014 URL urlForQuery; try { - urlForQuery = getURLForQuery(transformedQuery, pageNumber, transformer); + urlForQuery = getURLForQuery(transformedQuery, pageNumber); } catch (URISyntaxException | MalformedURLException e) { throw new FetcherException("Search URI crafted from complex search query is malformed", e); } @@ -44,15 +43,15 @@ private List getBibEntries(URL urlForQuery) throws FetcherException { * @param transformedQuery the search query * @param pageNumber the number of the page indexed from 0 */ - URL getURLForQuery(String transformedQuery, int pageNumber, AbstractQueryTransformer transformer) throws URISyntaxException, MalformedURLException; + URL getURLForQuery(String transformedQuery, int pageNumber) throws URISyntaxException, MalformedURLException; @Override - default URL getURLForQuery(String transformedQuery, AbstractQueryTransformer transformer) throws URISyntaxException, MalformedURLException, FetcherException { - return getURLForQuery(transformedQuery, 0, transformer); + default URL getURLForQuery(String transformedQuery) throws URISyntaxException, MalformedURLException, FetcherException { + return getURLForQuery(transformedQuery, 0); } @Override - default List performSearchForTransformedQuery(String transformedQuery, AbstractQueryTransformer transformer) throws FetcherException { - return new ArrayList<>(performSearchPagedForTransformedQuery(transformedQuery, 0, transformer).getContent()); + default List performSearchForTransformedQuery(String transformedQuery) throws FetcherException { + return new ArrayList<>(performSearchPagedForTransformedQuery(transformedQuery, 0).getContent()); } } diff --git a/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java b/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java index 1cc2f0bdd90..a343ab4cc3c 100644 --- a/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java +++ b/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java @@ -19,10 +19,9 @@ public interface SearchBasedFetcher extends WebFetcher { * This method is used to send complex queries using fielded search. * * @param transformedQuery the search query defining all fielded search parameters - * @param transformer transformer might be required to extract some parsing information * @return a list of {@link BibEntry}, which are matched by the query (may be empty) */ - List performSearchForTransformedQuery(String transformedQuery, AbstractQueryTransformer transformer) throws FetcherException; + List performSearchForTransformedQuery(String transformedQuery) throws FetcherException; /** * Looks for hits which are matched by the given free-text query. @@ -30,7 +29,8 @@ public interface SearchBasedFetcher extends WebFetcher { * @param searchQuery query string that can be parsed into a complex search query * @return a list of {@link BibEntry}, which are matched by the query (may be empty) */ - default List performSearch(String searchQuery) throws FetcherException { + default List performSearch(String searchQuery) throws JabRefException { + resetTransformer(); AbstractQueryTransformer transformer = getQueryTransformer(); if (searchQuery.isBlank()) { return Collections.emptyList(); @@ -42,10 +42,14 @@ default List performSearch(String searchQuery) throws FetcherException throw new FetcherException("Error occured during query transformation", e); } // Otherwise just use query as a default term - return this.performSearchForTransformedQuery(transformedQuery.orElse(""), transformer); + return this.performSearchForTransformedQuery(transformedQuery.orElse("")); } default AbstractQueryTransformer getQueryTransformer() { return new DefaultQueryTransformer(); } + + default void resetTransformer() { + + } } diff --git a/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java b/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java index 055eb4ab403..c99e2b5712c 100644 --- a/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java +++ b/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java @@ -9,7 +9,6 @@ import org.jabref.logic.JabRefException; import org.jabref.logic.cleanup.Formatter; -import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; import org.jabref.model.entry.BibEntry; /** @@ -28,14 +27,13 @@ public interface SearchBasedParserFetcher extends SearchBasedFetcher { * fielded search, such as a title, author, or year parameter. * * @param transformedQuery the search query defining all fielded search parameters - * @param transformer */ @Override - default List performSearchForTransformedQuery(String transformedQuery, AbstractQueryTransformer transformer) throws FetcherException { + default List performSearchForTransformedQuery(String transformedQuery) throws FetcherException { // ADR-0014 URL urlForQuery; try { - urlForQuery = getURLForQuery(transformedQuery, transformer); + urlForQuery = getURLForQuery(transformedQuery); } catch (URISyntaxException | MalformedURLException | JabRefException e) { throw new FetcherException("Search URI crafted from complex search query is malformed", e); } @@ -64,7 +62,7 @@ private List getBibEntries(URL urlForQuery) throws FetcherException { * * @param transformedQuery the search query */ - URL getURLForQuery(String transformedQuery, AbstractQueryTransformer transformer) throws URISyntaxException, MalformedURLException, FetcherException; + URL getURLForQuery(String transformedQuery) throws URISyntaxException, MalformedURLException, FetcherException; /** * Performs a cleanup of the fetched entry. diff --git a/src/main/java/org/jabref/logic/importer/fetcher/ArXiv.java b/src/main/java/org/jabref/logic/importer/fetcher/ArXiv.java index 4b838c258ec..cd2c364a4ab 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/ArXiv.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/ArXiv.java @@ -62,6 +62,7 @@ public class ArXiv implements FulltextFetcher, PagedSearchBasedFetcher, IdBasedF private static final String API_URL = "https://export.arxiv.org/api/query"; private final ImportFormatPreferences importFormatPreferences; + private ArXivQueryTransformer transformer; public ArXiv(ImportFormatPreferences importFormatPreferences) { this.importFormatPreferences = importFormatPreferences; @@ -258,15 +259,15 @@ public Optional getHelpPage() { * @return A list of entries matching the complex query */ @Override - public Page performSearchPagedForTransformedQuery(String transformedQuery, int pageNumber, AbstractQueryTransformer transformer) throws FetcherException { + public Page performSearchPagedForTransformedQuery(String transformedQuery, int pageNumber) throws FetcherException { List searchResult = searchForEntries(transformedQuery, pageNumber).stream() .map((arXivEntry) -> arXivEntry.toBibEntry(importFormatPreferences.getKeywordSeparator())) .collect(Collectors.toList()); - return new Page<>(transformedQuery, pageNumber, filterYears(searchResult, transformer)); + return new Page<>(transformedQuery, pageNumber, filterYears(searchResult)); } - private List filterYears(List searchResult, AbstractQueryTransformer transformer) { - ArXivQueryTransformer arXivQueryTransformer = ((ArXivQueryTransformer) transformer); + private List filterYears(List searchResult) { + ArXivQueryTransformer arXivQueryTransformer = transformer; return searchResult.stream() .filter(entry -> entry.getField(StandardField.DATE).isPresent()) // Filter the date field for year only @@ -426,4 +427,9 @@ public BibEntry toBibEntry(Character keywordDelimiter) { public AbstractQueryTransformer getQueryTransformer() { return new ArXivQueryTransformer(); } + + @Override + public void resetTransformer() { + transformer = null; + } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java index de41ef187c2..33a81285135 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java @@ -27,7 +27,6 @@ import org.jabref.logic.importer.PagedSearchBasedParserFetcher; import org.jabref.logic.importer.ParseException; import org.jabref.logic.importer.Parser; -import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; import org.jabref.logic.importer.fileformat.BibtexParser; import org.jabref.logic.net.URLDownload; import org.jabref.logic.util.BuildInfo; @@ -84,7 +83,7 @@ public String getName() { * @return URL which points to a search request for given query */ @Override - public URL getURLForQuery(String transformedQuery, int pageNumber, AbstractQueryTransformer transformer) throws URISyntaxException, MalformedURLException { + public URL getURLForQuery(String transformedQuery, int pageNumber) throws URISyntaxException, MalformedURLException { URIBuilder builder = new URIBuilder(API_SEARCH_URL); builder.addParameter("q", transformedQuery); builder.addParameter("fl", "bibcode"); @@ -275,10 +274,10 @@ private List performSearchByIds(Collection identifiers) throws } @Override - public Page performSearchPagedForTransformedQuery(String transformedQuery, int pageNumber, AbstractQueryTransformer transformer) throws FetcherException { + public Page performSearchPagedForTransformedQuery(String transformedQuery, int pageNumber) throws FetcherException { try { // This is currently just interpreting the complex query as a default string query - List bibcodes = fetchBibcodes(getURLForQuery(transformedQuery, pageNumber, transformer)); + List bibcodes = fetchBibcodes(getURLForQuery(transformedQuery, pageNumber)); Collection results = performSearchByIds(bibcodes); return new Page<>(transformedQuery, pageNumber, results); } catch (URISyntaxException e) { diff --git a/src/main/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcher.java index b39bce1463e..9cfb1a614b4 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcher.java @@ -6,11 +6,11 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import org.jabref.logic.JabRefException; import org.jabref.logic.help.HelpFile; import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.ImportCleanup; import org.jabref.logic.importer.SearchBasedFetcher; -import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; import org.jabref.model.database.BibDatabaseMode; import org.jabref.model.entry.BibEntry; @@ -47,14 +47,14 @@ public Optional getHelpPage() { } @Override - public List performSearchForTransformedQuery(String transformedQuery, AbstractQueryTransformer transformer) throws FetcherException { + public List performSearchForTransformedQuery(String transformedQuery) throws FetcherException { ImportCleanup cleanup = new ImportCleanup(BibDatabaseMode.BIBTEX); // All entries have to be converted into one format, this is necessary for the format conversion return fetchers.parallelStream() .flatMap(searchBasedFetcher -> { try { return searchBasedFetcher.performSearch(transformedQuery).stream(); - } catch (FetcherException e) { + } catch (JabRefException e) { LOGGER.warn(String.format("%s API request failed", searchBasedFetcher.getName()), e); return Stream.empty(); } @@ -63,4 +63,9 @@ public List performSearchForTransformedQuery(String transformedQuery, .map(cleanup::doPostCleanup) .collect(Collectors.toList()); } + + @Override + public void resetTransformer() { + fetchers.forEach(SearchBasedFetcher::resetTransformer); + } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/GoogleScholar.java b/src/main/java/org/jabref/logic/importer/fetcher/GoogleScholar.java index 895a43acad9..9a7846d52f9 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/GoogleScholar.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/GoogleScholar.java @@ -51,6 +51,7 @@ public class GoogleScholar implements FulltextFetcher, PagedSearchBasedFetcher { private static final int NUM_RESULTS = 10; private final ImportFormatPreferences importFormatPreferences; + private ScholarQueryTransformer transformer; public GoogleScholar(ImportFormatPreferences importFormatPreferences) { Objects.requireNonNull(importFormatPreferences); @@ -177,7 +178,7 @@ private void obtainAndModifyCookie() throws FetcherException { } @Override - public Page performSearchPagedForTransformedQuery(String transformedQuery, int pageNumber, AbstractQueryTransformer transformer) throws FetcherException { + public Page performSearchPagedForTransformedQuery(String transformedQuery, int pageNumber) throws FetcherException { try { obtainAndModifyCookie(); List foundEntries = new ArrayList<>(10); @@ -188,7 +189,7 @@ public Page performSearchPagedForTransformedQuery(String transformedQu uriBuilder.addParameter("q", transformedQuery); uriBuilder.addParameter("start", String.valueOf(pageNumber * getPageSize())); uriBuilder.addParameter("num", String.valueOf(getPageSize())); - ScholarQueryTransformer scholarQueryTransformer = ((ScholarQueryTransformer) transformer); + ScholarQueryTransformer scholarQueryTransformer = transformer; uriBuilder.addParameter("as_ylo", String.valueOf(scholarQueryTransformer.getStartYear())); uriBuilder.addParameter("as_yhi", String.valueOf(scholarQueryTransformer.getEndYear())); @@ -219,6 +220,14 @@ public Page performSearchPagedForTransformedQuery(String transformedQu @Override public AbstractQueryTransformer getQueryTransformer() { - return new ScholarQueryTransformer(); + if (Objects.isNull(transformer)) { + transformer = new ScholarQueryTransformer(); + } + return transformer; + } + + @Override + public void resetTransformer() { + transformer = null; } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java index 28d2d757c76..b008c975d34 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java @@ -6,14 +6,12 @@ import java.util.Collections; import java.util.List; import java.util.Optional; -import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.ParseException; import org.jabref.logic.importer.SearchBasedFetcher; -import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; import org.jabref.logic.importer.fileformat.BibtexParser; import org.jabref.logic.importer.util.GrobidService; import org.jabref.model.entry.BibEntry; @@ -78,13 +76,13 @@ public List performSearch(String searchQuery) throws FetcherException List collect; try { collect = Arrays.stream(searchQuery.split("\\r\\r+|\\n\\n+|\\r\\n(\\r\\n)+")) - .map(String::trim) - .filter(str -> !str.isBlank()) - .map(this::parseUsingGrobid) - .flatMap(Optional::stream) - .map(this::parseBibToBibEntry) - .flatMap(Optional::stream) - .collect(Collectors.toList()); + .map(String::trim) + .filter(str -> !str.isBlank()) + .map(this::parseUsingGrobid) + .flatMap(Optional::stream) + .map(this::parseBibToBibEntry) + .flatMap(Optional::stream) + .collect(Collectors.toList()); } catch (RuntimeException e) { throw new FetcherException(e.getMessage(), e.getCause()); } @@ -95,7 +93,7 @@ public List performSearch(String searchQuery) throws FetcherException * Not used */ @Override - public List performSearchForTransformedQuery(String transformedQuery, AbstractQueryTransformer transformer) throws FetcherException { + public List performSearchForTransformedQuery(String transformedQuery) throws FetcherException { return Collections.emptyList(); } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java b/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java index 17643a7e06c..2ee0a56f615 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java @@ -56,6 +56,7 @@ public class IEEE implements FulltextFetcher, PagedSearchBasedParserFetcher { private static final String API_KEY = new BuildInfo().ieeeAPIKey; private final ImportFormatPreferences preferences; + private IEEEQueryTransformer transformer; public IEEE(ImportFormatPreferences preferences) { this.preferences = Objects.requireNonNull(preferences); @@ -224,14 +225,14 @@ public Optional getHelpPage() { } @Override - public URL getURLForQuery(String transformedQuery, int pageNumber, AbstractQueryTransformer transformer) throws URISyntaxException, MalformedURLException { + public URL getURLForQuery(String transformedQuery, int pageNumber) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder("https://ieeexploreapi.ieee.org/api/v1/search/articles"); uriBuilder.addParameter("apikey", API_KEY); if (!transformedQuery.isBlank()) { uriBuilder.addParameter("querytext", transformedQuery); } uriBuilder.addParameter("max_records", String.valueOf(getPageSize())); - IEEEQueryTransformer ieeeQueryTransformer = ((IEEEQueryTransformer) transformer); + IEEEQueryTransformer ieeeQueryTransformer = transformer; // Currently not working as part of the query string if ((ieeeQueryTransformer).getJournal().isPresent()) { uriBuilder.addParameter("publication_title", ieeeQueryTransformer.getJournal().get()); @@ -255,6 +256,14 @@ public URL getURLForQuery(String transformedQuery, int pageNumber, AbstractQuery @Override public AbstractQueryTransformer getQueryTransformer() { - return new IEEEQueryTransformer(); + if (Objects.isNull(transformer)) { + transformer = new IEEEQueryTransformer(); + } + return transformer; + } + + @Override + public void resetTransformer() { + transformer = null; } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/MedlineFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/MedlineFetcher.java index 619c8c0a9d4..70fb08e1390 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/MedlineFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/MedlineFetcher.java @@ -28,7 +28,6 @@ import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.ParserResult; import org.jabref.logic.importer.SearchBasedFetcher; -import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; import org.jabref.logic.importer.fileformat.MedlineImporter; import org.jabref.logic.l10n.Localization; import org.jabref.model.entry.BibEntry; @@ -197,7 +196,7 @@ private List fetchMedline(List ids) throws FetcherException { } @Override - public List performSearchForTransformedQuery(String transformedQuery, AbstractQueryTransformer transformer) throws FetcherException { + public List performSearchForTransformedQuery(String transformedQuery) throws FetcherException { List entryList; if (transformedQuery.isBlank()) { diff --git a/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java index 985a819a62c..8939108e275 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java @@ -160,7 +160,7 @@ public Optional getHelpPage() { } @Override - public URL getURLForQuery(String transformedQuery, int pageNumber, AbstractQueryTransformer transformer) throws URISyntaxException, MalformedURLException { + public URL getURLForQuery(String transformedQuery, int pageNumber) throws URISyntaxException, MalformedURLException { URIBuilder uriBuilder = new URIBuilder(API_URL); uriBuilder.addParameter("q", transformedQuery); // Search query uriBuilder.addParameter("api_key", API_KEY); // API key diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformators/AbstractQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformators/AbstractQueryTransformer.java index 804c3dbef78..0e5ac366354 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/transformators/AbstractQueryTransformer.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformators/AbstractQueryTransformer.java @@ -13,7 +13,6 @@ import org.apache.lucene.queryparser.flexible.core.nodes.OrQueryNode; import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -23,7 +22,7 @@ */ public abstract class AbstractQueryTransformer { private static final Logger LOGGER = LoggerFactory.getLogger(AbstractQueryTransformer.class); - public static final String NO_EXPLICIT_FIELD = "default"; + private static final String NO_EXPLICIT_FIELD = "default"; /** * Transforms a and b and c to (a AND b AND c), where From 4b51151432d2867f85f40ba52a9663d2bc1741f1 Mon Sep 17 00:00:00 2001 From: Dominik Voigt Date: Sat, 16 Jan 2021 18:29:40 +0100 Subject: [PATCH 19/24] Change method structure according to suggestion. --- .../importer/PagedSearchBasedFetcher.java | 37 +++++++++++------- .../PagedSearchBasedParserFetcher.java | 22 +++++------ .../logic/importer/SearchBasedFetcher.java | 39 ++++++++----------- .../importer/SearchBasedParserFetcher.java | 14 ++++--- .../importer/fetcher/ACMPortalFetcher.java | 12 +++--- .../jabref/logic/importer/fetcher/ArXiv.java | 28 +++++-------- .../fetcher/AstrophysicsDataSystem.java | 14 ++++--- .../logic/importer/fetcher/CiteSeer.java | 7 ++-- ...fComputerScienceBibliographiesFetcher.java | 7 ++-- .../fetcher/CompositeSearchBasedFetcher.java | 10 ++--- .../logic/importer/fetcher/CrossRef.java | 7 ++-- .../logic/importer/fetcher/DBLPFetcher.java | 10 ++--- .../logic/importer/fetcher/DOAJFetcher.java | 7 ++-- .../logic/importer/fetcher/GoogleScholar.java | 26 +++---------- .../fetcher/GrobidCitationFetcher.java | 3 +- .../logic/importer/fetcher/GvkFetcher.java | 11 ++---- .../jabref/logic/importer/fetcher/IEEE.java | 38 +++++++----------- .../importer/fetcher/INSPIREFetcher.java | 7 ++-- .../logic/importer/fetcher/JstorFetcher.java | 7 ++-- .../logic/importer/fetcher/MathSciNet.java | 7 ++-- .../importer/fetcher/MedlineFetcher.java | 10 +++-- .../importer/fetcher/SpringerFetcher.java | 12 ++---- .../jabref/logic/importer/fetcher/ZbMATH.java | 11 ++---- .../AbstractQueryTransformer.java | 26 +++++-------- ...puterScienceBibliographiesFetcherTest.java | 8 +++- .../importer/fetcher/GvkFetcherTest.java | 14 +++---- .../ArXivQueryTransformerTest.java | 13 +++++-- .../DBLPQueryTransformerTest.java | 11 ++++-- .../GVKQueryTransformerTest.java | 7 +++- .../IEEEQueryTransformerTest.java | 17 ++++++-- .../transformators/InfixTransformerTest.java | 35 +++++++++++------ .../ScholarQueryTransformerTest.java | 12 +++++- .../SpringerQueryTransformerTest.java | 13 +++++-- .../ZbMathQueryTransformerTest.java | 11 ++++-- 34 files changed, 265 insertions(+), 248 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java b/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java index 99a21031bee..c83ea820703 100644 --- a/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java +++ b/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java @@ -3,24 +3,27 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Optional; import org.jabref.logic.JabRefException; -import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; import org.jabref.model.entry.BibEntry; import org.jabref.model.paging.Page; +import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; +import org.apache.lucene.queryparser.flexible.core.parser.SyntaxParser; +import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; + public interface PagedSearchBasedFetcher extends SearchBasedFetcher { /** - * @param transformedQuery the complex query defining all fielded search parameters + * @param luceneQuery the root node of the lucene query * @param pageNumber requested site number indexed from 0 * @return Page with search results */ - Page performSearchPagedForTransformedQuery(String transformedQuery, int pageNumber) throws FetcherException; + Page performSearchPaged(QueryNode luceneQuery, int pageNumber) throws FetcherException; /** - * @param searchQuery query string that can be parsed into a complex search query + * @param searchQuery query string that can be parsed into a lucene query * @param pageNumber requested site number indexed from 0 * @return Page with search results */ @@ -28,11 +31,13 @@ default Page performSearchPaged(String searchQuery, int pageNumber) th if (searchQuery.isBlank()) { return new Page<>(searchQuery, pageNumber, Collections.emptyList()); } - resetTransformer(); - AbstractQueryTransformer transformer = getQueryTransformer(); - Optional transformedQuery = transformer.parseQueryStringIntoComplexQuery(searchQuery); - // Otherwise just use query as a default term - return this.performSearchPagedForTransformedQuery(transformedQuery.orElse(""), pageNumber); + SyntaxParser parser = new StandardSyntaxParser(); + final String NO_EXPLICIT_FIELD = "default"; + try { + return this.performSearchPaged(parser.parse(searchQuery, NO_EXPLICIT_FIELD), pageNumber); + } catch (QueryNodeParseException e) { + throw new FetcherException("An error occurred during parsing of the query."); + } } /** @@ -42,8 +47,14 @@ default int getPageSize() { return 20; } - @Override - default List performSearchForTransformedQuery(String transformedQuery) throws FetcherException { - return new ArrayList<>(performSearchPagedForTransformedQuery(transformedQuery, 0).getContent()); + /** + * This method is used to send complex queries using fielded search. + * + * @param luceneQuery the root node of the lucene query + * @return a list of {@link BibEntry}, which are matched by the query (may be empty) + */ + default List performSearch(QueryNode luceneQuery) throws FetcherException { + return new ArrayList<>(performSearchPaged(luceneQuery, 0).getContent()); } + } diff --git a/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java b/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java index cba0b4fbe15..bbbc848cc99 100644 --- a/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java +++ b/src/main/java/org/jabref/logic/importer/PagedSearchBasedParserFetcher.java @@ -5,24 +5,25 @@ import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; -import java.util.ArrayList; import java.util.List; import org.jabref.model.entry.BibEntry; import org.jabref.model.paging.Page; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; + public interface PagedSearchBasedParserFetcher extends SearchBasedParserFetcher, PagedSearchBasedFetcher { @Override - default Page performSearchPagedForTransformedQuery(String transformedQuery, int pageNumber) throws FetcherException { + default Page performSearchPaged(QueryNode luceneQuery, int pageNumber) throws FetcherException { // ADR-0014 URL urlForQuery; try { - urlForQuery = getURLForQuery(transformedQuery, pageNumber); + urlForQuery = getURLForQuery(luceneQuery, pageNumber); } catch (URISyntaxException | MalformedURLException e) { throw new FetcherException("Search URI crafted from complex search query is malformed", e); } - return new Page<>(transformedQuery, pageNumber, getBibEntries(urlForQuery)); + return new Page<>(luceneQuery.toString(), pageNumber, getBibEntries(urlForQuery)); } private List getBibEntries(URL urlForQuery) throws FetcherException { @@ -39,19 +40,18 @@ private List getBibEntries(URL urlForQuery) throws FetcherException { /** * Constructs a URL based on the query, size and page number. - * - * @param transformedQuery the search query + * @param luceneQuery the search query * @param pageNumber the number of the page indexed from 0 */ - URL getURLForQuery(String transformedQuery, int pageNumber) throws URISyntaxException, MalformedURLException; + URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException, FetcherException; @Override - default URL getURLForQuery(String transformedQuery) throws URISyntaxException, MalformedURLException, FetcherException { - return getURLForQuery(transformedQuery, 0); + default URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { + return getURLForQuery(luceneQuery, 0); } @Override - default List performSearchForTransformedQuery(String transformedQuery) throws FetcherException { - return new ArrayList<>(performSearchPagedForTransformedQuery(transformedQuery, 0).getContent()); + default List performSearch(QueryNode luceneQuery) throws FetcherException { + return SearchBasedParserFetcher.super.performSearch(luceneQuery); } } diff --git a/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java b/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java index a343ab4cc3c..9c33fe27ba0 100644 --- a/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java +++ b/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java @@ -2,13 +2,17 @@ import java.util.Collections; import java.util.List; -import java.util.Optional; import org.jabref.logic.JabRefException; -import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; -import org.jabref.logic.importer.fetcher.transformators.DefaultQueryTransformer; import org.jabref.model.entry.BibEntry; +import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; +import org.apache.lucene.queryparser.flexible.core.parser.SyntaxParser; +import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; + +import static org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer.NO_EXPLICIT_FIELD; + /** * Searches web resources for bibliographic information based on a free-text query. * May return multiple search hits. @@ -18,38 +22,27 @@ public interface SearchBasedFetcher extends WebFetcher { /** * This method is used to send complex queries using fielded search. * - * @param transformedQuery the search query defining all fielded search parameters + * @param luceneQuery the root node of the lucene query * @return a list of {@link BibEntry}, which are matched by the query (may be empty) */ - List performSearchForTransformedQuery(String transformedQuery) throws FetcherException; + List performSearch(QueryNode luceneQuery) throws FetcherException; /** * Looks for hits which are matched by the given free-text query. * - * @param searchQuery query string that can be parsed into a complex search query + * @param searchQuery query string that can be parsed into a lucene query * @return a list of {@link BibEntry}, which are matched by the query (may be empty) */ - default List performSearch(String searchQuery) throws JabRefException { - resetTransformer(); - AbstractQueryTransformer transformer = getQueryTransformer(); + default List performSearch(String searchQuery) throws FetcherException { if (searchQuery.isBlank()) { return Collections.emptyList(); } - Optional transformedQuery; + SyntaxParser parser = new StandardSyntaxParser(); + try { - transformedQuery = transformer.parseQueryStringIntoComplexQuery(searchQuery); - } catch (JabRefException e) { - throw new FetcherException("Error occured during query transformation", e); + return this.performSearch(parser.parse(searchQuery, NO_EXPLICIT_FIELD)); + } catch (QueryNodeParseException e) { + throw new FetcherException("An error occured when parsing the query"); } - // Otherwise just use query as a default term - return this.performSearchForTransformedQuery(transformedQuery.orElse("")); - } - - default AbstractQueryTransformer getQueryTransformer() { - return new DefaultQueryTransformer(); - } - - default void resetTransformer() { - } } diff --git a/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java b/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java index c99e2b5712c..041540a6bac 100644 --- a/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java +++ b/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java @@ -11,6 +11,8 @@ import org.jabref.logic.cleanup.Formatter; import org.jabref.model.entry.BibEntry; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; + /** * Provides a convenient interface for search-based fetcher, which follow the usual three-step procedure: *
    @@ -26,14 +28,14 @@ public interface SearchBasedParserFetcher extends SearchBasedFetcher { * This method is necessary as the performSearch method does not support certain URL parameters that are used for * fielded search, such as a title, author, or year parameter. * - * @param transformedQuery the search query defining all fielded search parameters + * @param luceneQuery the root node of the lucene query */ @Override - default List performSearchForTransformedQuery(String transformedQuery) throws FetcherException { + default List performSearch(QueryNode luceneQuery) throws FetcherException { // ADR-0014 URL urlForQuery; try { - urlForQuery = getURLForQuery(transformedQuery); + urlForQuery = getURLForQuery(luceneQuery); } catch (URISyntaxException | MalformedURLException | JabRefException e) { throw new FetcherException("Search URI crafted from complex search query is malformed", e); } @@ -58,11 +60,11 @@ private List getBibEntries(URL urlForQuery) throws FetcherException { Parser getParser(); /** - * Constructs a URL based on the query. + * Constructs a URL based on the lucene query. * - * @param transformedQuery the search query + * @param luceneQuery the root node of the lucene query */ - URL getURLForQuery(String transformedQuery) throws URISyntaxException, MalformedURLException, FetcherException; + URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException; /** * Performs a cleanup of the fetched entry. diff --git a/src/main/java/org/jabref/logic/importer/fetcher/ACMPortalFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/ACMPortalFetcher.java index 5319b1a873f..d15e555e461 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/ACMPortalFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/ACMPortalFetcher.java @@ -11,11 +11,12 @@ import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; -import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; +import org.jabref.logic.importer.fetcher.transformators.DefaultQueryTransformer; import org.jabref.logic.importer.fileformat.BibtexParser; import org.jabref.model.util.DummyFileUpdateMonitor; import org.apache.http.client.utils.URIBuilder; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; public class ACMPortalFetcher implements SearchBasedParserFetcher { @@ -37,15 +38,16 @@ public Optional getHelpPage() { return Optional.of(HelpFile.FETCHER_ACM); } - private static String createQueryString(String query) { + private static String createQueryString(QueryNode query) throws FetcherException { + String queryString = new DefaultQueryTransformer().transformLuceneQuery(query).orElse(""); // Query syntax to search for an entry that matches "one" and "two" in any field is: (+one +two) - return "(%252B" + query.trim().replaceAll("\\s+", "%20%252B") + ")"; + return "(%252B" + queryString.trim().replaceAll("\\s+", "%20%252B") + ")"; } @Override - public URL getURLForQuery(String transformedQuery, AbstractQueryTransformer transformer) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder(SEARCH_URL); - uriBuilder.addParameter("query", createQueryString(transformedQuery)); // Search all fields + uriBuilder.addParameter("query", createQueryString(luceneQuery)); // Search all fields uriBuilder.addParameter("within", "owners.owner=GUIDE"); // Search within the ACM Guide to Computing Literature (encompasses the ACM Full-Text Collection) uriBuilder.addParameter("expformat", "bibtex"); // BibTeX format return uriBuilder.build().toURL(); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/ArXiv.java b/src/main/java/org/jabref/logic/importer/fetcher/ArXiv.java index cd2c364a4ab..e0569146721 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/ArXiv.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/ArXiv.java @@ -23,7 +23,6 @@ import org.jabref.logic.importer.IdFetcher; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.PagedSearchBasedFetcher; -import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; import org.jabref.logic.importer.fetcher.transformators.ArXivQueryTransformer; import org.jabref.logic.util.io.XMLUtil; import org.jabref.logic.util.strings.StringSimilarity; @@ -38,6 +37,7 @@ import org.jabref.model.util.OptionalUtil; import org.apache.http.client.utils.URIBuilder; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.w3c.dom.Document; @@ -62,7 +62,6 @@ public class ArXiv implements FulltextFetcher, PagedSearchBasedFetcher, IdBasedF private static final String API_URL = "https://export.arxiv.org/api/query"; private final ImportFormatPreferences importFormatPreferences; - private ArXivQueryTransformer transformer; public ArXiv(ImportFormatPreferences importFormatPreferences) { this.importFormatPreferences = importFormatPreferences; @@ -255,24 +254,25 @@ public Optional getHelpPage() { /** * Constructs a complex query string using the field prefixes specified at https://arxiv.org/help/api/user-manual * - * @param transformedQuery the search query defining all fielded search parameters + * @param luceneQuery the root node of the lucene query * @return A list of entries matching the complex query */ @Override - public Page performSearchPagedForTransformedQuery(String transformedQuery, int pageNumber) throws FetcherException { + public Page performSearchPaged(QueryNode luceneQuery, int pageNumber) throws FetcherException { + ArXivQueryTransformer transformer = new ArXivQueryTransformer(); + String transformedQuery = transformer.transformLuceneQuery(luceneQuery).orElse(""); List searchResult = searchForEntries(transformedQuery, pageNumber).stream() .map((arXivEntry) -> arXivEntry.toBibEntry(importFormatPreferences.getKeywordSeparator())) .collect(Collectors.toList()); - return new Page<>(transformedQuery, pageNumber, filterYears(searchResult)); + return new Page<>(transformedQuery, pageNumber, filterYears(searchResult, transformer)); } - private List filterYears(List searchResult) { - ArXivQueryTransformer arXivQueryTransformer = transformer; + private List filterYears(List searchResult, ArXivQueryTransformer transformer) { return searchResult.stream() .filter(entry -> entry.getField(StandardField.DATE).isPresent()) // Filter the date field for year only - .filter(entry -> !arXivQueryTransformer.getEndYear().isPresent() || Integer.parseInt(entry.getField(StandardField.DATE).get().substring(0, 4)) <= arXivQueryTransformer.getEndYear().get()) - .filter(entry -> !arXivQueryTransformer.getStartYear().isPresent() || Integer.parseInt(entry.getField(StandardField.DATE).get().substring(0, 4)) >= arXivQueryTransformer.getStartYear().get()) + .filter(entry -> transformer.getEndYear().isEmpty() || Integer.parseInt(entry.getField(StandardField.DATE).get().substring(0, 4)) <= transformer.getEndYear().get()) + .filter(entry -> transformer.getStartYear().isEmpty() || Integer.parseInt(entry.getField(StandardField.DATE).get().substring(0, 4)) >= transformer.getStartYear().get()) .collect(Collectors.toList()); } @@ -422,14 +422,4 @@ public BibEntry toBibEntry(Character keywordDelimiter) { return bibEntry; } } - - @Override - public AbstractQueryTransformer getQueryTransformer() { - return new ArXivQueryTransformer(); - } - - @Override - public void resetTransformer() { - transformer = null; - } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java b/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java index 33a81285135..6da8f0920d3 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/AstrophysicsDataSystem.java @@ -27,6 +27,7 @@ import org.jabref.logic.importer.PagedSearchBasedParserFetcher; import org.jabref.logic.importer.ParseException; import org.jabref.logic.importer.Parser; +import org.jabref.logic.importer.fetcher.transformators.DefaultQueryTransformer; import org.jabref.logic.importer.fileformat.BibtexParser; import org.jabref.logic.net.URLDownload; import org.jabref.logic.util.BuildInfo; @@ -41,6 +42,7 @@ import kong.unirest.json.JSONException; import kong.unirest.json.JSONObject; import org.apache.http.client.utils.URIBuilder; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; /** * Fetches data from the SAO/NASA Astrophysics Data System (https://ui.adsabs.harvard.edu/) @@ -79,13 +81,13 @@ public String getName() { } /** - * @param transformedQuery query string, matching the apache solr format + * @param luceneQuery query string, matching the apache solr format * @return URL which points to a search request for given query */ @Override - public URL getURLForQuery(String transformedQuery, int pageNumber) throws URISyntaxException, MalformedURLException { + public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder builder = new URIBuilder(API_SEARCH_URL); - builder.addParameter("q", transformedQuery); + builder.addParameter("q", new DefaultQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); builder.addParameter("fl", "bibcode"); builder.addParameter("rows", String.valueOf(getPageSize())); builder.addParameter("start", String.valueOf(getPageSize() * pageNumber)); @@ -274,12 +276,12 @@ private List performSearchByIds(Collection identifiers) throws } @Override - public Page performSearchPagedForTransformedQuery(String transformedQuery, int pageNumber) throws FetcherException { + public Page performSearchPaged(QueryNode luceneQuery, int pageNumber) throws FetcherException { try { // This is currently just interpreting the complex query as a default string query - List bibcodes = fetchBibcodes(getURLForQuery(transformedQuery, pageNumber)); + List bibcodes = fetchBibcodes(getURLForQuery(luceneQuery, pageNumber)); Collection results = performSearchByIds(bibcodes); - return new Page<>(transformedQuery, pageNumber, results); + return new Page<>(luceneQuery.toString(), pageNumber, results); } catch (URISyntaxException e) { throw new FetcherException("Search URI is malformed", e); } catch (IOException e) { diff --git a/src/main/java/org/jabref/logic/importer/fetcher/CiteSeer.java b/src/main/java/org/jabref/logic/importer/fetcher/CiteSeer.java index 41af880cf20..c6e4c2b6ba5 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/CiteSeer.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/CiteSeer.java @@ -20,7 +20,7 @@ import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; -import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; +import org.jabref.logic.importer.fetcher.transformators.DefaultQueryTransformer; import org.jabref.logic.importer.fileformat.CoinsParser; import org.jabref.logic.util.OS; import org.jabref.model.entry.BibEntry; @@ -28,6 +28,7 @@ import org.jabref.model.entry.field.StandardField; import org.apache.http.client.utils.URIBuilder; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; public class CiteSeer implements SearchBasedParserFetcher { @@ -45,10 +46,10 @@ public Optional getHelpPage() { } @Override - public URL getURLForQuery(String transformedQuery, AbstractQueryTransformer transformer) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder("https://citeseer.ist.psu.edu/search"); uriBuilder.addParameter("sort", "rlv"); // Sort by relevance - uriBuilder.addParameter("q", transformedQuery); // Query + uriBuilder.addParameter("q", new DefaultQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); // Query uriBuilder.addParameter("t", "doc"); // Type: documents // uriBuilder.addParameter("start", "0"); // Start index (not supported at the moment) return uriBuilder.build().toURL(); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcher.java index 757fb212221..b6556e0930f 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcher.java @@ -14,7 +14,7 @@ import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; -import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; +import org.jabref.logic.importer.fetcher.transformators.DefaultQueryTransformer; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.Field; import org.jabref.model.entry.field.FieldFactory; @@ -22,6 +22,7 @@ import org.jabref.model.entry.field.UnknownField; import org.apache.http.client.utils.URIBuilder; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; public class CollectionOfComputerScienceBibliographiesFetcher implements SearchBasedParserFetcher { @@ -34,9 +35,9 @@ public CollectionOfComputerScienceBibliographiesFetcher(ImportFormatPreferences } @Override - public URL getURLForQuery(String transformedQuery, AbstractQueryTransformer transformer) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { return new URIBuilder(BASIC_SEARCH_URL) - .addParameter("query", transformedQuery) + .addParameter("query", new DefaultQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")) .addParameter("sort", "score") .build() .toURL(); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcher.java index 9cfb1a614b4..2de2344b304 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcher.java @@ -14,6 +14,7 @@ import org.jabref.model.database.BibDatabaseMode; import org.jabref.model.entry.BibEntry; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,13 +48,13 @@ public Optional getHelpPage() { } @Override - public List performSearchForTransformedQuery(String transformedQuery) throws FetcherException { + public List performSearch(QueryNode luceneQuery) throws FetcherException { ImportCleanup cleanup = new ImportCleanup(BibDatabaseMode.BIBTEX); // All entries have to be converted into one format, this is necessary for the format conversion return fetchers.parallelStream() .flatMap(searchBasedFetcher -> { try { - return searchBasedFetcher.performSearch(transformedQuery).stream(); + return searchBasedFetcher.performSearch(luceneQuery).stream(); } catch (JabRefException e) { LOGGER.warn(String.format("%s API request failed", searchBasedFetcher.getName()), e); return Stream.empty(); @@ -63,9 +64,4 @@ public List performSearchForTransformedQuery(String transformedQuery) .map(cleanup::doPostCleanup) .collect(Collectors.toList()); } - - @Override - public void resetTransformer() { - fetchers.forEach(SearchBasedFetcher::resetTransformer); - } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/CrossRef.java b/src/main/java/org/jabref/logic/importer/fetcher/CrossRef.java index 55014ee0570..b644c56659d 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/CrossRef.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/CrossRef.java @@ -18,7 +18,7 @@ import org.jabref.logic.importer.ParseException; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; -import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; +import org.jabref.logic.importer.fetcher.transformators.DefaultQueryTransformer; import org.jabref.logic.importer.util.JsonReader; import org.jabref.logic.util.strings.StringSimilarity; import org.jabref.model.entry.AuthorList; @@ -33,6 +33,7 @@ import kong.unirest.json.JSONException; import kong.unirest.json.JSONObject; import org.apache.http.client.utils.URIBuilder; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; /** * A class for fetching DOIs from CrossRef @@ -64,9 +65,9 @@ public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedUR } @Override - public URL getURLForQuery(String transformedQuery, AbstractQueryTransformer transformer) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder(API_URL); - uriBuilder.addParameter("query", transformedQuery); + uriBuilder.addParameter("query", new DefaultQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); return uriBuilder.build().toURL(); } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/DBLPFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/DBLPFetcher.java index 75e95789da9..54776ccfa24 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/DBLPFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/DBLPFetcher.java @@ -26,6 +26,7 @@ import org.jabref.model.util.DummyFileUpdateMonitor; import org.apache.http.client.utils.URIBuilder; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; /** * Fetches BibTeX data from DBLP (dblp.org) @@ -44,9 +45,9 @@ public DBLPFetcher(ImportFormatPreferences importFormatPreferences) { } @Override - public URL getURLForQuery(String transformedQuery, AbstractQueryTransformer transformer) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder(BASIC_SEARCH_URL); - uriBuilder.addParameter("q", transformedQuery); + uriBuilder.addParameter("q", new DBLPQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); uriBuilder.addParameter("h", String.valueOf(100)); // number of hits uriBuilder.addParameter("c", String.valueOf(0)); // no need for auto-completion uriBuilder.addParameter("f", String.valueOf(0)); // "from", index of first hit to download @@ -83,9 +84,4 @@ public String getName() { public Optional getHelpPage() { return Optional.of(HelpFile.FETCHER_DBLP); } - - @Override - public AbstractQueryTransformer getQueryTransformer() { - return new DBLPQueryTransformer(); - } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java index cbfa2c728cf..5f62756cdf7 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/DOAJFetcher.java @@ -16,7 +16,7 @@ import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; -import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; +import org.jabref.logic.importer.fetcher.transformators.DefaultQueryTransformer; import org.jabref.logic.util.OS; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.Field; @@ -27,6 +27,7 @@ import kong.unirest.json.JSONArray; import kong.unirest.json.JSONObject; import org.apache.http.client.utils.URIBuilder; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -189,9 +190,9 @@ public Optional getHelpPage() { } @Override - public URL getURLForQuery(String transformedQuery, AbstractQueryTransformer transformer) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder(SEARCH_URL); - DOAJFetcher.addPath(uriBuilder, transformedQuery); + DOAJFetcher.addPath(uriBuilder, new DefaultQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); // Number of results uriBuilder.addParameter("pageSize", "30"); // Page (not needed so far) diff --git a/src/main/java/org/jabref/logic/importer/fetcher/GoogleScholar.java b/src/main/java/org/jabref/logic/importer/fetcher/GoogleScholar.java index 9a7846d52f9..ffd4f9a55e0 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/GoogleScholar.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/GoogleScholar.java @@ -19,7 +19,6 @@ import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.PagedSearchBasedFetcher; import org.jabref.logic.importer.ParserResult; -import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; import org.jabref.logic.importer.fetcher.transformators.ScholarQueryTransformer; import org.jabref.logic.importer.fileformat.BibtexParser; import org.jabref.logic.l10n.Localization; @@ -30,6 +29,7 @@ import org.jabref.model.util.DummyFileUpdateMonitor; import org.apache.http.client.utils.URIBuilder; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.select.Elements; @@ -51,7 +51,6 @@ public class GoogleScholar implements FulltextFetcher, PagedSearchBasedFetcher { private static final int NUM_RESULTS = 10; private final ImportFormatPreferences importFormatPreferences; - private ScholarQueryTransformer transformer; public GoogleScholar(ImportFormatPreferences importFormatPreferences) { Objects.requireNonNull(importFormatPreferences); @@ -178,20 +177,20 @@ private void obtainAndModifyCookie() throws FetcherException { } @Override - public Page performSearchPagedForTransformedQuery(String transformedQuery, int pageNumber) throws FetcherException { + public Page performSearchPaged(QueryNode luceneQuery, int pageNumber) throws FetcherException { + ScholarQueryTransformer queryTransformer = new ScholarQueryTransformer(); + String transformedQuery = queryTransformer.transformLuceneQuery(luceneQuery).orElse(""); try { obtainAndModifyCookie(); List foundEntries = new ArrayList<>(10); - URIBuilder uriBuilder = new URIBuilder(BASIC_SEARCH_URL); uriBuilder.addParameter("hl", "en"); uriBuilder.addParameter("btnG", "Search"); uriBuilder.addParameter("q", transformedQuery); uriBuilder.addParameter("start", String.valueOf(pageNumber * getPageSize())); uriBuilder.addParameter("num", String.valueOf(getPageSize())); - ScholarQueryTransformer scholarQueryTransformer = transformer; - uriBuilder.addParameter("as_ylo", String.valueOf(scholarQueryTransformer.getStartYear())); - uriBuilder.addParameter("as_yhi", String.valueOf(scholarQueryTransformer.getEndYear())); + uriBuilder.addParameter("as_ylo", String.valueOf(queryTransformer.getStartYear())); + uriBuilder.addParameter("as_yhi", String.valueOf(queryTransformer.getEndYear())); try { addHitsFromQuery(foundEntries, uriBuilder.toString()); @@ -217,17 +216,4 @@ public Page performSearchPagedForTransformedQuery(String transformedQu throw new FetcherException("Error while fetching from " + getName(), e); } } - - @Override - public AbstractQueryTransformer getQueryTransformer() { - if (Objects.isNull(transformer)) { - transformer = new ScholarQueryTransformer(); - } - return transformer; - } - - @Override - public void resetTransformer() { - transformer = null; - } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java index b008c975d34..bf16d71570d 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/GrobidCitationFetcher.java @@ -17,6 +17,7 @@ import org.jabref.model.entry.BibEntry; import org.jabref.model.util.DummyFileUpdateMonitor; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -93,7 +94,7 @@ public List performSearch(String searchQuery) throws FetcherException * Not used */ @Override - public List performSearchForTransformedQuery(String transformedQuery) throws FetcherException { + public List performSearch(QueryNode luceneQuery) throws FetcherException { return Collections.emptyList(); } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/GvkFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/GvkFetcher.java index 2a6c953d280..db6afbc6606 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/GvkFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/GvkFetcher.java @@ -11,11 +11,11 @@ import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; -import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; import org.jabref.logic.importer.fetcher.transformators.GVKQueryTransformer; import org.jabref.logic.importer.fileformat.GvkParser; import org.apache.http.client.utils.URIBuilder; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; public class GvkFetcher implements SearchBasedParserFetcher { @@ -38,11 +38,11 @@ public Optional getHelpPage() { } @Override - public URL getURLForQuery(String transformedQuery, AbstractQueryTransformer transformer) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder(URL_PATTERN); uriBuilder.addParameter("version", "1.1"); uriBuilder.addParameter("operation", "searchRetrieve"); - uriBuilder.addParameter("query", transformedQuery); + uriBuilder.addParameter("query", new GVKQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); uriBuilder.addParameter("maximumRecords", "50"); uriBuilder.addParameter("recordSchema", "picaxml"); uriBuilder.addParameter("sortKeys", "Year,,1"); @@ -53,9 +53,4 @@ public URL getURLForQuery(String transformedQuery, AbstractQueryTransformer tran public Parser getParser() { return new GvkParser(); } - - @Override - public AbstractQueryTransformer getQueryTransformer() { - return new GVKQueryTransformer(); - } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java b/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java index 2ee0a56f615..9cd5341d357 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/IEEE.java @@ -15,11 +15,11 @@ import java.util.stream.Collectors; import org.jabref.logic.help.HelpFile; +import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.FulltextFetcher; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.PagedSearchBasedParserFetcher; import org.jabref.logic.importer.Parser; -import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; import org.jabref.logic.importer.fetcher.transformators.IEEEQueryTransformer; import org.jabref.logic.net.URLDownload; import org.jabref.logic.util.BuildInfo; @@ -33,6 +33,7 @@ import kong.unirest.json.JSONArray; import kong.unirest.json.JSONObject; import org.apache.http.client.utils.URIBuilder; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -56,7 +57,6 @@ public class IEEE implements FulltextFetcher, PagedSearchBasedParserFetcher { private static final String API_KEY = new BuildInfo().ieeeAPIKey; private final ImportFormatPreferences preferences; - private IEEEQueryTransformer transformer; public IEEE(ImportFormatPreferences preferences) { this.preferences = Objects.requireNonNull(preferences); @@ -225,26 +225,27 @@ public Optional getHelpPage() { } @Override - public URL getURLForQuery(String transformedQuery, int pageNumber) throws URISyntaxException, MalformedURLException { + public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException, FetcherException { + IEEEQueryTransformer transformer = new IEEEQueryTransformer(); + String transformedQuery = transformer.transformLuceneQuery(luceneQuery).orElse(""); URIBuilder uriBuilder = new URIBuilder("https://ieeexploreapi.ieee.org/api/v1/search/articles"); uriBuilder.addParameter("apikey", API_KEY); if (!transformedQuery.isBlank()) { uriBuilder.addParameter("querytext", transformedQuery); } uriBuilder.addParameter("max_records", String.valueOf(getPageSize())); - IEEEQueryTransformer ieeeQueryTransformer = transformer; // Currently not working as part of the query string - if ((ieeeQueryTransformer).getJournal().isPresent()) { - uriBuilder.addParameter("publication_title", ieeeQueryTransformer.getJournal().get()); + if (transformer.getJournal().isPresent()) { + uriBuilder.addParameter("publication_title", transformer.getJournal().get()); } - if (ieeeQueryTransformer.getStartYear().isPresent()) { - uriBuilder.addParameter("start_year", String.valueOf(ieeeQueryTransformer.getStartYear().get())); + if (transformer.getStartYear().isPresent()) { + uriBuilder.addParameter("start_year", String.valueOf(transformer.getStartYear().get())); } - if (ieeeQueryTransformer.getEndYear().isPresent()) { - uriBuilder.addParameter("end_year", String.valueOf(ieeeQueryTransformer.getEndYear().get())); + if (transformer.getEndYear().isPresent()) { + uriBuilder.addParameter("end_year", String.valueOf(transformer.getEndYear().get())); } - if (ieeeQueryTransformer.getArticleNumber().isPresent()) { - uriBuilder.addParameter("article_number", String.valueOf(ieeeQueryTransformer.getArticleNumber().get())); + if (transformer.getArticleNumber().isPresent()) { + uriBuilder.addParameter("article_number", transformer.getArticleNumber().get()); } // Starts to index at 1 for the first entry uriBuilder.addParameter("start_record", String.valueOf(getPageSize() * pageNumber) + 1); @@ -253,17 +254,4 @@ public URL getURLForQuery(String transformedQuery, int pageNumber) throws URISyn return uriBuilder.build().toURL(); } - - @Override - public AbstractQueryTransformer getQueryTransformer() { - if (Objects.isNull(transformer)) { - transformer = new IEEEQueryTransformer(); - } - return transformer; - } - - @Override - public void resetTransformer() { - transformer = null; - } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/INSPIREFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/INSPIREFetcher.java index c8203e5c996..c594ebb5efc 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/INSPIREFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/INSPIREFetcher.java @@ -13,7 +13,7 @@ import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; -import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; +import org.jabref.logic.importer.fetcher.transformators.DefaultQueryTransformer; import org.jabref.logic.importer.fileformat.BibtexParser; import org.jabref.logic.importer.util.MediaTypes; import org.jabref.logic.layout.format.LatexToUnicodeFormatter; @@ -24,6 +24,7 @@ import org.jabref.model.util.DummyFileUpdateMonitor; import org.apache.http.client.utils.URIBuilder; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; /** * Fetches data from the INSPIRE database. @@ -49,9 +50,9 @@ public Optional getHelpPage() { } @Override - public URL getURLForQuery(String transformedQuery, AbstractQueryTransformer transformer) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder(INSPIRE_HOST); - uriBuilder.addParameter("q", transformedQuery); // Query + uriBuilder.addParameter("q", new DefaultQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); // Query return uriBuilder.build().toURL(); } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/JstorFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/JstorFetcher.java index 36e3a7b8c19..4c2f49a16d5 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/JstorFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/JstorFetcher.java @@ -19,7 +19,7 @@ import org.jabref.logic.importer.ParseException; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; -import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; +import org.jabref.logic.importer.fetcher.transformators.DefaultQueryTransformer; import org.jabref.logic.importer.fileformat.BibtexParser; import org.jabref.logic.net.URLDownload; import org.jabref.model.entry.BibEntry; @@ -27,6 +27,7 @@ import org.jabref.model.util.DummyFileUpdateMonitor; import org.apache.http.client.utils.URIBuilder; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.jsoup.Jsoup; import org.jsoup.nodes.Document; import org.jsoup.nodes.Element; @@ -48,9 +49,9 @@ public JstorFetcher(ImportFormatPreferences importFormatPreferences) { } @Override - public URL getURLForQuery(String transformedQuery, AbstractQueryTransformer transformer) throws URISyntaxException, MalformedURLException { + public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder(SEARCH_HOST); - uriBuilder.addParameter("Query", transformedQuery); + uriBuilder.addParameter("Query", new DefaultQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); return uriBuilder.build().toURL(); } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/MathSciNet.java b/src/main/java/org/jabref/logic/importer/fetcher/MathSciNet.java index 57ed8e67d5b..bfc2ca3ab8c 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/MathSciNet.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/MathSciNet.java @@ -23,7 +23,7 @@ import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; -import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; +import org.jabref.logic.importer.fetcher.transformators.DefaultQueryTransformer; import org.jabref.logic.importer.fileformat.BibtexParser; import org.jabref.logic.util.OS; import org.jabref.model.entry.BibEntry; @@ -32,6 +32,7 @@ import org.jabref.model.util.DummyFileUpdateMonitor; import org.apache.http.client.utils.URIBuilder; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; /** * Fetches data from the MathSciNet (http://www.ams.org/mathscinet) @@ -73,10 +74,10 @@ public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedUR } @Override - public URL getURLForQuery(String transformedQuery, AbstractQueryTransformer transformer) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder("https://mathscinet.ams.org/mathscinet/search/publications.html"); uriBuilder.addParameter("pg7", "ALLF"); // search all fields - uriBuilder.addParameter("s7", transformedQuery); // query + uriBuilder.addParameter("s7", new DefaultQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); // query uriBuilder.addParameter("r", "1"); // start index uriBuilder.addParameter("extend", "1"); // should return up to 100 items (instead of default 10) uriBuilder.addParameter("fmt", "bibtex"); // BibTeX format diff --git a/src/main/java/org/jabref/logic/importer/fetcher/MedlineFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/MedlineFetcher.java index 70fb08e1390..304b76da38a 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/MedlineFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/MedlineFetcher.java @@ -28,6 +28,7 @@ import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.ParserResult; import org.jabref.logic.importer.SearchBasedFetcher; +import org.jabref.logic.importer.fetcher.transformators.DefaultQueryTransformer; import org.jabref.logic.importer.fileformat.MedlineImporter; import org.jabref.logic.l10n.Localization; import org.jabref.model.entry.BibEntry; @@ -35,6 +36,7 @@ import org.jabref.model.entry.field.UnknownField; import org.apache.http.client.utils.URIBuilder; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -196,14 +198,16 @@ private List fetchMedline(List ids) throws FetcherException { } @Override - public List performSearchForTransformedQuery(String transformedQuery) throws FetcherException { + public List performSearch(QueryNode luceneQuery) throws FetcherException { List entryList; + DefaultQueryTransformer transformer = new DefaultQueryTransformer(); + Optional transformedQuery = transformer.transformLuceneQuery(luceneQuery); - if (transformedQuery.isBlank()) { + if (transformedQuery.isEmpty() || transformedQuery.get().isBlank()) { return Collections.emptyList(); } else { // searching for pubmed ids matching the query - List idList = getPubMedIdsFromQuery(transformedQuery); + List idList = getPubMedIdsFromQuery(transformedQuery.get()); if (idList.isEmpty()) { LOGGER.info("No results found."); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java index 8939108e275..5c3b34b6d01 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/SpringerFetcher.java @@ -11,9 +11,9 @@ import java.util.stream.Collectors; import org.jabref.logic.help.HelpFile; +import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.PagedSearchBasedParserFetcher; import org.jabref.logic.importer.Parser; -import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; import org.jabref.logic.importer.fetcher.transformators.SpringerQueryTransformer; import org.jabref.logic.util.BuildInfo; import org.jabref.logic.util.OS; @@ -27,6 +27,7 @@ import kong.unirest.json.JSONArray; import kong.unirest.json.JSONObject; import org.apache.http.client.utils.URIBuilder; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -160,9 +161,9 @@ public Optional getHelpPage() { } @Override - public URL getURLForQuery(String transformedQuery, int pageNumber) throws URISyntaxException, MalformedURLException { + public URL getURLForQuery(QueryNode luceneQuery, int pageNumber) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder(API_URL); - uriBuilder.addParameter("q", transformedQuery); // Search query + uriBuilder.addParameter("q", new SpringerQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); // Search query uriBuilder.addParameter("api_key", API_KEY); // API key uriBuilder.addParameter("s", String.valueOf(getPageSize() * pageNumber + 1)); // Start entry, starts indexing at 1 uriBuilder.addParameter("p", String.valueOf(getPageSize())); // Page size @@ -199,9 +200,4 @@ public Parser getParser() { return entries; }; } - - @Override - public AbstractQueryTransformer getQueryTransformer() { - return new SpringerQueryTransformer(); - } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/ZbMATH.java b/src/main/java/org/jabref/logic/importer/fetcher/ZbMATH.java index 47d0122b7db..69ea0c81b2a 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/ZbMATH.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/ZbMATH.java @@ -13,7 +13,6 @@ import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; -import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; import org.jabref.logic.importer.fetcher.transformators.ZbMathQueryTransformer; import org.jabref.logic.importer.fileformat.BibtexParser; import org.jabref.logic.net.URLDownload; @@ -23,6 +22,7 @@ import org.jabref.model.util.DummyFileUpdateMonitor; import org.apache.http.client.utils.URIBuilder; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; /** * Fetches data from the Zentralblatt Math (https://www.zbmath.org/) @@ -52,9 +52,9 @@ public URL getURLForEntry(BibEntry entry) throws URISyntaxException, MalformedUR } */ @Override - public URL getURLForQuery(String transformedQuery, AbstractQueryTransformer transformer) throws URISyntaxException, MalformedURLException, FetcherException { + public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, MalformedURLException, FetcherException { URIBuilder uriBuilder = new URIBuilder("https://zbmath.org/bibtexoutput/"); - uriBuilder.addParameter("q", transformedQuery); // search all fields + uriBuilder.addParameter("q", new ZbMathQueryTransformer().transformLuceneQuery(luceneQuery).orElse("")); // search all fields uriBuilder.addParameter("start", "0"); // start index uriBuilder.addParameter("count", "200"); // should return up to 200 items (instead of default 100) @@ -88,9 +88,4 @@ public void doPostCleanup(BibEntry entry) { new FieldFormatterCleanup(StandardField.JOURNAL, new RemoveBracesFormatter()).cleanup(entry); new FieldFormatterCleanup(StandardField.TITLE, new RemoveBracesFormatter()).cleanup(entry); } - - @Override - public AbstractQueryTransformer getQueryTransformer() { - return new ZbMathQueryTransformer(); - } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformators/AbstractQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformators/AbstractQueryTransformer.java index 0e5ac366354..32c7a6b8d18 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/transformators/AbstractQueryTransformer.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformators/AbstractQueryTransformer.java @@ -3,16 +3,12 @@ import java.util.Optional; import java.util.stream.Collectors; -import org.jabref.logic.JabRefException; - -import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; import org.apache.lucene.queryparser.flexible.core.nodes.BooleanQueryNode; import org.apache.lucene.queryparser.flexible.core.nodes.FieldQueryNode; import org.apache.lucene.queryparser.flexible.core.nodes.GroupQueryNode; import org.apache.lucene.queryparser.flexible.core.nodes.ModifierQueryNode; import org.apache.lucene.queryparser.flexible.core.nodes.OrQueryNode; import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; -import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -21,8 +17,8 @@ * Otherwise, a single instance QueryTransformer can be used. */ public abstract class AbstractQueryTransformer { + public static final String NO_EXPLICIT_FIELD = "default"; private static final Logger LOGGER = LoggerFactory.getLogger(AbstractQueryTransformer.class); - private static final String NO_EXPLICIT_FIELD = "default"; /** * Transforms a and b and c to (a AND b AND c), where @@ -180,19 +176,15 @@ private Optional transform(QueryNode query) { * Parses the given query string into a complex query using lucene. * Note: For unique fields, the alphabetically and numerically first instance in the query string is used in the complex query. * - * @param query The given query string - * @return A complex query containing all fields of the query string + * @param luceneQuery The lucene query tp transform + * @return A query string containing all fields that are contained in the original lucene query and + * that are expressible in the library specific query language, other information either is discarded or + * stored as part of the state of the transformer if it can be used e.g. as a URL parameter for the query. */ - public Optional parseQueryStringIntoComplexQuery(String query) throws JabRefException { - StandardSyntaxParser parser = new StandardSyntaxParser(); - try { - QueryNode luceneQuery = parser.parse(query, NO_EXPLICIT_FIELD); - Optional transformedQuery = transform(luceneQuery); - transformedQuery = transformedQuery.map(this::removeOuterBraces); - return transformedQuery; - } catch (QueryNodeParseException e) { - throw new JabRefException("Error parsing query", e); - } + public Optional transformLuceneQuery(QueryNode luceneQuery) { + Optional transformedQuery = transform(luceneQuery); + transformedQuery = transformedQuery.map(this::removeOuterBraces); + return transformedQuery; } /** diff --git a/src/test/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcherTest.java index 1a220323843..8f1ef0c3690 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcherTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcherTest.java @@ -12,6 +12,7 @@ import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.ImportFormatPreferences; +import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; import org.jabref.logic.importer.fetcher.transformators.DefaultQueryTransformer; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; @@ -19,6 +20,9 @@ import org.jabref.model.entry.types.StandardEntryType; import org.jabref.testutils.category.FetcherTest; +import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; +import org.apache.lucene.queryparser.flexible.core.parser.SyntaxParser; +import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.Answers; @@ -44,9 +48,9 @@ public void getNameReturnsCorrectName() { } @Test - public void getUrlForQueryReturnsCorrectUrl() throws MalformedURLException, URISyntaxException, FetcherException { + public void getUrlForQueryReturnsCorrectUrl() throws MalformedURLException, URISyntaxException, FetcherException, QueryNodeParseException { String query = "java jdk"; - URL url = fetcher.getURLForQuery(query, new DefaultQueryTransformer()); + URL url = fetcher.getURLForQuery(new StandardSyntaxParser().parse(query, AbstractQueryTransformer.NO_EXPLICIT_FIELD)); assertEquals("http://liinwww.ira.uka.de/bibliography/rss?query=java+jdk&sort=score", url.toString()); } diff --git a/src/test/java/org/jabref/logic/importer/fetcher/GvkFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/GvkFetcherTest.java index 8bd8bfc497e..7b832ea7f41 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/GvkFetcherTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/GvkFetcherTest.java @@ -1,12 +1,9 @@ package org.jabref.logic.importer.fetcher; -import java.net.MalformedURLException; -import java.net.URISyntaxException; import java.net.URL; import java.util.Collections; import java.util.List; -import org.jabref.logic.JabRefException; import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; import org.jabref.logic.importer.fetcher.transformators.GVKQueryTransformer; @@ -16,6 +13,9 @@ import org.jabref.model.entry.types.StandardEntryType; import org.jabref.testutils.category.FetcherTest; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; +import org.apache.lucene.queryparser.flexible.standard.StandardQueryParser; +import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -69,16 +69,16 @@ public void testGetName() { @Test public void simpleSearchQueryURLCorrect() throws Exception { String query = "java jdk"; - AbstractQueryTransformer transformer = new GVKQueryTransformer(); - URL url = fetcher.getURLForQuery(transformer.parseQueryStringIntoComplexQuery(query).get(), transformer); + QueryNode luceneQuery = new StandardSyntaxParser().parse(query, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + URL url = fetcher.getURLForQuery(luceneQuery); assertEquals("http://sru.gbv.de/gvk?version=1.1&operation=searchRetrieve&query=pica.all%3D%22java%22+and+pica.all%3D%22jdk%22&maximumRecords=50&recordSchema=picaxml&sortKeys=Year%2C%2C1", url.toString()); } @Test public void complexSearchQueryURLCorrect() throws Exception { String query = "kon:java tit:jdk"; - AbstractQueryTransformer transformer = new GVKQueryTransformer(); - URL url = fetcher.getURLForQuery(transformer.parseQueryStringIntoComplexQuery(query).get(), new GVKQueryTransformer()); + QueryNode luceneQuery = new StandardSyntaxParser().parse(query, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + URL url = fetcher.getURLForQuery(luceneQuery); assertEquals("http://sru.gbv.de/gvk?version=1.1&operation=searchRetrieve&query=pica.kon%3D%22java%22+and+pica.tit%3D%22jdk%22&maximumRecords=50&recordSchema=picaxml&sortKeys=Year%2C%2C1", url.toString()); } diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/ArXivQueryTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/ArXivQueryTransformerTest.java index f9d03654c43..aec511ffac4 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/transformators/ArXivQueryTransformerTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/ArXivQueryTransformerTest.java @@ -2,7 +2,8 @@ import java.util.Optional; -import org.junit.jupiter.api.Disabled; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; +import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; import static org.junit.jupiter.api.Assertions.*; @@ -36,8 +37,10 @@ public String getTitlePrefix() { @Override public void convertYearField() throws Exception { ArXivQueryTransformer transformer = ((ArXivQueryTransformer) getTransformator()); - Optional query = transformer.parseQueryStringIntoComplexQuery("year:2018"); - Optional expected = Optional.of("2018"); + String queryString = "2018"; + QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + Optional query = transformer.transformLuceneQuery(luceneQuery); + Optional expected = Optional.of(queryString); assertEquals(expected, query); assertEquals(2018, transformer.getStartYear()); assertEquals(2018, transformer.getEndYear()); @@ -47,7 +50,9 @@ public void convertYearField() throws Exception { public void convertYearRangeField() throws Exception { ArXivQueryTransformer transformer = ((ArXivQueryTransformer) getTransformator()); - transformer.parseQueryStringIntoComplexQuery("year-range:2018-2021"); + String queryString = "year-range:2018-2021"; + QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + transformer.transformLuceneQuery(luceneQuery); assertEquals(2018, transformer.getStartYear()); assertEquals(2021, transformer.getEndYear()); diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/DBLPQueryTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/DBLPQueryTransformerTest.java index a1ae7c00452..536592b97a7 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/transformators/DBLPQueryTransformerTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/DBLPQueryTransformerTest.java @@ -2,7 +2,8 @@ import java.util.Optional; -import org.junit.jupiter.api.Test; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; +import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; import static org.junit.jupiter.api.Assertions.*; @@ -34,14 +35,18 @@ public String getTitlePrefix() { @Override public void convertYearField() throws Exception { - Optional searchQuery = getTransformator().parseQueryStringIntoComplexQuery("year:2015"); + String queryString = "year:2015"; + QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + Optional searchQuery = getTransformator().transformLuceneQuery(luceneQuery); Optional expected = Optional.of("2015"); assertEquals(expected, searchQuery); } @Override public void convertYearRangeField() throws Exception { - Optional searchQuery = getTransformator().parseQueryStringIntoComplexQuery("year-range:2012-2015"); + String queryString = "year-range:2012-2015"; + QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + Optional searchQuery = getTransformator().transformLuceneQuery(luceneQuery); Optional expected = Optional.of("2012|2013|2014|2015"); assertEquals(expected, searchQuery); } diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/GVKQueryTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/GVKQueryTransformerTest.java index 86a6934830d..50412fd340b 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/transformators/GVKQueryTransformerTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/GVKQueryTransformerTest.java @@ -2,6 +2,8 @@ import java.util.Optional; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; +import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; import org.junit.jupiter.api.Disabled; import static org.junit.jupiter.api.Assertions.*; @@ -35,7 +37,10 @@ public String getTitlePrefix() { @Override public void convertYearField() throws Exception { - Optional query = getTransformator().parseQueryStringIntoComplexQuery("year:2018"); + + String queryString = "year:2018"; + QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + Optional query = getTransformator().transformLuceneQuery(luceneQuery); Optional expected = Optional.of("ver:2018"); assertEquals(expected, query); diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformerTest.java index e22372a675a..28531603f4b 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformerTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformerTest.java @@ -1,5 +1,8 @@ package org.jabref.logic.importer.fetcher.transformators; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; +import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; + import static org.junit.jupiter.api.Assertions.*; class IEEEQueryTransformerTest implements InfixTransformerTest{ @@ -33,16 +36,20 @@ public String getTitlePrefix() { public void convertJournalField() throws Exception { IEEEQueryTransformer transformer = ((IEEEQueryTransformer) getTransformator()); - transformer.parseQueryStringIntoComplexQuery("journal:Nature"); + String queryString = "journal:Nature"; + QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + transformer.transformLuceneQuery(luceneQuery); - assertEquals("\"Nautre\"", transformer.getJournal().get()); + assertEquals("\"Nature\"", transformer.getJournal().get()); } @Override public void convertYearField() throws Exception { IEEEQueryTransformer transformer = ((IEEEQueryTransformer) getTransformator()); - transformer.parseQueryStringIntoComplexQuery("year:2021"); + String queryString = "year:2021"; + QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + transformer.transformLuceneQuery(luceneQuery); assertEquals(2021, transformer.getStartYear()); assertEquals(2021, transformer.getEndYear()); @@ -53,7 +60,9 @@ public void convertYearRangeField() throws Exception { IEEEQueryTransformer transformer = ((IEEEQueryTransformer) getTransformator()); - transformer.parseQueryStringIntoComplexQuery("year-range:2018-2021"); + String queryString = "year-range:2018-2021"; + QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + transformer.transformLuceneQuery(luceneQuery); assertEquals(2018, transformer.getStartYear()); assertEquals(2021, transformer.getEndYear()); diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/InfixTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/InfixTransformerTest.java index ee9f8932883..f986a04c7e0 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/transformators/InfixTransformerTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/InfixTransformerTest.java @@ -2,8 +2,8 @@ import java.util.Optional; -import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; - +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; +import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; import org.junit.jupiter.api.Test; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -13,7 +13,6 @@ */ public interface InfixTransformerTest { - AbstractQueryTransformer getTransformator(); /* All prefixes have to include the used separator @@ -29,28 +28,36 @@ public interface InfixTransformerTest { @Test default void convertAuthorField() throws Exception { - Optional searchQuery = getTransformator().parseQueryStringIntoComplexQuery("author:\"Igor Steinmacher\""); + String queryString = "author:\"Igor Steinmacher\""; + QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + Optional searchQuery = getTransformator().transformLuceneQuery(luceneQuery); Optional expected = Optional.of(getAuthorPrefix() + "\"Igor Steinmacher\""); assertEquals(expected, searchQuery); } @Test default void convertUnFieldedTerm() throws Exception { - Optional searchQuery = getTransformator().parseQueryStringIntoComplexQuery("\"default value\""); - Optional expected = Optional.of(getUnFieldedPrefix() + "\"default value\""); + String queryString = "\"default value\""; + QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + Optional searchQuery = getTransformator().transformLuceneQuery(luceneQuery); + Optional expected = Optional.of(getUnFieldedPrefix() + queryString); assertEquals(expected, searchQuery); } @Test default void convertExplicitUnFieldedTerm() throws Exception { - Optional searchQuery = getTransformator().parseQueryStringIntoComplexQuery("default:\"default value\""); + String queryString = "default:\"default value\""; + QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + Optional searchQuery = getTransformator().transformLuceneQuery(luceneQuery); Optional expected = Optional.of(getUnFieldedPrefix() + "\"default value\""); assertEquals(expected, searchQuery); } @Test default void convertJournalField() throws Exception { - Optional searchQuery = getTransformator().parseQueryStringIntoComplexQuery("journal:Nature"); + String queryString = "journal:Nature"; + QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + Optional searchQuery = getTransformator().transformLuceneQuery(luceneQuery); Optional expected = Optional.of(getJournalPrefix() + "\"Nature\""); assertEquals(expected, searchQuery); } @@ -63,21 +70,27 @@ default void convertJournalField() throws Exception { @Test default void convertMultipleValuesWithTheSameField() throws Exception { - Optional searchQuery = getTransformator().parseQueryStringIntoComplexQuery("author:\"Igor Steinmacher\" author:\"Christoph Treude\""); + String queryString = "author:\"Igor Steinmacher\" author:\"Christoph Treude\""; + QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + Optional searchQuery = getTransformator().transformLuceneQuery(luceneQuery); Optional expected = Optional.of(getAuthorPrefix() + "\"Igor Steinmacher\"" + getTransformator().getLogicalAndOperator() + getAuthorPrefix() + "\"Christoph Treude\""); assertEquals(expected, searchQuery); } @Test default void groupedOperations() throws Exception { - Optional searchQuery = getTransformator().parseQueryStringIntoComplexQuery("(author:\"Igor Steinmacher\" OR author:\"Christoph Treude\" AND author:\"Christoph Freunde\") AND title:test"); + String queryString = "(author:\"Igor Steinmacher\" OR author:\"Christoph Treude\" AND author:\"Christoph Freunde\") AND title:test"; + QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + Optional searchQuery = getTransformator().transformLuceneQuery(luceneQuery); Optional expected = Optional.of("(" + getAuthorPrefix() + "\"Igor Steinmacher\"" + getTransformator().getLogicalOrOperator() + "(" + getAuthorPrefix() + "\"Christoph Treude\"" + getTransformator().getLogicalAndOperator() + getAuthorPrefix() + "\"Christoph Freunde\"))" + getTransformator().getLogicalAndOperator() + getTitlePrefix() + "\"test\""); assertEquals(expected, searchQuery); } @Test default void notOperator() throws Exception { - Optional searchQuery = getTransformator().parseQueryStringIntoComplexQuery("!(author:\"Igor Steinmacher\" OR author:\"Christoph Treude\")"); + String queryString = "!(author:\"Igor Steinmacher\" OR author:\"Christoph Treude\")"; + QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + Optional searchQuery = getTransformator().transformLuceneQuery(luceneQuery); Optional expected = Optional.of(getTransformator().getLogicalNotOperator() + "(" + getAuthorPrefix() + "\"Igor Steinmacher\"" + getTransformator().getLogicalOrOperator() + getAuthorPrefix() + "\"Christoph Treude\")"); assertEquals(expected, searchQuery); } diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/ScholarQueryTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/ScholarQueryTransformerTest.java index 6a913ed1c2f..3fd20ba9c05 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/transformators/ScholarQueryTransformerTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/ScholarQueryTransformerTest.java @@ -1,5 +1,8 @@ package org.jabref.logic.importer.fetcher.transformators; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; +import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; + import static org.junit.jupiter.api.Assertions.*; class ScholarQueryTransformerTest implements InfixTransformerTest { @@ -33,7 +36,10 @@ public String getTitlePrefix() { public void convertYearField() throws Exception { ScholarQueryTransformer transformer = ((ScholarQueryTransformer) getTransformator()); - transformer.parseQueryStringIntoComplexQuery("year:2021"); + String queryString = "year:2021"; + QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + transformer.transformLuceneQuery(luceneQuery); + assertEquals(2021, transformer.getStartYear()); assertEquals(2021, transformer.getEndYear()); @@ -44,7 +50,9 @@ public void convertYearRangeField() throws Exception { ScholarQueryTransformer transformer = ((ScholarQueryTransformer) getTransformator()); - transformer.parseQueryStringIntoComplexQuery("year-range:2018-2021"); + String queryString = "year-range:2018-2021"; + QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + transformer.transformLuceneQuery(luceneQuery); assertEquals(2018, transformer.getStartYear()); assertEquals(2021, transformer.getEndYear()); diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/SpringerQueryTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/SpringerQueryTransformerTest.java index b71da7449b9..88509c0a0e5 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/transformators/SpringerQueryTransformerTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/SpringerQueryTransformerTest.java @@ -2,7 +2,8 @@ import java.util.Optional; -import org.junit.jupiter.api.Test; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; +import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; import static org.junit.jupiter.api.Assertions.*; @@ -35,14 +36,20 @@ public String getTitlePrefix() { @Override public void convertYearField() throws Exception { - Optional searchQuery = getTransformator().parseQueryStringIntoComplexQuery("year:2015"); + String queryString = "year:2015"; + QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + Optional searchQuery = getTransformator().transformLuceneQuery(luceneQuery); + Optional expected = Optional.of("date:2015*"); assertEquals(expected, searchQuery); } @Override public void convertYearRangeField() throws Exception { - Optional searchQuery = getTransformator().parseQueryStringIntoComplexQuery("year-range:2012-2015"); + String queryString = "year-range:2012-2015"; + QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + Optional searchQuery = getTransformator().transformLuceneQuery(luceneQuery); + Optional expected = Optional.of("date:2012* OR date:2013* OR date:2014* OR date:2015*"); assertEquals(expected, searchQuery); } diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/ZbMathQueryTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/ZbMathQueryTransformerTest.java index ba46d97b40d..b8511cb180a 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/transformators/ZbMathQueryTransformerTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/ZbMathQueryTransformerTest.java @@ -2,7 +2,8 @@ import java.util.Optional; -import org.junit.jupiter.api.Test; +import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; +import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; import static org.junit.jupiter.api.Assertions.*; @@ -35,14 +36,18 @@ public String getTitlePrefix() { @Override public void convertYearField() throws Exception { - Optional searchQuery = getTransformator().parseQueryStringIntoComplexQuery("year:2015"); + String queryString = "year:2015"; + QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + Optional searchQuery = getTransformator().transformLuceneQuery(luceneQuery); Optional expected = Optional.of("py:2015"); assertEquals(expected, searchQuery); } @Override public void convertYearRangeField() throws Exception { - Optional searchQuery = getTransformator().parseQueryStringIntoComplexQuery("year-range:2012-2015"); + String queryString = "year-range:2012-2015"; + QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); + Optional searchQuery = getTransformator().transformLuceneQuery(luceneQuery); Optional expected = Optional.of("py:2012-2015"); assertEquals(expected, searchQuery); } From 1bcf7ab9cafeb095d05fc1211c0a03b26d63fc08 Mon Sep 17 00:00:00 2001 From: Dominik Voigt Date: Sat, 16 Jan 2021 20:30:42 +0100 Subject: [PATCH 20/24] Fix checkstyle --- src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java | 1 - .../java/org/jabref/logic/importer/fetcher/DBLPFetcher.java | 1 - .../importer/fetcher/transformators/DBLPQueryTransformer.java | 2 +- .../fetcher/transformators/ScholarQueryTransformer.java | 2 +- .../fetcher/transformators/ScholarQueryTransformerTest.java | 2 +- 5 files changed, 3 insertions(+), 5 deletions(-) diff --git a/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java b/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java index 9c33fe27ba0..8b860c3eeeb 100644 --- a/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java +++ b/src/main/java/org/jabref/logic/importer/SearchBasedFetcher.java @@ -3,7 +3,6 @@ import java.util.Collections; import java.util.List; -import org.jabref.logic.JabRefException; import org.jabref.model.entry.BibEntry; import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; diff --git a/src/main/java/org/jabref/logic/importer/fetcher/DBLPFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/DBLPFetcher.java index 54776ccfa24..2d12d9820f5 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/DBLPFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/DBLPFetcher.java @@ -16,7 +16,6 @@ import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.Parser; import org.jabref.logic.importer.SearchBasedParserFetcher; -import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; import org.jabref.logic.importer.fetcher.transformators.DBLPQueryTransformer; import org.jabref.logic.importer.fileformat.BibtexParser; import org.jabref.logic.layout.LayoutFormatterBasedFormatter; diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformators/DBLPQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformators/DBLPQueryTransformer.java index 3b50ce590c6..df943509320 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/transformators/DBLPQueryTransformer.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformators/DBLPQueryTransformer.java @@ -61,6 +61,6 @@ protected String handleYearRange(String yearRange) { @Override protected String handleUnFieldedTerm(String term) { - return String.format("\"%s\"",term); + return String.format("\"%s\"", term); } } diff --git a/src/main/java/org/jabref/logic/importer/fetcher/transformators/ScholarQueryTransformer.java b/src/main/java/org/jabref/logic/importer/fetcher/transformators/ScholarQueryTransformer.java index d661b2f9ba3..a83db9364b1 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/transformators/ScholarQueryTransformer.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/transformators/ScholarQueryTransformer.java @@ -60,6 +60,6 @@ public int getStartYear() { } public int getEndYear() { - return endYear == Integer.MIN_VALUE ? Integer.MAX_VALUE: endYear; + return endYear == Integer.MIN_VALUE ? Integer.MAX_VALUE : endYear; } } diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/ScholarQueryTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/ScholarQueryTransformerTest.java index 3fd20ba9c05..8837b6fb794 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/transformators/ScholarQueryTransformerTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/ScholarQueryTransformerTest.java @@ -3,7 +3,7 @@ import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; class ScholarQueryTransformerTest implements InfixTransformerTest { From bd61075506d2cde876f0eac7bfd527506ceddedc Mon Sep 17 00:00:00 2001 From: Dominik Voigt Date: Tue, 19 Jan 2021 09:39:54 +0100 Subject: [PATCH 21/24] Fix imports --- .../CollectionOfComputerScienceBibliographiesFetcherTest.java | 4 ---- .../importer/fetcher/CompositeSearchBasedFetcherTest.java | 1 - .../org/jabref/logic/importer/fetcher/GvkFetcherTest.java | 2 -- .../fetcher/transformators/ArXivQueryTransformerTest.java | 2 +- .../fetcher/transformators/DBLPQueryTransformerTest.java | 2 +- .../fetcher/transformators/GVKQueryTransformerTest.java | 2 +- .../fetcher/transformators/IEEEQueryTransformerTest.java | 2 +- .../fetcher/transformators/JstorQueryTransformerTest.java | 2 -- .../fetcher/transformators/SpringerQueryTransformerTest.java | 2 +- .../fetcher/transformators/ZbMathQueryTransformerTest.java | 2 +- 10 files changed, 6 insertions(+), 15 deletions(-) diff --git a/src/test/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcherTest.java index 8f1ef0c3690..539decbf37c 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcherTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/CollectionOfComputerScienceBibliographiesFetcherTest.java @@ -4,8 +4,6 @@ import java.net.URISyntaxException; import java.net.URL; import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -13,7 +11,6 @@ import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; -import org.jabref.logic.importer.fetcher.transformators.DefaultQueryTransformer; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.field.UnknownField; @@ -21,7 +18,6 @@ import org.jabref.testutils.category.FetcherTest; import org.apache.lucene.queryparser.flexible.core.QueryNodeParseException; -import org.apache.lucene.queryparser.flexible.core.parser.SyntaxParser; import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/src/test/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcherTest.java index b81913d1e64..1c329d419a8 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcherTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcherTest.java @@ -18,7 +18,6 @@ import org.jabref.testutils.category.FetcherTest; import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; diff --git a/src/test/java/org/jabref/logic/importer/fetcher/GvkFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/GvkFetcherTest.java index 7b832ea7f41..88cd4b91ef0 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/GvkFetcherTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/GvkFetcherTest.java @@ -6,7 +6,6 @@ import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.fetcher.transformators.AbstractQueryTransformer; -import org.jabref.logic.importer.fetcher.transformators.GVKQueryTransformer; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.field.StandardField; import org.jabref.model.entry.field.UnknownField; @@ -14,7 +13,6 @@ import org.jabref.testutils.category.FetcherTest; import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; -import org.apache.lucene.queryparser.flexible.standard.StandardQueryParser; import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/ArXivQueryTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/ArXivQueryTransformerTest.java index aec511ffac4..ac5933ae7bb 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/transformators/ArXivQueryTransformerTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/ArXivQueryTransformerTest.java @@ -5,7 +5,7 @@ import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; class ArXivQueryTransformerTest implements InfixTransformerTest { diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/DBLPQueryTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/DBLPQueryTransformerTest.java index 536592b97a7..e75a02cf174 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/transformators/DBLPQueryTransformerTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/DBLPQueryTransformerTest.java @@ -5,7 +5,7 @@ import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; class DBLPQueryTransformerTest implements InfixTransformerTest { diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/GVKQueryTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/GVKQueryTransformerTest.java index 50412fd340b..e13b31d2a0a 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/transformators/GVKQueryTransformerTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/GVKQueryTransformerTest.java @@ -6,7 +6,7 @@ import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; import org.junit.jupiter.api.Disabled; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; class GVKQueryTransformerTest implements InfixTransformerTest{ diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformerTest.java index 28531603f4b..d6c115191f0 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformerTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformerTest.java @@ -3,7 +3,7 @@ import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; class IEEEQueryTransformerTest implements InfixTransformerTest{ diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/JstorQueryTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/JstorQueryTransformerTest.java index aeeaa5ac625..e8a5a6014b9 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/transformators/JstorQueryTransformerTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/JstorQueryTransformerTest.java @@ -1,7 +1,5 @@ package org.jabref.logic.importer.fetcher.transformators; -import static org.junit.jupiter.api.Assertions.*; - class JstorQueryTransformerTest implements InfixTransformerTest { @Override diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/SpringerQueryTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/SpringerQueryTransformerTest.java index 88509c0a0e5..c0fbd484251 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/transformators/SpringerQueryTransformerTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/SpringerQueryTransformerTest.java @@ -5,7 +5,7 @@ import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; class SpringerQueryTransformerTest implements InfixTransformerTest { diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/ZbMathQueryTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/ZbMathQueryTransformerTest.java index b8511cb180a..ec4ef5cea85 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/transformators/ZbMathQueryTransformerTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/ZbMathQueryTransformerTest.java @@ -5,7 +5,7 @@ import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode; import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser; -import static org.junit.jupiter.api.Assertions.*; +import static org.junit.jupiter.api.Assertions.assertEquals; class ZbMathQueryTransformerTest implements InfixTransformerTest{ From d40f640ebf563cc51d74048e1918eab537cdf4e5 Mon Sep 17 00:00:00 2001 From: Dominik Voigt Date: Tue, 19 Jan 2021 09:55:47 +0100 Subject: [PATCH 22/24] Fix imports & checkstyle --- .../importer/fetcher/SearchBasedFetcherCapabilityTest.java | 1 - .../fetcher/transformators/DBLPQueryTransformerTest.java | 3 ++- .../fetcher/transformators/GVKQueryTransformerTest.java | 2 +- .../fetcher/transformators/IEEEQueryTransformerTest.java | 2 +- .../fetcher/transformators/ScholarQueryTransformerTest.java | 1 - .../fetcher/transformators/ZbMathQueryTransformerTest.java | 2 +- src/test/java/org/jabref/model/entry/CrossrefTest.java | 1 - 7 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/test/java/org/jabref/logic/importer/fetcher/SearchBasedFetcherCapabilityTest.java b/src/test/java/org/jabref/logic/importer/fetcher/SearchBasedFetcherCapabilityTest.java index 6653a0d3e09..a5bd223cdf2 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/SearchBasedFetcherCapabilityTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/SearchBasedFetcherCapabilityTest.java @@ -35,7 +35,6 @@ default void supportsAuthorSearch() throws Exception { StringJoiner queryBuilder = new StringJoiner("\" AND author:\"", "author:\"", "\""); getTestAuthors().forEach(queryBuilder::add); - List result = getFetcher().performSearch(queryBuilder.toString()); new ImportCleanup(BibDatabaseMode.BIBTEX).doPostCleanup(result); diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/DBLPQueryTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/DBLPQueryTransformerTest.java index e75a02cf174..d531f19da9d 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/transformators/DBLPQueryTransformerTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/DBLPQueryTransformerTest.java @@ -21,7 +21,8 @@ public String getAuthorPrefix() { @Override public String getUnFieldedPrefix() { - return ""; } + return ""; + } @Override public String getJournalPrefix() { diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/GVKQueryTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/GVKQueryTransformerTest.java index e13b31d2a0a..9dab863dc10 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/transformators/GVKQueryTransformerTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/GVKQueryTransformerTest.java @@ -8,7 +8,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; -class GVKQueryTransformerTest implements InfixTransformerTest{ +class GVKQueryTransformerTest implements InfixTransformerTest { @Override public AbstractQueryTransformer getTransformator() { diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformerTest.java index d6c115191f0..7d2042a34e3 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformerTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/IEEEQueryTransformerTest.java @@ -5,7 +5,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; -class IEEEQueryTransformerTest implements InfixTransformerTest{ +class IEEEQueryTransformerTest implements InfixTransformerTest { @Override public AbstractQueryTransformer getTransformator() { diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/ScholarQueryTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/ScholarQueryTransformerTest.java index 8837b6fb794..e83329b893a 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/transformators/ScholarQueryTransformerTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/ScholarQueryTransformerTest.java @@ -40,7 +40,6 @@ public void convertYearField() throws Exception { QueryNode luceneQuery = new StandardSyntaxParser().parse(queryString, AbstractQueryTransformer.NO_EXPLICIT_FIELD); transformer.transformLuceneQuery(luceneQuery); - assertEquals(2021, transformer.getStartYear()); assertEquals(2021, transformer.getEndYear()); } diff --git a/src/test/java/org/jabref/logic/importer/fetcher/transformators/ZbMathQueryTransformerTest.java b/src/test/java/org/jabref/logic/importer/fetcher/transformators/ZbMathQueryTransformerTest.java index ec4ef5cea85..73737455534 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/transformators/ZbMathQueryTransformerTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/transformators/ZbMathQueryTransformerTest.java @@ -7,7 +7,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; -class ZbMathQueryTransformerTest implements InfixTransformerTest{ +class ZbMathQueryTransformerTest implements InfixTransformerTest { @Override public AbstractQueryTransformer getTransformator() { diff --git a/src/test/java/org/jabref/model/entry/CrossrefTest.java b/src/test/java/org/jabref/model/entry/CrossrefTest.java index 98aa8ca54f0..9f212b5b519 100644 --- a/src/test/java/org/jabref/model/entry/CrossrefTest.java +++ b/src/test/java/org/jabref/model/entry/CrossrefTest.java @@ -11,7 +11,6 @@ import org.jabref.model.entry.types.StandardEntryType; import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Disabled; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; import org.junit.jupiter.params.provider.EnumSource; From 25a1de1ecac0fcca9efa85434852e884d058c7a1 Mon Sep 17 00:00:00 2001 From: Dominik Voigt Date: Tue, 26 Jan 2021 23:04:57 +0100 Subject: [PATCH 23/24] Add comment about curly brackets in lucene --- .../java/org/jabref/logic/importer/fetcher/DBLPFetcherTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/org/jabref/logic/importer/fetcher/DBLPFetcherTest.java b/src/test/java/org/jabref/logic/importer/fetcher/DBLPFetcherTest.java index bf788b6fe8b..001d5071fa5 100644 --- a/src/test/java/org/jabref/logic/importer/fetcher/DBLPFetcherTest.java +++ b/src/test/java/org/jabref/logic/importer/fetcher/DBLPFetcherTest.java @@ -50,6 +50,7 @@ public void setUp() { @Test public void findSingleEntry() throws FetcherException { + // In Lucene curly brackets are used for range queries, therefore they have to be escaped using "". See https://lucene.apache.org/core/5_4_0/queryparser/org/apache/lucene/queryparser/classic/package-summary.html String query = "Process Engine Benchmarking with Betsy in the Context of \"{ISO/IEC}\" Quality Standards"; List result = dblpFetcher.performSearch(query); From 3ae05a8edc7e90eabe998a1b1b2705d2a3e5efba Mon Sep 17 00:00:00 2001 From: Dominik Voigt Date: Tue, 26 Jan 2021 23:14:41 +0100 Subject: [PATCH 24/24] Replace JabRefException with FetcherException --- src/main/java/org/jabref/logic/crawler/StudyFetcher.java | 4 ++-- .../org/jabref/logic/importer/PagedSearchBasedFetcher.java | 3 +-- .../org/jabref/logic/importer/SearchBasedParserFetcher.java | 3 +-- .../logic/importer/fetcher/CompositeSearchBasedFetcher.java | 3 +-- 4 files changed, 5 insertions(+), 8 deletions(-) diff --git a/src/main/java/org/jabref/logic/crawler/StudyFetcher.java b/src/main/java/org/jabref/logic/crawler/StudyFetcher.java index 6f5dbef01fe..c39ba7efe52 100644 --- a/src/main/java/org/jabref/logic/crawler/StudyFetcher.java +++ b/src/main/java/org/jabref/logic/crawler/StudyFetcher.java @@ -5,7 +5,7 @@ import java.util.Objects; import java.util.stream.Collectors; -import org.jabref.logic.JabRefException; +import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.PagedSearchBasedFetcher; import org.jabref.logic.importer.SearchBasedFetcher; import org.jabref.model.database.BibDatabase; @@ -72,7 +72,7 @@ private FetchResult performSearchOnQueryForFetcher(String searchQuery, SearchBas fetchResult = fetcher.performSearch(searchQuery); } return new FetchResult(fetcher.getName(), new BibDatabase(fetchResult)); - } catch (JabRefException e) { + } catch (FetcherException e) { LOGGER.warn(String.format("%s API request failed", fetcher.getName()), e); return null; } diff --git a/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java b/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java index c83ea820703..a6b1cd4aea1 100644 --- a/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java +++ b/src/main/java/org/jabref/logic/importer/PagedSearchBasedFetcher.java @@ -4,7 +4,6 @@ import java.util.Collections; import java.util.List; -import org.jabref.logic.JabRefException; import org.jabref.model.entry.BibEntry; import org.jabref.model.paging.Page; @@ -27,7 +26,7 @@ public interface PagedSearchBasedFetcher extends SearchBasedFetcher { * @param pageNumber requested site number indexed from 0 * @return Page with search results */ - default Page performSearchPaged(String searchQuery, int pageNumber) throws JabRefException { + default Page performSearchPaged(String searchQuery, int pageNumber) throws FetcherException { if (searchQuery.isBlank()) { return new Page<>(searchQuery, pageNumber, Collections.emptyList()); } diff --git a/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java b/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java index 041540a6bac..9aadba697b9 100644 --- a/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java +++ b/src/main/java/org/jabref/logic/importer/SearchBasedParserFetcher.java @@ -7,7 +7,6 @@ import java.net.URL; import java.util.List; -import org.jabref.logic.JabRefException; import org.jabref.logic.cleanup.Formatter; import org.jabref.model.entry.BibEntry; @@ -36,7 +35,7 @@ default List performSearch(QueryNode luceneQuery) throws FetcherExcept URL urlForQuery; try { urlForQuery = getURLForQuery(luceneQuery); - } catch (URISyntaxException | MalformedURLException | JabRefException e) { + } catch (URISyntaxException | MalformedURLException | FetcherException e) { throw new FetcherException("Search URI crafted from complex search query is malformed", e); } return getBibEntries(urlForQuery); diff --git a/src/main/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcher.java b/src/main/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcher.java index 2de2344b304..c616b018cb1 100644 --- a/src/main/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcher.java +++ b/src/main/java/org/jabref/logic/importer/fetcher/CompositeSearchBasedFetcher.java @@ -6,7 +6,6 @@ import java.util.stream.Collectors; import java.util.stream.Stream; -import org.jabref.logic.JabRefException; import org.jabref.logic.help.HelpFile; import org.jabref.logic.importer.FetcherException; import org.jabref.logic.importer.ImportCleanup; @@ -55,7 +54,7 @@ public List performSearch(QueryNode luceneQuery) throws FetcherExcepti .flatMap(searchBasedFetcher -> { try { return searchBasedFetcher.performSearch(luceneQuery).stream(); - } catch (JabRefException e) { + } catch (FetcherException e) { LOGGER.warn(String.format("%s API request failed", searchBasedFetcher.getName()), e); return Stream.empty(); }