Skip to content

Commit

Permalink
Merge pull request #10211 from JabRef/updateGVKIsbn
Browse files Browse the repository at this point in the history
Enable ISBN search with GVK
  • Loading branch information
Siedlerchr authored Aug 27, 2023
2 parents 8d2a8e1 + 503c0df commit 4da1443
Show file tree
Hide file tree
Showing 13 changed files with 188 additions and 36 deletions.
2 changes: 1 addition & 1 deletion src/main/java/org/jabref/gui/EntryTypeViewModel.java
Original file line number Diff line number Diff line change
Expand Up @@ -159,7 +159,7 @@ public void runFetcherWorker() {
dialogService.showInformationDialogAndWait(Localization.lang("Failed to import by ID"), Localization.lang("Error message %0", fetcherExceptionMessage));
}

LOGGER.error(String.format("Exception during fetching when using fetcher '%s' with entry id '%s'.", searchId, fetcher), exception);
LOGGER.error(String.format("Exception during fetching when using fetcher '%s' with entry id '%s'.", fetcher, searchId), exception);

searchingProperty.set(false);
fetcherWorker = new FetcherWorker();
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/org/jabref/logic/importer/WebFetchers.java
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ public static SortedSet<SearchBasedFetcher> getSearchBasedFetchers(ImportFormatP
SortedSet<SearchBasedFetcher> set = new TreeSet<>(Comparator.comparing(WebFetcher::getName));
set.add(new ArXivFetcher(importFormatPreferences));
set.add(new INSPIREFetcher(importFormatPreferences));
set.add(new GvkFetcher());
set.add(new GvkFetcher(importFormatPreferences));
set.add(new BvbFetcher());
set.add(new MedlineFetcher());
set.add(new AstrophysicsDataSystem(importFormatPreferences, importerPreferences));
Expand Down
38 changes: 36 additions & 2 deletions src/main/java/org/jabref/logic/importer/fetcher/GvkFetcher.java
Original file line number Diff line number Diff line change
Expand Up @@ -7,26 +7,36 @@
import java.util.Collection;
import java.util.Optional;

import org.jabref.logic.cleanup.FieldFormatterCleanup;
import org.jabref.logic.formatter.bibtexfields.NormalizeNamesFormatter;
import org.jabref.logic.formatter.bibtexfields.NormalizePagesFormatter;
import org.jabref.logic.help.HelpFile;
import org.jabref.logic.importer.FetcherException;
import org.jabref.logic.importer.ImportFormatPreferences;
import org.jabref.logic.importer.Parser;
import org.jabref.logic.importer.SearchBasedParserFetcher;
import org.jabref.logic.importer.fetcher.transformers.GVKQueryTransformer;
import org.jabref.logic.importer.fileformat.PicaXmlParser;
import org.jabref.model.entry.BibEntry;
import org.jabref.model.entry.field.StandardField;

import org.apache.http.client.utils.URIBuilder;
import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode;

public class GvkFetcher implements SearchBasedParserFetcher {
public class GvkFetcher extends AbstractIsbnFetcher implements SearchBasedParserFetcher {

private static final String URL_PATTERN = "https://sru.k10plus.de/gvk?";
private static final String URL_PATTERN = "https://sru.k10plus.de/opac-de-627?";

/**
* Searchkeys are used to specify a search request. For example "tit" stands for "title".
* If no searchkey is used, the default searchkey "all" is used.
*/
private final Collection<String> searchKeys = Arrays.asList("all", "tit", "per", "thm", "slw", "txt", "num", "kon", "ppn", "bkl", "erj");

public GvkFetcher(ImportFormatPreferences importFormatPreferences) {
super(importFormatPreferences);
}

@Override
public String getName() {
return "GVK";
Expand All @@ -49,8 +59,32 @@ public URL getURLForQuery(QueryNode luceneQuery) throws URISyntaxException, Malf
return uriBuilder.build().toURL();
}

@Override
public URL getUrlForIdentifier(String identifier) throws URISyntaxException, MalformedURLException, FetcherException {
this.ensureThatIsbnIsValid(identifier);
URIBuilder uriBuilder = new URIBuilder(URL_PATTERN);
uriBuilder.addParameter("version", "1.1");
uriBuilder.addParameter("operation", "searchRetrieve");
uriBuilder.addParameter("query", "pica.isb=" + identifier);
uriBuilder.addParameter("maximumRecords", "50");
uriBuilder.addParameter("recordSchema", "picaxml");
uriBuilder.addParameter("sortKeys", "Year,,1");
return uriBuilder.build().toURL();
}

@Override
public Parser getParser() {
return new PicaXmlParser();
}

@Override
public void doPostCleanup(BibEntry entry) {
super.doPostCleanup(entry);

// Fetcher returns page numbers as "30 Seiten" -> remove every non-digit character in the PAGETOTAL field
entry.getField(StandardField.PAGETOTAL).ifPresent(pages ->
entry.setField(StandardField.PAGETOTAL, pages.replaceAll("[\\D]", "")));
new FieldFormatterCleanup(StandardField.PAGETOTAL, new NormalizePagesFormatter()).cleanup(entry);
new FieldFormatterCleanup(StandardField.AUTHOR, new NormalizeNamesFormatter()).cleanup(entry);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,10 @@
import org.jabref.logic.importer.IdBasedFetcher;
import org.jabref.logic.importer.ImportFormatPreferences;
import org.jabref.logic.importer.fetcher.AbstractIsbnFetcher;
import org.jabref.logic.importer.fetcher.GvkFetcher;
import org.jabref.model.entry.BibEntry;
import org.jabref.model.entry.field.StandardField;
import org.jabref.model.strings.StringUtil;
import org.jabref.model.entry.identifier.ISBN;
import org.jabref.model.util.OptionalUtil;

import org.slf4j.Logger;
Expand All @@ -35,11 +36,14 @@ public class IsbnFetcher implements EntryBasedFetcher, IdBasedFetcher {
protected final ImportFormatPreferences importFormatPreferences;
private final OpenLibraryIsbnFetcher openLibraryIsbnFetcher;
private final List<AbstractIsbnFetcher> retryIsbnFetcher;
private final GvkFetcher gvkIbsnFetcher;

public IsbnFetcher(ImportFormatPreferences importFormatPreferences) {
this.importFormatPreferences = importFormatPreferences;
this.openLibraryIsbnFetcher = new OpenLibraryIsbnFetcher(importFormatPreferences);
this.gvkIbsnFetcher = new GvkFetcher(importFormatPreferences);
this.retryIsbnFetcher = new ArrayList<>();
this.addRetryFetcher(openLibraryIsbnFetcher);
}

@Override
Expand All @@ -54,15 +58,14 @@ public Optional<HelpFile> getHelpPage() {

@Override
public Optional<BibEntry> performSearchById(String identifier) throws FetcherException {
if (StringUtil.isBlank(identifier)) {
return Optional.empty();
}

Optional<BibEntry> bibEntry = Optional.empty();

try {
identifier = removeNewlinesAndSpacesFromIdentifier(identifier);
bibEntry = openLibraryIsbnFetcher.performSearchById(identifier);
Optional<ISBN> isbn = ISBN.parse(identifier);
if (isbn.isPresent()) {
bibEntry = gvkIbsnFetcher.performSearchById(isbn.get().getNormalized());
}
} catch (FetcherException ex) {
LOGGER.debug("Got a fetcher exception for IBSN search", ex);
if (retryIsbnFetcher.isEmpty()) {
Expand Down
2 changes: 2 additions & 0 deletions src/test/java/org/jabref/logic/importer/WebFetchersTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import org.jabref.logic.importer.fetcher.AbstractIsbnFetcher;
import org.jabref.logic.importer.fetcher.GoogleScholar;
import org.jabref.logic.importer.fetcher.GrobidCitationFetcher;
import org.jabref.logic.importer.fetcher.GvkFetcher;
import org.jabref.logic.importer.fetcher.JstorFetcher;
import org.jabref.logic.importer.fetcher.MrDLibFetcher;
import org.jabref.logic.importer.fetcher.isbntobibtex.DoiToBibtexConverterComIsbnFetcher;
Expand Down Expand Up @@ -74,6 +75,7 @@ void getIdBasedFetchersReturnsAllFetcherDerivingFromIdBasedFetcher() {
// Remove special ISBN fetcher since we don't want to expose them to the user
expected.remove(OpenLibraryIsbnFetcher.class);
expected.remove(EbookDeIsbnFetcher.class);
expected.remove(GvkFetcher.class);
expected.remove(DoiToBibtexConverterComIsbnFetcher.class);

// Remove the following, because they don't work at the moment
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -79,12 +79,16 @@ public static Stream<Arguments> performSearchByIdReturnsCorrectEntryForIdentifie
Arguments.arguments(
"performSearchByIdReturnsCorrectEntryForIsbnId",
new BibEntry(StandardEntryType.Book)
.withField(StandardField.TITLE, "Effective Java")
.withField(StandardField.PUBLISHER, "Addison-Wesley Professional")
.withField(StandardField.YEAR, "2017")
.withField(StandardField.AUTHOR, "Bloch, Joshua")
.withField(StandardField.PAGES, "416")
.withField(StandardField.ISBN, "9780134685991"),
.withField(StandardField.TITLE, "Effective Java")
.withField(StandardField.PUBLISHER, "Addison-Wesley")
.withField(StandardField.YEAR, "2018")
.withField(StandardField.ISBN, "9780134685991")
.withField(StandardField.NOTE, "Titelzusätze auf dem Umschlag: \"Updated for Java 9. Best practices for ... the Java platform\"")
.withField(StandardField.PAGETOTAL, "392")
.withField(new UnknownField("ppn_gvk"), "100121840X")
.withField(StandardField.EDITION, "Third edition")
.withField(StandardField.ADDRESS, "Boston"),
"9780134685991"
),
Arguments.arguments(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ static Stream<Arguments> performSearchParameters() {
List<SearchBasedFetcher> list = List.of(
new ArXivFetcher(importFormatPreferences),
new INSPIREFetcher(importFormatPreferences),
new GvkFetcher(),
new GvkFetcher(importFormatPreferences),
new AstrophysicsDataSystem(importFormatPreferences, importerPreferences),
new MathSciNet(importFormatPreferences),
new ZbMATH(importFormatPreferences),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ public void testPerformSearchById() throws Exception {
entry.setType(StandardEntryType.Article);
entry.setCitationKey("Gustafsson260746");
entry.setField(StandardField.AUTHOR, "Gustafsson, Oscar");
entry.setField(StandardField.INSTITUTION, "Linköping University, The Institute of Technology");
entry.setField(StandardField.INSTITUTION, "The Institute of Technology");
entry.setField(StandardField.JOURNAL,
"IEEE transactions on circuits and systems. 2, Analog and digital signal processing (Print)");
entry.setField(StandardField.NUMBER, "11");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import java.util.List;

import org.jabref.logic.importer.FetcherException;
import org.jabref.logic.importer.ImportFormatPreferences;
import org.jabref.logic.importer.fetcher.transformers.AbstractQueryTransformer;
import org.jabref.model.entry.BibEntry;
import org.jabref.model.entry.field.StandardField;
Expand All @@ -16,9 +17,11 @@
import org.apache.lucene.queryparser.flexible.standard.parser.StandardSyntaxParser;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.Answers;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;

@FetcherTest
public class GvkFetcherTest {
Expand All @@ -29,30 +32,30 @@ public class GvkFetcherTest {

@BeforeEach
public void setUp() {
fetcher = new GvkFetcher();
fetcher = new GvkFetcher(mock(ImportFormatPreferences.class, Answers.RETURNS_DEEP_STUBS));

bibEntryPPN591166003 = new BibEntry(StandardEntryType.Book)
.withField(StandardField.TITLE, "Effective Java")
.withField(StandardField.PUBLISHER, "Addison-Wesley")
.withField(StandardField.YEAR, "2008")
.withField(StandardField.AUTHOR, "Joshua Bloch")
.withField(StandardField.AUTHOR, "Bloch, Joshua")
.withField(StandardField.SERIES, "The Java series ... from the source")
.withField(StandardField.ADDRESS, "Upper Saddle River, NJ")
.withField(StandardField.EDITION, "2. ed., 5. print.")
.withField(StandardField.NOTE, "Includes bibliographical references and index. - Previous ed.: 2001. - Hier auch später erschienene, unveränderte Nachdrucke")
.withField(StandardField.ISBN, "9780321356680")
.withField(StandardField.PAGETOTAL, "XXI, 346")
.withField(StandardField.PAGETOTAL, "346")
.withField(new UnknownField("ppn_gvk"), "591166003")
.withField(StandardField.SUBTITLE, "[revised and updated for JAVA SE 6]");

bibEntryPPN66391437X = new BibEntry(StandardEntryType.Book)
.withField(StandardField.TITLE, "Effective unit testing")
.withField(StandardField.PUBLISHER, "Manning")
.withField(StandardField.YEAR, "2013")
.withField(StandardField.AUTHOR, "Lasse Koskela")
.withField(StandardField.AUTHOR, "Koskela, Lasse")
.withField(StandardField.ADDRESS, "Shelter Island, NY")
.withField(StandardField.ISBN, "9781935182573")
.withField(StandardField.PAGETOTAL, "XXIV, 223")
.withField(StandardField.PAGETOTAL, "223")
.withField(new UnknownField("ppn_gvk"), "66391437X")
.withField(StandardField.SUBTITLE, "A guide for Java developers");
}
Expand All @@ -67,15 +70,15 @@ public void simpleSearchQueryURLCorrect() throws Exception {
String query = "java jdk";
QueryNode luceneQuery = new StandardSyntaxParser().parse(query, AbstractQueryTransformer.NO_EXPLICIT_FIELD);
URL url = fetcher.getURLForQuery(luceneQuery);
assertEquals("https://sru.k10plus.de/gvk?version=1.1&operation=searchRetrieve&query=pica.all%3Djava+and+pica.all%3Djdk&maximumRecords=50&recordSchema=picaxml&sortKeys=Year%2C%2C1", url.toString());
assertEquals("https://sru.k10plus.de/opac-de-627?version=1.1&operation=searchRetrieve&query=pica.all%3Djava+and+pica.all%3Djdk&maximumRecords=50&recordSchema=picaxml&sortKeys=Year%2C%2C1", url.toString());
}

@Test
public void complexSearchQueryURLCorrect() throws Exception {
String query = "kon:java tit:jdk";
QueryNode luceneQuery = new StandardSyntaxParser().parse(query, AbstractQueryTransformer.NO_EXPLICIT_FIELD);
URL url = fetcher.getURLForQuery(luceneQuery);
assertEquals("https://sru.k10plus.de/gvk?version=1.1&operation=searchRetrieve&query=pica.kon%3Djava+and+pica.tit%3Djdk&maximumRecords=50&recordSchema=picaxml&sortKeys=Year%2C%2C1", url.toString());
assertEquals("https://sru.k10plus.de/opac-de-627?version=1.1&operation=searchRetrieve&query=pica.kon%3Djava+and+pica.tit%3Djdk&maximumRecords=50&recordSchema=picaxml&sortKeys=Year%2C%2C1", url.toString());
}

@Test
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
package org.jabref.logic.importer.fetcher.isbntobibtex;

import java.util.Optional;

import org.jabref.logic.importer.FetcherException;
import org.jabref.logic.importer.ImportFormatPreferences;
import org.jabref.logic.importer.fetcher.AbstractIsbnFetcherTest;
import org.jabref.logic.importer.fetcher.GvkFetcher;
import org.jabref.model.entry.BibEntry;
import org.jabref.model.entry.field.StandardField;
import org.jabref.model.entry.field.UnknownField;
import org.jabref.model.entry.types.StandardEntryType;

import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.Answers;

import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.Mockito.mock;

public class GVKIsbnFetcherTest extends AbstractIsbnFetcherTest {

private BibEntry bibEntryEffectiveJavaLongISBN;

@BeforeEach
public void setUp() {
bibEntryEffectiveJava = new BibEntry(StandardEntryType.Book)
.withField(StandardField.TITLE, "Effective Java(TM) Programming Language Guide (2nd Edition) (The Java Series)")
.withField(StandardField.PUBLISHER, "Prentice Hall PTR")
.withField(StandardField.YEAR, "2007")
.withField(StandardField.AUTHOR, "Bloch, Joshua")
.withField(StandardField.ISBN, "9780321356680")
.withField(StandardField.PAGES, "256");

bibEntryEffectiveJavaLongISBN = new BibEntry(StandardEntryType.Book)
.withField(StandardField.TITLE, "Effective Java")
.withField(StandardField.PUBLISHER, "Addison-Wesley")
.withField(StandardField.YEAR, "2011")
.withField(StandardField.AUTHOR, "Bloch, Joshua")
.withField(StandardField.SERIES, "The @Java series")
.withField(StandardField.ADDRESS, "Upper Saddle River, NJ [u.a.]")
.withField(StandardField.EDITION, "2. ed., rev. and updated for Java SE 6")
.withField(StandardField.NOTE, "*Hier auch später erschienene, unveränderte Nachdrucke*")
.withField(StandardField.ISBN, "9780321356680")
.withField(StandardField.PAGETOTAL, "346")
.withField(new UnknownField("ppn_gvk"), "67954951X")
.withField(StandardField.SUBTITLE, "[revised and updated for Java SE 6]");

fetcher = new GvkFetcher(mock(ImportFormatPreferences.class, Answers.RETURNS_DEEP_STUBS));
}

@Test
@Override
public void testName() {
assertEquals("GVK", fetcher.getName());
}

@Test
@Override
public void searchByIdSuccessfulWithShortISBN() throws FetcherException {
Optional<BibEntry> fetchedEntry = fetcher.performSearchById("0321356683");
assertEquals(Optional.of(bibEntryEffectiveJavaLongISBN), fetchedEntry);
}

@Test
@Override
public void searchByIdSuccessfulWithLongISBN() throws FetcherException {
Optional<BibEntry> fetchedEntry = fetcher.performSearchById("9780321356680");
assertEquals(Optional.of(bibEntryEffectiveJavaLongISBN), fetchedEntry);
}

@Test
@Override
public void authorsAreCorrectlyFormatted() throws Exception {
BibEntry bibEntry = new BibEntry(StandardEntryType.Misc)
.withField(StandardField.TITLE, "Repository")
.withField(StandardField.SUBTITLE, "Eine Einführung")
.withField(StandardField.PUBLISHER, "De Gruyter Oldenbourg")
.withField(StandardField.AUTHOR, "Habermann, Hans-Joachim")
.withField(StandardField.ISBN, "9783110702125")
.withField(StandardField.YEAR, "2020")
.withField(StandardField.ADDRESS, "München")
.withField(StandardField.EDITION, "Reprint 2020")
.withField(StandardField.EDITOR, "Frank Leymann")
.withField(StandardField.NUMBER, "8.1")
.withField(StandardField.PAGETOTAL, "1294")
.withField(StandardField.SERIES, "Handbuch der Informatik")
.withField(new UnknownField("ppn_gvk"), "1738076555");

Optional<BibEntry> fetchedEntry = fetcher.performSearchById("9783110702125");
assertEquals(Optional.of(bibEntry), fetchedEntry);
}

/**
* Checks whether the given ISBN is <emph>NOT</emph> available at any ISBN fetcher
*/
@Test
public void testIsbnNeitherAvailableOnEbookDeNorOrViaOpenLibrary() throws Exception {
// In this test, the ISBN needs to be a valid (syntax+checksum) ISBN number
// However, the ISBN number must not be assigned to a real book
assertEquals(Optional.empty(), fetcher.performSearchById("9785646216541"));
}
}
Loading

0 comments on commit 4da1443

Please sign in to comment.