Skip to content

Commit

Permalink
Unify month field formatting in test bib files (#5578)
Browse files Browse the repository at this point in the history 
* Unify month field formatting in test bib files (fixes #5116)

* Fix month creation. #name# is really used within JabRef to determine a BibTeX string content (and not plain text)

- Fix tests
- Sort BibEntryTests: default constructor, keywords, all others
- Fix name of "CanonicalBibEntry" (from CanonicalBibtexEntry)
- Revert "fix jabref format of month", this also reverts commit 77c25ca.
- Format some code
- Add comments

* Merge all BibEntry test classes into one
  • Loading branch information
@koppor @tobiasdiez
koppor authored and tobiasdiez committed Nov 13, 2019
1 parent 31cabdb commit 19393bf
Show file tree
Hide file tree
Showing 61 changed files with 662 additions and 591 deletions.
15 changes: 15 additions & 0 deletions src/main/java/org/jabref/logic/exporter/BibTeXMLExporter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.stream.Collectors;

import javax.xml.bind.JAXBContext;
Expand Down Expand Up @@ -40,6 +41,7 @@
import org.jabref.logic.util.StandardFileType;
import org.jabref.model.database.BibDatabaseContext;
import org.jabref.model.entry.BibEntry;
import org.jabref.model.entry.Month;
import org.jabref.model.entry.field.Field;
import org.jabref.model.entry.field.StandardField;
import org.jabref.model.entry.types.EntryType;
Expand Down Expand Up @@ -153,6 +155,13 @@ private void parseInbook(Inbook inbook, BibEntry bibEntry, Entry entry) {
JAXBElement<BigInteger> number = new JAXBElement<>(new QName(BIBTEXML_NAMESPACE_URI, "number"),
BigInteger.class, new BigInteger(value));
inbook.getContent().add(number);
} else if (StandardField.MONTH.equals(key)) {
Optional<Month> month = bibEntry.getMonth();
if (month.isPresent()) {
JAXBElement<String> element = new JAXBElement<>(new QName(BIBTEXML_NAMESPACE_URI, key.getName()),
String.class, month.get().getFullName());
inbook.getContent().add(element);
}
} else {
JAXBElement<String> element = new JAXBElement<>(new QName(BIBTEXML_NAMESPACE_URI, key.getName()), String.class,
value);
Expand Down Expand Up @@ -205,6 +214,12 @@ private <T> void parse(T entryType, BibEntry bibEntry, Entry entry) {
LOGGER.warn("The value %s of the 'number' field is not an integer and thus is ignored for the export", value);
}
break;
} else if (StandardField.MONTH.equals(key)) {
Optional<Month> month = bibEntry.getMonth();
if (month.isPresent()) {
method.invoke(entryType, month.get().getFullName());
}
break;
} else {
method.invoke(entryType, value);
break;
Expand Down
17 changes: 16 additions & 1 deletion src/main/java/org/jabref/logic/importer/fileformat/BibTeXMLImporter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.regex.Pattern;

import javax.xml.bind.JAXBContext;
Expand All @@ -27,6 +28,7 @@
import org.jabref.logic.importer.fileformat.bibtexml.Incollection;
import org.jabref.logic.util.StandardFileType;
import org.jabref.model.entry.BibEntry;
import org.jabref.model.entry.Month;
import org.jabref.model.entry.field.Field;
import org.jabref.model.entry.field.FieldFactory;
import org.jabref.model.entry.field.StandardField;
Expand Down Expand Up @@ -176,6 +178,9 @@ private <T> void parse(T entryType, Map<Field, String> fields) {
} else if (method.getName().equals("getNumber")) {
putNumber(fields, (BigInteger) method.invoke(entryType));
continue;
} else if (method.getName().equals("getMonth")) {
putMonth(fields, Month.parse((String) method.invoke(entryType)));
continue;
} else if (isMethodToIgnore(method.getName())) {
continue;
} else if (method.getName().startsWith("get")) {
Expand Down Expand Up @@ -208,7 +213,11 @@ private void parseInbook(Inbook inbook, Map<Field, String> fields) {
Object elementValue = element.getValue();
if (elementValue instanceof String) {
String value = (String) elementValue;
putIfValueNotNull(fields, field, value);
if (StandardField.MONTH.equals(field)) {
putMonth(fields, Month.parse(value));
} else {
putIfValueNotNull(fields, field, value);
}
} else if (elementValue instanceof BigInteger) {
BigInteger value = (BigInteger) elementValue;
if (StandardField.NUMBER.equals(field)) {
Expand Down Expand Up @@ -241,6 +250,12 @@ private void putNumber(Map<Field, String> fields, BigInteger number) {
}
}

private void putMonth(Map<Field, String> fields, Optional<Month> month) {
if (month.isPresent()) {
fields.put(StandardField.MONTH, month.get().getJabRefFormat());
}
}

private void putIfValueNotNull(Map<Field, String> fields, Field field, String value) {
if (value != null) {
fields.put(field, value);
Expand Down
3 changes: 0 additions & 3 deletions src/main/java/org/jabref/logic/importer/fileformat/BibtexParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -589,7 +589,6 @@ private String parseFieldContent(Field field) throws IOException {
int character;

while (((character = peek()) != ',') && (character != '}') && (character != ')')) {

if (eof) {
throw new IOException("Error in line " + line + ": EOF in mid-string");
}
Expand All @@ -602,7 +601,6 @@ private String parseFieldContent(Field field) throws IOException {
// brackets to know when the string is finished.
StringBuilder text = parseBracketedTextExactly();
value.append(fieldContentParser.format(text, field));

} else if (Character.isDigit((char) character)) { // value is a number
String number = parseTextToken();
value.append(number);
Expand All @@ -619,7 +617,6 @@ private String parseFieldContent(Field field) throws IOException {
skipWhitespace();
}
return value.toString();

}

/**
Expand Down
16 changes: 7 additions & 9 deletions src/main/java/org/jabref/logic/importer/fileformat/IsiImporter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ private static void processCapitalization(Map<Field, String> map) {
public ParserResult importDatabase(BufferedReader reader) throws IOException {
Objects.requireNonNull(reader);

List<BibEntry> bibitems = new ArrayList<>();
List<BibEntry> bibEntries = new ArrayList<>();
StringBuilder sb = new StringBuilder();

// Pattern fieldPattern = Pattern.compile("^AU |^TI |^SO |^DT |^C1 |^AB
Expand Down Expand Up @@ -233,7 +233,6 @@ public ParserResult importDatabase(BufferedReader reader) throws IOException {
} else if ("SO".equals(beg) || "JA".equals(beg)) {
hm.put(StandardField.JOURNAL, EOL_PATTERN.matcher(value).replaceAll(" "));
} else if ("ID".equals(beg) || "KW".equals(beg)) {

value = EOL_PATTERN.matcher(value).replaceAll(" ");
String existingKeywords = hm.get(StandardField.KEYWORDS);
if ((existingKeywords == null) || existingKeywords.contains(value)) {
Expand All @@ -242,7 +241,6 @@ public ParserResult importDatabase(BufferedReader reader) throws IOException {
existingKeywords += ", " + value;
}
hm.put(StandardField.KEYWORDS, existingKeywords);

} else if ("AB".equals(beg)) {
hm.put(StandardField.ABSTRACT, EOL_PATTERN.matcher(value).replaceAll(" "));
} else if ("BP".equals(beg) || "BR".equals(beg) || "SP".equals(beg)) {
Expand Down Expand Up @@ -271,12 +269,10 @@ public ParserResult importDatabase(BufferedReader reader) throws IOException {
} else if ("DI".equals(beg)) {
hm.put(StandardField.DOI, value);
} else if ("PD".equals(beg)) {

String month = IsiImporter.parseMonth(value);
if (month != null) {
hm.put(StandardField.MONTH, month);
}

} else if ("DT".equals(beg)) {
if ("Review".equals(value)) {
type = StandardEntryType.Article; // set "Review" in Note/Comment?
Expand Down Expand Up @@ -327,17 +323,19 @@ public ParserResult importDatabase(BufferedReader reader) throws IOException {

b.setField(hm);

bibitems.add(b);
bibEntries.add(b);
}
return new ParserResult(bibitems);
return new ParserResult(bibEntries);
}

private static String parsePages(String value) {
return value.replace("-", "--");
}

public static String parseMonth(String value) {

/**
* Parses the month and returns it in the JabRef format
*/
static String parseMonth(String value) {
String[] parts = value.split("\\s|\\-");
for (String part1 : parts) {
Optional<Month> month = Month.getMonthByShortName(part1.toLowerCase(Locale.ROOT));
Expand Down
7 changes: 6 additions & 1 deletion src/main/java/org/jabref/logic/importer/fileformat/MedlineImporter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;

import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBElement;
Expand Down Expand Up @@ -74,6 +75,7 @@
import org.jabref.logic.importer.fileformat.medline.Text;
import org.jabref.logic.util.StandardFileType;
import org.jabref.model.entry.BibEntry;
import org.jabref.model.entry.Month;
import org.jabref.model.entry.field.Field;
import org.jabref.model.entry.field.FieldFactory;
import org.jabref.model.entry.field.InternalField;
Expand Down Expand Up @@ -603,7 +605,10 @@ private void addPubDate(Map<Field, String> fields, PubDate pubDate) {
} else {
fields.put(StandardField.YEAR, pubDate.getYear());
if (pubDate.getMonth() != null) {
fields.put(StandardField.MONTH, pubDate.getMonth());
Optional<Month> month = Month.parse(pubDate.getMonth());
if (month.isPresent()) {
fields.put(StandardField.MONTH, month.get().getJabRefFormat());
}
} else if (pubDate.getSeason() != null) {
fields.put(new UnknownField("season"), pubDate.getSeason());
}
Expand Down
10 changes: 4 additions & 6 deletions src/main/java/org/jabref/logic/importer/fileformat/RepecNepImporter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -330,22 +330,22 @@ private void parseAdditionalFields(BibEntry be, boolean multilineUrlFieldAllowed
// skip keyword
this.lastLine = "".equals(this.lastLine) ? "" : this.lastLine.substring(this.lastLine.indexOf(':') + 1).trim();

// parse keywords field
if ("Keywords".equals(keyword)) {
// parse keywords field
String content = readMultipleLines(in);
String[] keywords = content.split("[,;]");
be.addKeywords(Arrays.asList(keywords),
importFormatPreferences.getKeywordSeparator());
// parse JEL field
} else if ("JEL".equals(keyword)) {
// parse JEL field
be.setField(new UnknownField("jel"), readMultipleLines(in));

} else if (keyword.startsWith("Date")) {
// parse date field
String content = readMultipleLines(in);
Date.parse(content).ifPresent(be::setDate);
// parse URL field
} else if (keyword.startsWith("URL")) {
// parse URL field
String content;
if (multilineUrlFieldAllowed) {
content = readMultipleLines(in);
Expand All @@ -354,10 +354,8 @@ private void parseAdditionalFields(BibEntry be, boolean multilineUrlFieldAllowed
readLine(in);
}
be.setField(StandardField.URL, content);

// authors field
} else if (keyword.startsWith("By")) {
// parse authors
// parse authors field
parseAuthors(be, in);
} else {
readLine(in);
Expand Down
58 changes: 27 additions & 31 deletions src/main/java/org/jabref/logic/xmp/DublinCoreExtractor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,13 +67,11 @@ private void extractAuthor() {

/**
* Year in BibTex - Date in DublinCore is only the year information, because dc interprets empty months as January.
* Tries to extract the month as well.
* In JabRef the bibtex/month/value is prioritized.
* <br/>
* The problem is the default value of the calendar, which is always January, also if there is no month information in
* the xmp metdata. The idea is, to reject all information with YYYY-01-01. In cases, where xmp is written with JabRef
* the month property filled with jan will override this behavior and no data is lost. In the cases, where xmp
* is written by another service, the assumption is, that the 1st January is not a publication date at all.
* Tries to extract the month as well. In JabRef the bibtex/month/value is prioritized. <br/> The problem is the
* default value of the calendar, which is always January, also if there is no month information in the xmp metdata.
* The idea is, to reject all information with YYYY-01-01. In cases, where xmp is written with JabRef the month
* property filled with jan will override this behavior and no data is lost. In the cases, where xmp is written by
* another service, the assumption is, that the 1st January is not a publication date at all.
*/
private void extractYearAndMonth() {
List<String> dates = dcSchema.getUnqualifiedSequenceValueList("date");
Expand All @@ -87,12 +85,11 @@ private void extractYearAndMonth() {
}
if (calender != null) {
bibEntry.setField(StandardField.YEAR, String.valueOf(calender.get(Calendar.YEAR)));
int monthNumber = calender.get(Calendar.MONTH) + 1;
// not the 1st of January
if (!((calender.get(Calendar.MONTH) == 0) && (calender.get(Calendar.DAY_OF_MONTH) == 1))) {
Optional<Month> month = Month.getMonthByNumber(calender.get(Calendar.MONTH) + 1);
if (month.isPresent()) {
bibEntry.setField(StandardField.MONTH, month.get().getShortName());
}
if (!((monthNumber == 1) && (calender.get(Calendar.DAY_OF_MONTH) == 1))) {
Month.getMonthByNumber(monthNumber)
.ifPresent(month -> bibEntry.setMonth(month));
}
}
}
Expand Down Expand Up @@ -129,15 +126,13 @@ private void extractPublisher() {
}

/**
* This method sets all fields, which are custom in bibtext and therefore supported by jabref, but which are not included in the DublinCore format.
* <p/>
* This method sets all fields, which are custom in BibTeX and therefore supported by JabRef, but which are not
* included in the DublinCore format.
* <p>
* The relation attribute of DublinCore is abused to insert these custom fields.
*/
private void extractBibTexFields() {
List<String> relationships = dcSchema.getRelations();

Predicate<String> isBibTeXElement = s -> s.startsWith("bibtex/");

Consumer<String> splitBibTeXElement = s -> {
// the default pattern is bibtex/key/value, but some fields contains url etc.
// so the value property contains additional slashes, which makes the usage of
Expand All @@ -154,15 +149,15 @@ private void extractBibTexFields() {
// see also DublinCoreExtractor#extractYearAndMonth
if (StandardField.MONTH.equals(key)) {
Optional<Month> parsedMonth = Month.parse(value);
parsedMonth.ifPresent(month -> bibEntry.setField(key, month.getShortName()));
parsedMonth.ifPresent(bibEntry::setMonth);
}
}

};
List<String> relationships = dcSchema.getRelations();
if (relationships != null) {
relationships.stream()
.filter(isBibTeXElement)
.forEach(splitBibTeXElement);
.filter(isBibTeXElement)
.forEach(splitBibTeXElement);
}
}

Expand Down Expand Up @@ -220,26 +215,27 @@ private void extractType() {
}

/**
* Helper function for retrieving a BibEntry from the DublinCore metadata
* in a PDF file.
* Helper function for retrieving a BibEntry from the DublinCore metadata in a PDF file.
* <p>
* To understand how to get hold of a DublinCore have a look in the test cases for XMPUtil.
* <p>
* The BibEntry is build by mapping individual fields in the dublin core (like creator, title, subject) to fields in
* a bibtex bibEntry. In case special "bibtex/" entries are contained, the normal dublin core fields take
* precedence. For instance, the dublin core date takes precedence over bibtex/month.
*
* To understand how to get hold of a DublinCore have a look in the
* test cases for XMPUtil.
*
* The BibEntry is build by mapping individual fields in the dublin core
* (like creator, title, subject) to fields in a bibtex bibEntry.
*
* @return The bibtex bibEntry found in the document information.
* @return The bibEntry extracted from the document information.
*/
public Optional<BibEntry> extractBibtexEntry() {
// first extract "bibtex/" entries
this.extractBibTexFields();

// then extract all "standard" dublin core entries
this.extractEditor();
this.extractAuthor();
this.extractYearAndMonth();
this.extractAbstract();
this.extractDOI();
this.extractPublisher();
this.extractBibTexFields();
this.extractRights();
this.extractSource();
this.extractSubject();
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/org/jabref/model/entry/BibEntry.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -614,7 +614,7 @@ public Object clone() {
*/
@Override
public String toString() {
return CanonicalBibtexEntry.getCanonicalRepresentation(this);
return CanonicalBibEntry.getCanonicalRepresentation(this);
}

/**
Expand Down
Loading

0 comments on commit 19393bf

Please sign in to comment.