Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Unify month field formatting in test bib files #5578

Merged
merged 3 commits into from
Nov 13, 2019
Merged

Unify month field formatting in test bib files #5578

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
fafc402
Unify month field formatting in test bib files (fixes JabRef/jabref#5…
sfo Sep 5, 2019
28bd008
Fix month creation. #name# is really used within JabRef to determine …
koppor Nov 8, 2019
7c74886
Merge all BibEntry test classes into one
koppor Nov 13, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions src/main/java/org/jabref/logic/exporter/BibTeXMLExporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.stream.Collectors;

import javax.xml.bind.JAXBContext;
Expand Down Expand Up @@ -40,6 +41,7 @@
import org.jabref.logic.util.StandardFileType;
import org.jabref.model.database.BibDatabaseContext;
import org.jabref.model.entry.BibEntry;
import org.jabref.model.entry.Month;
import org.jabref.model.entry.field.Field;
import org.jabref.model.entry.field.StandardField;
import org.jabref.model.entry.types.EntryType;
Expand Down Expand Up @@ -153,6 +155,13 @@ private void parseInbook(Inbook inbook, BibEntry bibEntry, Entry entry) {
JAXBElement<BigInteger> number = new JAXBElement<>(new QName(BIBTEXML_NAMESPACE_URI, "number"),
BigInteger.class, new BigInteger(value));
inbook.getContent().add(number);
} else if (StandardField.MONTH.equals(key)) {
Optional<Month> month = bibEntry.getMonth();
if (month.isPresent()) {
JAXBElement<String> element = new JAXBElement<>(new QName(BIBTEXML_NAMESPACE_URI, key.getName()),
String.class, month.get().getFullName());
inbook.getContent().add(element);
}
} else {
JAXBElement<String> element = new JAXBElement<>(new QName(BIBTEXML_NAMESPACE_URI, key.getName()), String.class,
value);
Expand Down Expand Up @@ -205,6 +214,12 @@ private <T> void parse(T entryType, BibEntry bibEntry, Entry entry) {
LOGGER.warn("The value %s of the 'number' field is not an integer and thus is ignored for the export", value);
}
break;
} else if (StandardField.MONTH.equals(key)) {
Optional<Month> month = bibEntry.getMonth();
if (month.isPresent()) {
method.invoke(entryType, month.get().getFullName());
}
break;
} else {
method.invoke(entryType, value);
break;
Expand Down
17 changes: 16 additions & 1 deletion src/main/java/org/jabref/logic/importer/fileformat/BibTeXMLImporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.regex.Pattern;

import javax.xml.bind.JAXBContext;
Expand All @@ -27,6 +28,7 @@
import org.jabref.logic.importer.fileformat.bibtexml.Incollection;
import org.jabref.logic.util.StandardFileType;
import org.jabref.model.entry.BibEntry;
import org.jabref.model.entry.Month;
import org.jabref.model.entry.field.Field;
import org.jabref.model.entry.field.FieldFactory;
import org.jabref.model.entry.field.StandardField;
Expand Down Expand Up @@ -176,6 +178,9 @@ private <T> void parse(T entryType, Map<Field, String> fields) {
} else if (method.getName().equals("getNumber")) {
putNumber(fields, (BigInteger) method.invoke(entryType));
continue;
} else if (method.getName().equals("getMonth")) {
putMonth(fields, Month.parse((String) method.invoke(entryType)));
continue;
} else if (isMethodToIgnore(method.getName())) {
continue;
} else if (method.getName().startsWith("get")) {
Expand Down Expand Up @@ -208,7 +213,11 @@ private void parseInbook(Inbook inbook, Map<Field, String> fields) {
Object elementValue = element.getValue();
if (elementValue instanceof String) {
String value = (String) elementValue;
putIfValueNotNull(fields, field, value);
if (StandardField.MONTH.equals(field)) {
putMonth(fields, Month.parse(value));
} else {
putIfValueNotNull(fields, field, value);
}
} else if (elementValue instanceof BigInteger) {
BigInteger value = (BigInteger) elementValue;
if (StandardField.NUMBER.equals(field)) {
Expand Down Expand Up @@ -241,6 +250,12 @@ private void putNumber(Map<Field, String> fields, BigInteger number) {
}
}

private void putMonth(Map<Field, String> fields, Optional<Month> month) {
if (month.isPresent()) {
fields.put(StandardField.MONTH, month.get().getJabRefFormat());
}
}

private void putIfValueNotNull(Map<Field, String> fields, Field field, String value) {
if (value != null) {
fields.put(field, value);
Expand Down
3 changes: 0 additions & 3 deletions src/main/java/org/jabref/logic/importer/fileformat/BibtexParser.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -589,7 +589,6 @@ private String parseFieldContent(Field field) throws IOException {
int character;

while (((character = peek()) != ',') && (character != '}') && (character != ')')) {

if (eof) {
throw new IOException("Error in line " + line + ": EOF in mid-string");
}
Expand All @@ -602,7 +601,6 @@ private String parseFieldContent(Field field) throws IOException {
// brackets to know when the string is finished.
StringBuilder text = parseBracketedTextExactly();
value.append(fieldContentParser.format(text, field));

} else if (Character.isDigit((char) character)) { // value is a number
String number = parseTextToken();
value.append(number);
Expand All @@ -619,7 +617,6 @@ private String parseFieldContent(Field field) throws IOException {
skipWhitespace();
}
return value.toString();

}

/**
Expand Down
16 changes: 7 additions & 9 deletions src/main/java/org/jabref/logic/importer/fileformat/IsiImporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,7 @@ private static void processCapitalization(Map<Field, String> map) {
public ParserResult importDatabase(BufferedReader reader) throws IOException {
Objects.requireNonNull(reader);

List<BibEntry> bibitems = new ArrayList<>();
List<BibEntry> bibEntries = new ArrayList<>();
StringBuilder sb = new StringBuilder();

// Pattern fieldPattern = Pattern.compile("^AU |^TI |^SO |^DT |^C1 |^AB
Expand Down Expand Up @@ -233,7 +233,6 @@ public ParserResult importDatabase(BufferedReader reader) throws IOException {
} else if ("SO".equals(beg) || "JA".equals(beg)) {
hm.put(StandardField.JOURNAL, EOL_PATTERN.matcher(value).replaceAll(" "));
} else if ("ID".equals(beg) || "KW".equals(beg)) {

value = EOL_PATTERN.matcher(value).replaceAll(" ");
String existingKeywords = hm.get(StandardField.KEYWORDS);
if ((existingKeywords == null) || existingKeywords.contains(value)) {
Expand All @@ -242,7 +241,6 @@ public ParserResult importDatabase(BufferedReader reader) throws IOException {
existingKeywords += ", " + value;
}
hm.put(StandardField.KEYWORDS, existingKeywords);

} else if ("AB".equals(beg)) {
hm.put(StandardField.ABSTRACT, EOL_PATTERN.matcher(value).replaceAll(" "));
} else if ("BP".equals(beg) || "BR".equals(beg) || "SP".equals(beg)) {
Expand Down Expand Up @@ -271,12 +269,10 @@ public ParserResult importDatabase(BufferedReader reader) throws IOException {
} else if ("DI".equals(beg)) {
hm.put(StandardField.DOI, value);
} else if ("PD".equals(beg)) {

String month = IsiImporter.parseMonth(value);
if (month != null) {
hm.put(StandardField.MONTH, month);
}

} else if ("DT".equals(beg)) {
if ("Review".equals(value)) {
type = StandardEntryType.Article; // set "Review" in Note/Comment?
Expand Down Expand Up @@ -327,17 +323,19 @@ public ParserResult importDatabase(BufferedReader reader) throws IOException {

b.setField(hm);

bibitems.add(b);
bibEntries.add(b);
}
return new ParserResult(bibitems);
return new ParserResult(bibEntries);
}

private static String parsePages(String value) {
return value.replace("-", "--");
}

public static String parseMonth(String value) {

/**
* Parses the month and returns it in the JabRef format
*/
static String parseMonth(String value) {
String[] parts = value.split("\\s|\\-");
for (String part1 : parts) {
Optional<Month> month = Month.getMonthByShortName(part1.toLowerCase(Locale.ROOT));
Expand Down
7 changes: 6 additions & 1 deletion src/main/java/org/jabref/logic/importer/fileformat/MedlineImporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;

import javax.xml.bind.JAXBContext;
import javax.xml.bind.JAXBElement;
Expand Down Expand Up @@ -74,6 +75,7 @@
import org.jabref.logic.importer.fileformat.medline.Text;
import org.jabref.logic.util.StandardFileType;
import org.jabref.model.entry.BibEntry;
import org.jabref.model.entry.Month;
import org.jabref.model.entry.field.Field;
import org.jabref.model.entry.field.FieldFactory;
import org.jabref.model.entry.field.InternalField;
Expand Down Expand Up @@ -603,7 +605,10 @@ private void addPubDate(Map<Field, String> fields, PubDate pubDate) {
} else {
fields.put(StandardField.YEAR, pubDate.getYear());
if (pubDate.getMonth() != null) {
fields.put(StandardField.MONTH, pubDate.getMonth());
Optional<Month> month = Month.parse(pubDate.getMonth());
if (month.isPresent()) {
fields.put(StandardField.MONTH, month.get().getJabRefFormat());
}
} else if (pubDate.getSeason() != null) {
fields.put(new UnknownField("season"), pubDate.getSeason());
}
Expand Down
10 changes: 4 additions & 6 deletions src/main/java/org/jabref/logic/importer/fileformat/RepecNepImporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -330,22 +330,22 @@ private void parseAdditionalFields(BibEntry be, boolean multilineUrlFieldAllowed
// skip keyword
this.lastLine = "".equals(this.lastLine) ? "" : this.lastLine.substring(this.lastLine.indexOf(':') + 1).trim();

// parse keywords field
if ("Keywords".equals(keyword)) {
// parse keywords field
String content = readMultipleLines(in);
String[] keywords = content.split("[,;]");
be.addKeywords(Arrays.asList(keywords),
importFormatPreferences.getKeywordSeparator());
// parse JEL field
} else if ("JEL".equals(keyword)) {
// parse JEL field
be.setField(new UnknownField("jel"), readMultipleLines(in));

} else if (keyword.startsWith("Date")) {
// parse date field
String content = readMultipleLines(in);
Date.parse(content).ifPresent(be::setDate);
// parse URL field
} else if (keyword.startsWith("URL")) {
// parse URL field
String content;
if (multilineUrlFieldAllowed) {
content = readMultipleLines(in);
Expand All @@ -354,10 +354,8 @@ private void parseAdditionalFields(BibEntry be, boolean multilineUrlFieldAllowed
readLine(in);
}
be.setField(StandardField.URL, content);

// authors field
} else if (keyword.startsWith("By")) {
// parse authors
// parse authors field
parseAuthors(be, in);
} else {
readLine(in);
Expand Down
58 changes: 27 additions & 31 deletions src/main/java/org/jabref/logic/xmp/DublinCoreExtractor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,13 +67,11 @@ private void extractAuthor() {

/**
* Year in BibTex - Date in DublinCore is only the year information, because dc interprets empty months as January.
* Tries to extract the month as well.
* In JabRef the bibtex/month/value is prioritized.
* <br/>
* The problem is the default value of the calendar, which is always January, also if there is no month information in
* the xmp metdata. The idea is, to reject all information with YYYY-01-01. In cases, where xmp is written with JabRef
* the month property filled with jan will override this behavior and no data is lost. In the cases, where xmp
* is written by another service, the assumption is, that the 1st January is not a publication date at all.
* Tries to extract the month as well. In JabRef the bibtex/month/value is prioritized. <br/> The problem is the
* default value of the calendar, which is always January, also if there is no month information in the xmp metdata.
* The idea is, to reject all information with YYYY-01-01. In cases, where xmp is written with JabRef the month
* property filled with jan will override this behavior and no data is lost. In the cases, where xmp is written by
* another service, the assumption is, that the 1st January is not a publication date at all.
*/
private void extractYearAndMonth() {
List<String> dates = dcSchema.getUnqualifiedSequenceValueList("date");
Expand All @@ -87,12 +85,11 @@ private void extractYearAndMonth() {
}
if (calender != null) {
bibEntry.setField(StandardField.YEAR, String.valueOf(calender.get(Calendar.YEAR)));
int monthNumber = calender.get(Calendar.MONTH) + 1;
// not the 1st of January
if (!((calender.get(Calendar.MONTH) == 0) && (calender.get(Calendar.DAY_OF_MONTH) == 1))) {
Optional<Month> month = Month.getMonthByNumber(calender.get(Calendar.MONTH) + 1);
if (month.isPresent()) {
bibEntry.setField(StandardField.MONTH, month.get().getShortName());
}
if (!((monthNumber == 1) && (calender.get(Calendar.DAY_OF_MONTH) == 1))) {
Month.getMonthByNumber(monthNumber)
.ifPresent(month -> bibEntry.setMonth(month));
}
}
}
Expand Down Expand Up @@ -129,15 +126,13 @@ private void extractPublisher() {
}

/**
* This method sets all fields, which are custom in bibtext and therefore supported by jabref, but which are not included in the DublinCore format.
* <p/>
* This method sets all fields, which are custom in BibTeX and therefore supported by JabRef, but which are not
* included in the DublinCore format.
* <p>
* The relation attribute of DublinCore is abused to insert these custom fields.
*/
private void extractBibTexFields() {
List<String> relationships = dcSchema.getRelations();

Predicate<String> isBibTeXElement = s -> s.startsWith("bibtex/");

Consumer<String> splitBibTeXElement = s -> {
// the default pattern is bibtex/key/value, but some fields contains url etc.
// so the value property contains additional slashes, which makes the usage of
Expand All @@ -154,15 +149,15 @@ private void extractBibTexFields() {
// see also DublinCoreExtractor#extractYearAndMonth
if (StandardField.MONTH.equals(key)) {
Optional<Month> parsedMonth = Month.parse(value);
parsedMonth.ifPresent(month -> bibEntry.setField(key, month.getShortName()));
parsedMonth.ifPresent(bibEntry::setMonth);
}
}

};
List<String> relationships = dcSchema.getRelations();
if (relationships != null) {
relationships.stream()
.filter(isBibTeXElement)
.forEach(splitBibTeXElement);
.filter(isBibTeXElement)
.forEach(splitBibTeXElement);
}
}

Expand Down Expand Up @@ -220,26 +215,27 @@ private void extractType() {
}

/**
* Helper function for retrieving a BibEntry from the DublinCore metadata
* in a PDF file.
* Helper function for retrieving a BibEntry from the DublinCore metadata in a PDF file.
* <p>
* To understand how to get hold of a DublinCore have a look in the test cases for XMPUtil.
* <p>
* The BibEntry is build by mapping individual fields in the dublin core (like creator, title, subject) to fields in
* a bibtex bibEntry. In case special "bibtex/" entries are contained, the normal dublin core fields take
* precedence. For instance, the dublin core date takes precedence over bibtex/month.
*
* To understand how to get hold of a DublinCore have a look in the
* test cases for XMPUtil.
*
* The BibEntry is build by mapping individual fields in the dublin core
* (like creator, title, subject) to fields in a bibtex bibEntry.
*
* @return The bibtex bibEntry found in the document information.
* @return The bibEntry extracted from the document information.
*/
public Optional<BibEntry> extractBibtexEntry() {
// first extract "bibtex/" entries
this.extractBibTexFields();

// then extract all "standard" dublin core entries
this.extractEditor();
this.extractAuthor();
this.extractYearAndMonth();
this.extractAbstract();
this.extractDOI();
this.extractPublisher();
this.extractBibTexFields();
this.extractRights();
this.extractSource();
this.extractSubject();
Expand Down
2 changes: 1 addition & 1 deletion src/main/java/org/jabref/model/entry/BibEntry.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -614,7 +614,7 @@ public Object clone() {
*/
@Override
public String toString() {
return CanonicalBibtexEntry.getCanonicalRepresentation(this);
return CanonicalBibEntry.getCanonicalRepresentation(this);
}

/**
Expand Down
Loading