From 11d7620ae0f9a464618870228e111e8eab738df2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20M=C3=A9ndez?= Date: Fri, 31 May 2019 02:05:04 +0200 Subject: [PATCH 01/57] TEX parser: classes and tests --- .../logic/texparser/DefaultTexParser.java | 139 ++++++++ .../org/jabref/model/texparser/Citation.java | 90 +++++ .../org/jabref/model/texparser/TexParser.java | 21 ++ .../model/texparser/TexParserResult.java | 66 ++++ .../jabref/logic/texparser/TexParserTest.java | 329 ++++++++++++++++++ .../org/jabref/logic/texparser/config.bib | 26 ++ .../org/jabref/logic/texparser/origin.bib | 39 +++ .../org/jabref/logic/texparser/paper.tex | 13 + .../org/jabref/logic/texparser/paper2.tex | 10 + .../jabref/logic/texparser/unknown_key.tex | 10 + 10 files changed, 743 insertions(+) create mode 100644 src/main/java/org/jabref/logic/texparser/DefaultTexParser.java create mode 100644 src/main/java/org/jabref/model/texparser/Citation.java create mode 100644 src/main/java/org/jabref/model/texparser/TexParser.java create mode 100644 src/main/java/org/jabref/model/texparser/TexParserResult.java create mode 100644 src/test/java/org/jabref/logic/texparser/TexParserTest.java create mode 100644 src/test/resources/org/jabref/logic/texparser/config.bib create mode 100644 src/test/resources/org/jabref/logic/texparser/origin.bib create mode 100644 src/test/resources/org/jabref/logic/texparser/paper.tex create mode 100644 src/test/resources/org/jabref/logic/texparser/paper2.tex create mode 100644 src/test/resources/org/jabref/logic/texparser/unknown_key.tex diff --git a/src/main/java/org/jabref/logic/texparser/DefaultTexParser.java b/src/main/java/org/jabref/logic/texparser/DefaultTexParser.java new file mode 100644 index 00000000000..ea366c9b7b9 --- /dev/null +++ b/src/main/java/org/jabref/logic/texparser/DefaultTexParser.java @@ -0,0 +1,139 @@ +package org.jabref.logic.texparser; + +import java.io.IOException; +import java.io.LineNumberReader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.jabref.model.database.BibDatabase; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.texparser.Citation; +import org.jabref.model.texparser.TexParser; +import org.jabref.model.texparser.TexParserResult; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class DefaultTexParser implements TexParser { + + private static final Logger LOGGER = LoggerFactory.getLogger(DefaultTexParser.class); + + /** + * It is allowed to add new cite commands for pattern matching. + */ + private static final String[] CITE_COMMANDS = new String[] {"cite", "citep", "citet"}; + private static final String CITE_REGEX = String.format("\\\\(?:%s)\\{([^\\}]*)\\}", + String.join("|", CITE_COMMANDS)); + + private final BibDatabase masterDatabase; + + public DefaultTexParser(BibDatabase database) { + masterDatabase = database; + } + + @Override + public TexParserResult parse(Path texFile) { + return parseTexFiles(Arrays.asList(texFile)); + } + + @Override + public TexParserResult parse(List texFiles) { + return parseTexFiles(texFiles); + } + + private TexParserResult parseTexFiles(List texFiles) { + TexParserResult result = new TexParserResult(masterDatabase); + int fileIndex = 0; + + while (fileIndex < texFiles.size()) { + Path file = texFiles.get(fileIndex++); + + try (LineNumberReader lnr = new LineNumberReader(Files.newBufferedReader(file))) { + for (String line = lnr.readLine(); line != null; line = lnr.readLine()) { + // Skip comment lines. + if (line.startsWith("%")) { + continue; + } + + matchCitation(result, file, lnr.getLineNumber(), line); + } + } catch (IOException e) { + LOGGER.warn("Error opening the TEX file", e); + } + } + + resolveTags(result); + return result; + } + + /* + * Find cites along a specific line and store them. + */ + private void matchCitation(TexParserResult result, Path file, int lineNumber, String line) { + Matcher citeMatch = Pattern.compile(CITE_REGEX).matcher(line); + + while (citeMatch.find()) { + String[] keys = citeMatch.group(1).split(","); + + for (String key : keys) { + addKey(result, key.trim(), new Citation(file, lineNumber, citeMatch.start(), citeMatch.end(), line)); + } + } + } + + /* + * Add a citation to the uniqueKeys map. + */ + private void addKey(TexParserResult result, String key, Citation citation) { + Map> uniqueKeys = result.getUniqueKeys(); + + if (!uniqueKeys.containsKey(key)) { + uniqueKeys.put(key, new ArrayList<>()); + } + + if (!uniqueKeys.get(key).contains(citation)) { + uniqueKeys.get(key).add(citation); + } + } + + /* + * Look for an equivalent BibTeX entry within the reference database for all keys inside of the TEX files. + */ + private void resolveTags(TexParserResult result) { + Set keySet = result.getUniqueKeys().keySet(); + + for (String key : keySet) { + if (!result.getGeneratedBibDatabase().getEntryByKey(key).isPresent()) { + Optional entry = masterDatabase.getEntryByKey(key); + + if (entry.isPresent()) { + insertEntry(result, entry.get()); + } else { + result.getUnresolvedKeys().add(key); + } + } + } + + // Copy database definitions + if (result.getGeneratedBibDatabase().hasEntries()) { + result.getGeneratedBibDatabase().copyPreamble(masterDatabase); + result.insertStrings(masterDatabase.getUsedStrings(result.getGeneratedBibDatabase().getEntries())); + } + } + + /* + * Insert into the database a clone of the given entry. The cloned entry has a new unique ID. + */ + private void insertEntry(TexParserResult result, BibEntry entry) { + BibEntry clonedEntry = (BibEntry) entry.clone(); + result.getGeneratedBibDatabase().insertEntry(clonedEntry); + } +} diff --git a/src/main/java/org/jabref/model/texparser/Citation.java b/src/main/java/org/jabref/model/texparser/Citation.java new file mode 100644 index 00000000000..6a9206ddc77 --- /dev/null +++ b/src/main/java/org/jabref/model/texparser/Citation.java @@ -0,0 +1,90 @@ +package org.jabref.model.texparser; + +import java.nio.file.Path; +import java.util.Objects; + +public class Citation { + + /** + * The total number of characters that are shown around a cite (cite width included). + */ + private static final int CONTEXT_WIDTH = 50; + + private final Path path; + private final int line; + private final int colStart; + private final int colEnd; + private final String lineText; + + public Citation(Path path, int line, int colStart, int colEnd, String lineText) { + this.path = path; + this.line = line; + this.colStart = colStart; + this.colEnd = colEnd; + this.lineText = lineText; + } + + public Path getPath() { + return path; + } + + public int getLine() { + return line; + } + + public int getColStart() { + return colStart; + } + + public int getColEnd() { + return colEnd; + } + + public String getLineText() { + return lineText; + } + + /** + * @return String that contains a cite and the text that surrounds it along the same line. + */ + public String getContext() { + int center = (colStart + colEnd) / 2; + int lineLength = lineText.length(); + + int start = Math.max(0, (center + CONTEXT_WIDTH / 2 < lineLength) + ? center - CONTEXT_WIDTH / 2 + : lineLength - CONTEXT_WIDTH); + int end = Math.min(lineLength, start + CONTEXT_WIDTH); + + return lineText.substring(start, end); + } + + @Override + public String toString() { + return String.format("%s\t%d:%d-%d\t%s", path.toString(), line, colStart, colEnd, getContext()); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + + if (o == null || getClass() != o.getClass()) { + return false; + } + + Citation citation = (Citation) o; + + return line == citation.line + && colStart == citation.colStart + && colEnd == citation.colEnd + && Objects.equals(path, citation.path) + && Objects.equals(lineText, citation.lineText); + } + + @Override + public int hashCode() { + return Objects.hash(path, line, colStart, colEnd, lineText); + } +} diff --git a/src/main/java/org/jabref/model/texparser/TexParser.java b/src/main/java/org/jabref/model/texparser/TexParser.java new file mode 100644 index 00000000000..6afa191564d --- /dev/null +++ b/src/main/java/org/jabref/model/texparser/TexParser.java @@ -0,0 +1,21 @@ +package org.jabref.model.texparser; + +import java.nio.file.Path; +import java.util.List; + +public interface TexParser { + + /** + * @param texFile Path to a TEX file + * @return a TexParserResult, which contains the generated BibDatabase and all data related to the bibliographic + * entries + */ + TexParserResult parse(Path texFile); + + /** + * @param texFiles List of Path objects linked to a TEX file + * @return a list of TexParserResult objects, which contains the generated BibDatabase and all data related to the + * bibliographic entries + */ + TexParserResult parse(List texFiles); +} diff --git a/src/main/java/org/jabref/model/texparser/TexParserResult.java b/src/main/java/org/jabref/model/texparser/TexParserResult.java new file mode 100644 index 00000000000..952982780e0 --- /dev/null +++ b/src/main/java/org/jabref/model/texparser/TexParserResult.java @@ -0,0 +1,66 @@ +package org.jabref.model.texparser; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.jabref.model.database.BibDatabase; +import org.jabref.model.entry.BibtexString; + +public class TexParserResult { + + private final BibDatabase masterDatabase; + private final Map> uniqueKeys = new HashMap<>(); + private final List unresolvedKeys = new ArrayList<>(); + private final BibDatabase texDatabase = new BibDatabase(); + private int insertedStrings = 0; + + public TexParserResult(BibDatabase masterDatabase) { + this.masterDatabase = masterDatabase; + } + + public BibDatabase getMasterDatabase() { + return masterDatabase; + } + + public Map> getUniqueKeys() { + return uniqueKeys; + } + + public int getFoundKeysInTex() { + return uniqueKeys.size(); + } + + public int getCitationsCountByKey(String key) { + return uniqueKeys.get(key).size(); + } + + public List getUnresolvedKeys() { + return unresolvedKeys; + } + + public int getUnresolvedKeysCount() { + return unresolvedKeys.size(); + } + + public BibDatabase getGeneratedBibDatabase() { + return texDatabase; + } + + public int getResolvedKeysCount() { + return texDatabase.getEntryCount(); + } + + public int getInsertedStrings() { + return insertedStrings; + } + + public void insertStrings(Collection usedStrings) { + for (BibtexString string : usedStrings) { + texDatabase.addString(string); + insertedStrings++; + } + } +} diff --git a/src/test/java/org/jabref/logic/texparser/TexParserTest.java b/src/test/java/org/jabref/logic/texparser/TexParserTest.java new file mode 100644 index 00000000000..abc8562db16 --- /dev/null +++ b/src/test/java/org/jabref/logic/texparser/TexParserTest.java @@ -0,0 +1,329 @@ +package org.jabref.logic.texparser; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Optional; + +import org.jabref.logic.importer.ImportFormatPreferences; +import org.jabref.logic.importer.ParserResult; +import org.jabref.logic.importer.fileformat.BibtexParser; +import org.jabref.model.database.BibDatabase; +import org.jabref.model.texparser.TexParser; +import org.jabref.model.texparser.TexParserResult; +import org.jabref.model.util.DummyFileUpdateMonitor; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Answers; + +import static org.junit.Assert.assertFalse; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; + +class TexParserTest { + private ImportFormatPreferences importFormatPreferences; + + @BeforeEach + void setUp() { + importFormatPreferences = mock(ImportFormatPreferences.class, Answers.RETURNS_DEEP_STUBS); + } + + @AfterEach + void tearDown() { + importFormatPreferences = null; + } + + @Test + void testSingleFile() throws URISyntaxException, IOException { + final String DARWIN = "Darwin1888"; + final String EINSTEIN = "Einstein1920"; + + InputStream originalStream = TexParserTest.class.getResourceAsStream("origin.bib"); + Path texFile = Paths.get(TexParserTest.class.getResource("paper.tex").toURI()); + + try (InputStreamReader originalReader = new InputStreamReader(originalStream, StandardCharsets.UTF_8)) { + ParserResult result = new BibtexParser(importFormatPreferences, + new DummyFileUpdateMonitor()).parse(originalReader); + TexParser texParser = new DefaultTexParser(result.getDatabase()); + TexParserResult texResult = texParser.parse(texFile); + BibDatabase newDatabase = texResult.getGeneratedBibDatabase(); + + assertEquals(result.getDatabase(), texResult.getMasterDatabase()); + + // Check entries + assertTrue(texResult.getGeneratedBibDatabase().hasEntries()); + assertEquals(2, texResult.getCitationsCountByKey(DARWIN)); + assertEquals(2, texResult.getCitationsCountByKey(EINSTEIN)); + assertEquals(2, texResult.getFoundKeysInTex()); + assertEquals(0, texResult.getUnresolvedKeysCount()); + assertEquals(2, texResult.getResolvedKeysCount()); + assertEquals(2, newDatabase.getEntries().size()); + + // Check paths + assertEquals(texFile, texResult.getUniqueKeys().get(DARWIN).get(0).getPath()); + assertEquals(texFile, texResult.getUniqueKeys().get(EINSTEIN).get(0).getPath()); + + // Check lines + assertEquals(4, texResult.getUniqueKeys().get(EINSTEIN).get(0).getLine()); + assertEquals(5, texResult.getUniqueKeys().get(DARWIN).get(0).getLine()); + assertEquals(6, texResult.getUniqueKeys().get(EINSTEIN).get(1).getLine()); + assertEquals(7, texResult.getUniqueKeys().get(DARWIN).get(1).getLine()); + + // Check columns + assertEquals(0, texResult.getUniqueKeys().get(EINSTEIN).get(0).getColStart()); + assertEquals(19, texResult.getUniqueKeys().get(EINSTEIN).get(0).getColEnd()); + + assertEquals(67, texResult.getUniqueKeys().get(DARWIN).get(0).getColStart()); + assertEquals(84, texResult.getUniqueKeys().get(DARWIN).get(0).getColEnd()); + + assertEquals(14, texResult.getUniqueKeys().get(EINSTEIN).get(1).getColStart()); + assertEquals(33, texResult.getUniqueKeys().get(EINSTEIN).get(1).getColEnd()); + + assertEquals(0, texResult.getUniqueKeys().get(DARWIN).get(1).getColStart()); + assertEquals(17, texResult.getUniqueKeys().get(DARWIN).get(1).getColEnd()); + + // Check line texts + assertEquals("\\cite{Einstein1920}", texResult.getUniqueKeys().get(EINSTEIN).get(0).getLineText()); + assertEquals("Nunc ultricies leo nec libero rhoncus, eu vehicula enim efficitur.~\\cite{Darwin1888}", + texResult.getUniqueKeys().get(DARWIN).get(0).getLineText()); + assertEquals("Einstein said~\\cite{Einstein1920} that lorem impsum, consectetur adipiscing elit. Integer eros tortor, dictum nec aliquet in, pharetra nec justo.", + texResult.getUniqueKeys().get(EINSTEIN).get(1).getLineText()); + assertEquals("\\cite{Darwin1888}.", texResult.getUniqueKeys().get(DARWIN).get(1).getLineText()); + + // Check contexts + assertEquals("\\cite{Einstein1920}", texResult.getUniqueKeys().get(EINSTEIN).get(0).getContext()); + assertEquals("cus, eu vehicula enim efficitur.~\\cite{Darwin1888}", + texResult.getUniqueKeys().get(DARWIN).get(0).getContext()); + assertEquals("Einstein said~\\cite{Einstein1920} that lorem impsu", + texResult.getUniqueKeys().get(EINSTEIN).get(1).getContext()); + assertEquals("\\cite{Darwin1888}.", texResult.getUniqueKeys().get(DARWIN).get(1).getContext()); + } + } + + @Test + void testTwoFiles() throws URISyntaxException, IOException { + final String DARWIN = "Darwin1888"; + final String EINSTEIN = "Einstein1920"; + final String EINSTEIN_A = "Einstein1920a"; + final String EINSTEIN_B = "Einstein1920b"; + final String EINSTEIN_C = "Einstein1920c"; + + InputStream originalStream = TexParserTest.class.getResourceAsStream("origin.bib"); + Path texFile = Paths.get(TexParserTest.class.getResource("paper.tex").toURI()); + Path texFile2 = Paths.get(TexParserTest.class.getResource("paper2.tex").toURI()); + + try (InputStreamReader originalReader = new InputStreamReader(originalStream, StandardCharsets.UTF_8)) { + ParserResult result = new BibtexParser(importFormatPreferences, + new DummyFileUpdateMonitor()).parse(originalReader); + TexParser texParser = new DefaultTexParser(result.getDatabase()); + TexParserResult texResult = texParser.parse(Arrays.asList(texFile, texFile2)); + BibDatabase newDatabase = texResult.getGeneratedBibDatabase(); + + assertEquals(result.getDatabase(), texResult.getMasterDatabase()); + + // Check entries + assertTrue(texResult.getGeneratedBibDatabase().hasEntries()); + assertEquals(2, texResult.getCitationsCountByKey(DARWIN)); + assertEquals(2, texResult.getCitationsCountByKey(EINSTEIN)); + assertEquals(5, texResult.getFoundKeysInTex()); + assertEquals(0, texResult.getUnresolvedKeysCount()); + assertEquals(5, texResult.getResolvedKeysCount()); + assertEquals(5, newDatabase.getEntries().size()); + + // Check paths + assertEquals(texFile, texResult.getUniqueKeys().get(DARWIN).get(0).getPath()); + assertEquals(texFile, texResult.getUniqueKeys().get(EINSTEIN).get(0).getPath()); + + assertEquals(texFile2, texResult.getUniqueKeys().get(EINSTEIN_A).get(0).getPath()); + assertEquals(texFile2, texResult.getUniqueKeys().get(EINSTEIN_B).get(0).getPath()); + assertEquals(texFile2, texResult.getUniqueKeys().get(EINSTEIN_C).get(0).getPath()); + + // Check lines + assertEquals(4, texResult.getUniqueKeys().get(EINSTEIN).get(0).getLine()); + assertEquals(5, texResult.getUniqueKeys().get(DARWIN).get(0).getLine()); + assertEquals(6, texResult.getUniqueKeys().get(EINSTEIN).get(1).getLine()); + assertEquals(7, texResult.getUniqueKeys().get(DARWIN).get(1).getLine()); + + assertEquals(4, texResult.getUniqueKeys().get(EINSTEIN_A).get(0).getLine()); + assertEquals(5, texResult.getUniqueKeys().get(EINSTEIN_B).get(0).getLine()); + assertEquals(6, texResult.getUniqueKeys().get(EINSTEIN_C).get(0).getLine()); + + // Check columns + assertEquals(0, texResult.getUniqueKeys().get(EINSTEIN).get(0).getColStart()); + assertEquals(19, texResult.getUniqueKeys().get(EINSTEIN).get(0).getColEnd()); + + assertEquals(67, texResult.getUniqueKeys().get(DARWIN).get(0).getColStart()); + assertEquals(84, texResult.getUniqueKeys().get(DARWIN).get(0).getColEnd()); + + assertEquals(14, texResult.getUniqueKeys().get(EINSTEIN).get(1).getColStart()); + assertEquals(33, texResult.getUniqueKeys().get(EINSTEIN).get(1).getColEnd()); + + assertEquals(0, texResult.getUniqueKeys().get(DARWIN).get(1).getColStart()); + assertEquals(17, texResult.getUniqueKeys().get(DARWIN).get(1).getColEnd()); + + assertEquals(48, texResult.getUniqueKeys().get(EINSTEIN_A).get(0).getColStart()); + assertEquals(68, texResult.getUniqueKeys().get(EINSTEIN_A).get(0).getColEnd()); + + assertEquals(48, texResult.getUniqueKeys().get(EINSTEIN_B).get(0).getColStart()); + assertEquals(68, texResult.getUniqueKeys().get(EINSTEIN_B).get(0).getColEnd()); + + assertEquals(48, texResult.getUniqueKeys().get(EINSTEIN_C).get(0).getColStart()); + assertEquals(68, texResult.getUniqueKeys().get(EINSTEIN_C).get(0).getColEnd()); + + // Check line texts + assertEquals("\\cite{Einstein1920}", texResult.getUniqueKeys().get(EINSTEIN).get(0).getLineText()); + assertEquals("Nunc ultricies leo nec libero rhoncus, eu vehicula enim efficitur.~\\cite{Darwin1888}", + texResult.getUniqueKeys().get(DARWIN).get(0).getLineText()); + assertEquals("Einstein said~\\cite{Einstein1920} that lorem impsum, consectetur adipiscing elit. Integer eros tortor, dictum nec aliquet in, pharetra nec justo.", + texResult.getUniqueKeys().get(EINSTEIN).get(1).getLineText()); + assertEquals("\\cite{Darwin1888}.", texResult.getUniqueKeys().get(DARWIN).get(1).getLineText()); + + assertEquals("This is some content trying to cite a bib file: \\cite{Einstein1920a}", + texResult.getUniqueKeys().get(EINSTEIN_A).get(0).getLineText()); + assertEquals("This is some content trying to cite a bib file: \\cite{Einstein1920b}", + texResult.getUniqueKeys().get(EINSTEIN_B).get(0).getLineText()); + assertEquals("This is some content trying to cite a bib file: \\cite{Einstein1920c}", + texResult.getUniqueKeys().get(EINSTEIN_C).get(0).getLineText()); + + // Check contexts + assertEquals("\\cite{Einstein1920}", texResult.getUniqueKeys().get(EINSTEIN).get(0).getContext()); + assertEquals("cus, eu vehicula enim efficitur.~\\cite{Darwin1888}", + texResult.getUniqueKeys().get(DARWIN).get(0).getContext()); + assertEquals("Einstein said~\\cite{Einstein1920} that lorem impsu", + texResult.getUniqueKeys().get(EINSTEIN).get(1).getContext()); + assertEquals("\\cite{Darwin1888}.", texResult.getUniqueKeys().get(DARWIN).get(1).getContext()); + + assertEquals("nt trying to cite a bib file: \\cite{Einstein1920a}", + texResult.getUniqueKeys().get(EINSTEIN_A).get(0).getContext()); + assertEquals("nt trying to cite a bib file: \\cite{Einstein1920b}", + texResult.getUniqueKeys().get(EINSTEIN_B).get(0).getContext()); + assertEquals("nt trying to cite a bib file: \\cite{Einstein1920c}", + texResult.getUniqueKeys().get(EINSTEIN_C).get(0).getContext()); + } + } + + @Test + void testDuplicateFiles() throws URISyntaxException, IOException { + final String DARWIN = "Darwin1888"; + final String EINSTEIN = "Einstein1920"; + + InputStream originalStream = TexParserTest.class.getResourceAsStream("origin.bib"); + Path texFile = Paths.get(TexParserTest.class.getResource("paper.tex").toURI()); + + try (InputStreamReader originalReader = new InputStreamReader(originalStream, StandardCharsets.UTF_8)) { + ParserResult result = new BibtexParser(importFormatPreferences, + new DummyFileUpdateMonitor()).parse(originalReader); + TexParser texParser = new DefaultTexParser(result.getDatabase()); + TexParserResult texResult = texParser.parse(Arrays.asList(texFile, texFile)); + BibDatabase newDatabase = texResult.getGeneratedBibDatabase(); + + assertEquals(result.getDatabase(), texResult.getMasterDatabase()); + + // Check entries + assertTrue(texResult.getGeneratedBibDatabase().hasEntries()); + assertEquals(2, texResult.getCitationsCountByKey(DARWIN)); + assertEquals(2, texResult.getCitationsCountByKey(EINSTEIN)); + assertEquals(2, texResult.getFoundKeysInTex()); + assertEquals(0, texResult.getUnresolvedKeysCount()); + assertEquals(2, texResult.getResolvedKeysCount()); + assertEquals(2, newDatabase.getEntries().size()); + + // Check paths + assertEquals(texFile, texResult.getUniqueKeys().get(DARWIN).get(0).getPath()); + assertEquals(texFile, texResult.getUniqueKeys().get(EINSTEIN).get(0).getPath()); + + // Check lines + assertEquals(4, texResult.getUniqueKeys().get(EINSTEIN).get(0).getLine()); + assertEquals(5, texResult.getUniqueKeys().get(DARWIN).get(0).getLine()); + assertEquals(6, texResult.getUniqueKeys().get(EINSTEIN).get(1).getLine()); + assertEquals(7, texResult.getUniqueKeys().get(DARWIN).get(1).getLine()); + + // Check columns + assertEquals(0, texResult.getUniqueKeys().get(EINSTEIN).get(0).getColStart()); + assertEquals(19, texResult.getUniqueKeys().get(EINSTEIN).get(0).getColEnd()); + + assertEquals(67, texResult.getUniqueKeys().get(DARWIN).get(0).getColStart()); + assertEquals(84, texResult.getUniqueKeys().get(DARWIN).get(0).getColEnd()); + + assertEquals(14, texResult.getUniqueKeys().get(EINSTEIN).get(1).getColStart()); + assertEquals(33, texResult.getUniqueKeys().get(EINSTEIN).get(1).getColEnd()); + + assertEquals(0, texResult.getUniqueKeys().get(DARWIN).get(1).getColStart()); + assertEquals(17, texResult.getUniqueKeys().get(DARWIN).get(1).getColEnd()); + + // Check line texts + assertEquals("\\cite{Einstein1920}", texResult.getUniqueKeys().get(EINSTEIN).get(0).getLineText()); + assertEquals("Nunc ultricies leo nec libero rhoncus, eu vehicula enim efficitur.~\\cite{Darwin1888}", + texResult.getUniqueKeys().get(DARWIN).get(0).getLineText()); + assertEquals("Einstein said~\\cite{Einstein1920} that lorem impsum, consectetur adipiscing elit. Integer eros tortor, dictum nec aliquet in, pharetra nec justo.", + texResult.getUniqueKeys().get(EINSTEIN).get(1).getLineText()); + assertEquals("\\cite{Darwin1888}.", texResult.getUniqueKeys().get(DARWIN).get(1).getLineText()); + + // Check contexts + assertEquals("\\cite{Einstein1920}", texResult.getUniqueKeys().get(EINSTEIN).get(0).getContext()); + assertEquals("cus, eu vehicula enim efficitur.~\\cite{Darwin1888}", + texResult.getUniqueKeys().get(DARWIN).get(0).getContext()); + assertEquals("Einstein said~\\cite{Einstein1920} that lorem impsu", + texResult.getUniqueKeys().get(EINSTEIN).get(1).getContext()); + assertEquals("\\cite{Darwin1888}.", texResult.getUniqueKeys().get(DARWIN).get(1).getContext()); + } + } + + @Test + void testUnknownKey() throws URISyntaxException, IOException { + InputStream originalStream = TexParserTest.class.getResourceAsStream("origin.bib"); + Path texFile = Paths.get(TexParserTest.class.getResource("unknown_key.tex").toURI()); + try (InputStreamReader originalReader = new InputStreamReader(originalStream, StandardCharsets.UTF_8)) { + ParserResult result = new BibtexParser(importFormatPreferences, + new DummyFileUpdateMonitor()).parse(originalReader); + + TexParser texParser = new DefaultTexParser(result.getDatabase()); + TexParserResult texResult = texParser.parse(texFile); + BibDatabase newDatabase = texResult.getGeneratedBibDatabase(); + + assertTrue(texResult.getGeneratedBibDatabase().hasEntries()); + assertEquals(3, texResult.getFoundKeysInTex()); + assertEquals(1, texResult.getUnresolvedKeysCount()); + assertEquals(2, texResult.getResolvedKeysCount()); + assertEquals(2, newDatabase.getEntries().size()); + } + } + + @Test + void testFileNotFound() { + TexParser texParser = new DefaultTexParser(new BibDatabase()); + TexParserResult texResult = texParser.parse(Paths.get("file_not_found.tex")); + BibDatabase newDatabase = texResult.getGeneratedBibDatabase(); + + assertFalse(texResult.getGeneratedBibDatabase().hasEntries()); + assertEquals(0, texResult.getFoundKeysInTex()); + assertEquals(0, texResult.getUnresolvedKeysCount()); + assertEquals(0, texResult.getResolvedKeysCount()); + assertEquals(0, newDatabase.getEntries().size()); + } + + @Test + void testDuplicateBibDatabaseConfiguration() throws URISyntaxException, IOException { + InputStream originalStream = TexParserTest.class.getResourceAsStream("config.bib"); + Path texFile = Paths.get(TexParserTest.class.getResource("paper.tex").toURI()); + try (InputStreamReader originalReader = new InputStreamReader(originalStream, StandardCharsets.UTF_8)) { + ParserResult result = new BibtexParser(importFormatPreferences, + new DummyFileUpdateMonitor()).parse(originalReader); + + TexParser texParser = new DefaultTexParser(result.getDatabase()); + TexParserResult texResult = texParser.parse(texFile); + BibDatabase newDatabase = texResult.getGeneratedBibDatabase(); + + assertEquals(Optional.of("\"Maintained by \" # maintainer"), newDatabase.getPreamble()); + assertEquals(1, newDatabase.getStringCount()); + } + } +} diff --git a/src/test/resources/org/jabref/logic/texparser/config.bib b/src/test/resources/org/jabref/logic/texparser/config.bib new file mode 100644 index 00000000000..3e3cac593a9 --- /dev/null +++ b/src/test/resources/org/jabref/logic/texparser/config.bib @@ -0,0 +1,26 @@ +@String {maintainer = "Stefan Kolb"} +@preamble {"Maintained by " # maintainer} +@String {einstein = "Einstein, Albert"} + +@Book{Newton1999, + title = {The Principia: mathematical principles of natural philosophy}, + publisher = {Univ of California Press}, + year = {1999}, + author = {Newton, Isaac} +} + +@Book{Darwin1888, + title = {The descent of man, and selection in relation to sex}, + publisher = {J. Murray}, + year = {1888}, + author = {Darwin, Charles} +} + +@Book{Einstein1920, + title = {Relativity: The special and general theory}, + publisher = {Penguin}, + year = {1920}, + author = einstein +} + +@Comment{jabref-meta: databaseType:bibtex;} diff --git a/src/test/resources/org/jabref/logic/texparser/origin.bib b/src/test/resources/org/jabref/logic/texparser/origin.bib new file mode 100644 index 00000000000..fbea84e2bc4 --- /dev/null +++ b/src/test/resources/org/jabref/logic/texparser/origin.bib @@ -0,0 +1,39 @@ +% Encoding: UTF-8 + +@Book{Newton1999, + title = {The Principia: mathematical principles of natural philosophy}, + publisher = {Univ of California Press}, + year = {1999}, + author = {Newton, Isaac} +} + +@Book{Darwin1888, + title = {The descent of man, and selection in relation to sex}, + publisher = {J. Murray}, + year = {1888}, + author = {Darwin, Charles} +} + +@Book{Einstein1920, + title = {Relativity: The special and general theory}, + publisher = {Penguin}, + year = {1920}, + author = {Einstein, Albert} +} + +@InBook{Einstein1920a, + crossref = {Einstein1920}, + pages = {22--23} +} + +@InBook{Einstein1920b, + crossref = {Einstein1921}, + pages = {22--23} +} + +@InBook{Einstein1920c, + crossref = {Einstein1920}, + pages = {25--33} +} + +@Comment{jabref-meta: databaseType:bibtex;} diff --git a/src/test/resources/org/jabref/logic/texparser/paper.tex b/src/test/resources/org/jabref/logic/texparser/paper.tex new file mode 100644 index 00000000000..6379c6ee027 --- /dev/null +++ b/src/test/resources/org/jabref/logic/texparser/paper.tex @@ -0,0 +1,13 @@ +\documentclass{article} + +\begin{document} +\cite{Einstein1920} +Nunc ultricies leo nec libero rhoncus, eu vehicula enim efficitur.~\cite{Darwin1888} +Einstein said~\cite{Einstein1920} that lorem impsum, consectetur adipiscing elit. Integer eros tortor, dictum nec aliquet in, pharetra nec justo. +\cite{Darwin1888}. + +% Comment with \cite{Darwin1888} + +\bibliographystyle{plain} +\bibliography{origin} +\end{document} diff --git a/src/test/resources/org/jabref/logic/texparser/paper2.tex b/src/test/resources/org/jabref/logic/texparser/paper2.tex new file mode 100644 index 00000000000..4609cd6812c --- /dev/null +++ b/src/test/resources/org/jabref/logic/texparser/paper2.tex @@ -0,0 +1,10 @@ +\documentclass{article} + +\begin{document} +This is some content trying to cite a bib file: \cite{Einstein1920a} +This is some content trying to cite a bib file: \cite{Einstein1920b} +This is some content trying to cite a bib file: \cite{Einstein1920c} + +\bibliographystyle{plain} +\bibliography{origin} +\end{document} diff --git a/src/test/resources/org/jabref/logic/texparser/unknown_key.tex b/src/test/resources/org/jabref/logic/texparser/unknown_key.tex new file mode 100644 index 00000000000..68f72ed2edc --- /dev/null +++ b/src/test/resources/org/jabref/logic/texparser/unknown_key.tex @@ -0,0 +1,10 @@ +\documentclass{article} + +\begin{document} +This is some content trying to cite a bib file: \cite{Darwin1888} +This is some content trying to cite a bib file: \cite{Einstein1920} +This is some content trying to cite a bib file: \cite{UnknownKey} + +\bibliographystyle{plain} +\bibliography{origin} +\end{document} From 08d8baa87402ecdecfbe6dab27619b3e628972ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20M=C3=A9ndez?= Date: Fri, 31 May 2019 13:18:23 +0200 Subject: [PATCH 02/57] Add support for parsing nested files and cross-references --- .../logic/texparser/DefaultTexParser.java | 46 ++++++- .../model/texparser/TexParserResult.java | 20 ++- .../jabref/logic/texparser/TexParserTest.java | 130 +++++++++++++----- .../org/jabref/logic/texparser/crossref.tex | 11 ++ .../org/jabref/logic/texparser/nested.tex | 9 ++ .../org/jabref/logic/texparser/paper2.tex | 6 +- 6 files changed, 184 insertions(+), 38 deletions(-) create mode 100644 src/test/resources/org/jabref/logic/texparser/crossref.tex create mode 100644 src/test/resources/org/jabref/logic/texparser/nested.tex diff --git a/src/main/java/org/jabref/logic/texparser/DefaultTexParser.java b/src/main/java/org/jabref/logic/texparser/DefaultTexParser.java index ea366c9b7b9..005981f4b49 100644 --- a/src/main/java/org/jabref/logic/texparser/DefaultTexParser.java +++ b/src/main/java/org/jabref/logic/texparser/DefaultTexParser.java @@ -4,8 +4,9 @@ import java.io.LineNumberReader; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; -import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; @@ -15,6 +16,7 @@ import org.jabref.model.database.BibDatabase; import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.FieldName; import org.jabref.model.texparser.Citation; import org.jabref.model.texparser.TexParser; import org.jabref.model.texparser.TexParserResult; @@ -33,6 +35,10 @@ public class DefaultTexParser implements TexParser { private static final String CITE_REGEX = String.format("\\\\(?:%s)\\{([^\\}]*)\\}", String.join("|", CITE_COMMANDS)); + private static final String[] INCLUDE_COMMANDS = new String[] {"include", "includeonly", "input"}; + private static final String INCLUDE_REGEX = String.format("\\\\(?:%s)\\{([^\\}]*)\\}", + String.join("|", INCLUDE_COMMANDS)); + private final BibDatabase masterDatabase; public DefaultTexParser(BibDatabase database) { @@ -41,7 +47,7 @@ public DefaultTexParser(BibDatabase database) { @Override public TexParserResult parse(Path texFile) { - return parseTexFiles(Arrays.asList(texFile)); + return parseTexFiles(new ArrayList<>(Collections.singletonList(texFile))); } @Override @@ -64,6 +70,7 @@ private TexParserResult parseTexFiles(List texFiles) { } matchCitation(result, file, lnr.getLineNumber(), line); + matchNestedFile(result, file, texFiles, line); } } catch (IOException e) { LOGGER.warn("Error opening the TEX file", e); @@ -89,6 +96,25 @@ private void matchCitation(TexParserResult result, Path file, int lineNumber, St } } + private void matchNestedFile(TexParserResult result, Path file, List fileList, String line) { + Matcher includeMatch = Pattern.compile(INCLUDE_REGEX).matcher(line); + + while (includeMatch.find()) { + String[] includes = includeMatch.group(1).split(","); + + for (String include : includes) { + Path inputFile = (file.getParent() != null) + ? file.getParent().resolve(include) + : Paths.get(include); + + if (!fileList.contains(inputFile)) { + fileList.add(inputFile); + result.increaseNestedFilesCounter(); + } + } + } + } + /* * Add a citation to the uniqueKeys map. */ @@ -116,6 +142,7 @@ private void resolveTags(TexParserResult result) { if (entry.isPresent()) { insertEntry(result, entry.get()); + resolveCrossReferences(result, entry.get()); } else { result.getUnresolvedKeys().add(key); } @@ -129,6 +156,21 @@ private void resolveTags(TexParserResult result) { } } + private void resolveCrossReferences(TexParserResult result, BibEntry entry) { + entry.getField(FieldName.CROSSREF).ifPresent(crossRef -> { + if (!result.getGeneratedBibDatabase().getEntryByKey(crossRef).isPresent()) { + Optional refEntry = masterDatabase.getEntryByKey(crossRef); + + if (refEntry.isPresent()) { + insertEntry(result, refEntry.get()); + result.increaseCrossRefEntriesCounter(); + } else { + result.getUnresolvedKeys().add(crossRef); + } + } + }); + } + /* * Insert into the database a clone of the given entry. The cloned entry has a new unique ID. */ diff --git a/src/main/java/org/jabref/model/texparser/TexParserResult.java b/src/main/java/org/jabref/model/texparser/TexParserResult.java index 952982780e0..4d702823f3b 100644 --- a/src/main/java/org/jabref/model/texparser/TexParserResult.java +++ b/src/main/java/org/jabref/model/texparser/TexParserResult.java @@ -16,6 +16,8 @@ public class TexParserResult { private final List unresolvedKeys = new ArrayList<>(); private final BibDatabase texDatabase = new BibDatabase(); private int insertedStrings = 0; + private int nestedFilesCount = 0; + private int crossRefEntriesCount = 0; public TexParserResult(BibDatabase masterDatabase) { this.masterDatabase = masterDatabase; @@ -50,7 +52,7 @@ public BibDatabase getGeneratedBibDatabase() { } public int getResolvedKeysCount() { - return texDatabase.getEntryCount(); + return texDatabase.getEntryCount() - crossRefEntriesCount; } public int getInsertedStrings() { @@ -63,4 +65,20 @@ public void insertStrings(Collection usedStrings) { insertedStrings++; } } + + public int getNestedFilesCount() { + return nestedFilesCount; + } + + public void increaseNestedFilesCounter() { + nestedFilesCount++; + } + + public int getCrossRefEntriesCount() { + return crossRefEntriesCount; + } + + public void increaseCrossRefEntriesCounter() { + crossRefEntriesCount++; + } } diff --git a/src/test/java/org/jabref/logic/texparser/TexParserTest.java b/src/test/java/org/jabref/logic/texparser/TexParserTest.java index abc8562db16..f32d6b100aa 100644 --- a/src/test/java/org/jabref/logic/texparser/TexParserTest.java +++ b/src/test/java/org/jabref/logic/texparser/TexParserTest.java @@ -65,6 +65,10 @@ void testSingleFile() throws URISyntaxException, IOException { assertEquals(2, texResult.getFoundKeysInTex()); assertEquals(0, texResult.getUnresolvedKeysCount()); assertEquals(2, texResult.getResolvedKeysCount()); + assertEquals(0, texResult.getNestedFilesCount()); + assertEquals(0, texResult.getCrossRefEntriesCount()); + assertEquals(texResult.getFoundKeysInTex() + texResult.getCrossRefEntriesCount(), + texResult.getResolvedKeysCount() + texResult.getUnresolvedKeysCount()); assertEquals(2, newDatabase.getEntries().size()); // Check paths @@ -112,9 +116,7 @@ void testSingleFile() throws URISyntaxException, IOException { void testTwoFiles() throws URISyntaxException, IOException { final String DARWIN = "Darwin1888"; final String EINSTEIN = "Einstein1920"; - final String EINSTEIN_A = "Einstein1920a"; - final String EINSTEIN_B = "Einstein1920b"; - final String EINSTEIN_C = "Einstein1920c"; + final String NEWTON = "Newton1999"; InputStream originalStream = TexParserTest.class.getResourceAsStream("origin.bib"); Path texFile = Paths.get(TexParserTest.class.getResource("paper.tex").toURI()); @@ -131,20 +133,24 @@ void testTwoFiles() throws URISyntaxException, IOException { // Check entries assertTrue(texResult.getGeneratedBibDatabase().hasEntries()); - assertEquals(2, texResult.getCitationsCountByKey(DARWIN)); - assertEquals(2, texResult.getCitationsCountByKey(EINSTEIN)); - assertEquals(5, texResult.getFoundKeysInTex()); + assertEquals(3, texResult.getCitationsCountByKey(DARWIN)); + assertEquals(3, texResult.getCitationsCountByKey(EINSTEIN)); + assertEquals(3, texResult.getFoundKeysInTex()); assertEquals(0, texResult.getUnresolvedKeysCount()); - assertEquals(5, texResult.getResolvedKeysCount()); - assertEquals(5, newDatabase.getEntries().size()); + assertEquals(3, texResult.getResolvedKeysCount()); + assertEquals(0, texResult.getNestedFilesCount()); + assertEquals(0, texResult.getCrossRefEntriesCount()); + assertEquals(texResult.getFoundKeysInTex() + texResult.getCrossRefEntriesCount(), + texResult.getResolvedKeysCount() + texResult.getUnresolvedKeysCount()); + assertEquals(3, newDatabase.getEntries().size()); // Check paths assertEquals(texFile, texResult.getUniqueKeys().get(DARWIN).get(0).getPath()); assertEquals(texFile, texResult.getUniqueKeys().get(EINSTEIN).get(0).getPath()); - assertEquals(texFile2, texResult.getUniqueKeys().get(EINSTEIN_A).get(0).getPath()); - assertEquals(texFile2, texResult.getUniqueKeys().get(EINSTEIN_B).get(0).getPath()); - assertEquals(texFile2, texResult.getUniqueKeys().get(EINSTEIN_C).get(0).getPath()); + assertEquals(texFile2, texResult.getUniqueKeys().get(DARWIN).get(2).getPath()); + assertEquals(texFile2, texResult.getUniqueKeys().get(EINSTEIN).get(2).getPath()); + assertEquals(texFile2, texResult.getUniqueKeys().get(NEWTON).get(0).getPath()); // Check lines assertEquals(4, texResult.getUniqueKeys().get(EINSTEIN).get(0).getLine()); @@ -152,9 +158,9 @@ void testTwoFiles() throws URISyntaxException, IOException { assertEquals(6, texResult.getUniqueKeys().get(EINSTEIN).get(1).getLine()); assertEquals(7, texResult.getUniqueKeys().get(DARWIN).get(1).getLine()); - assertEquals(4, texResult.getUniqueKeys().get(EINSTEIN_A).get(0).getLine()); - assertEquals(5, texResult.getUniqueKeys().get(EINSTEIN_B).get(0).getLine()); - assertEquals(6, texResult.getUniqueKeys().get(EINSTEIN_C).get(0).getLine()); + assertEquals(4, texResult.getUniqueKeys().get(DARWIN).get(2).getLine()); + assertEquals(5, texResult.getUniqueKeys().get(EINSTEIN).get(2).getLine()); + assertEquals(6, texResult.getUniqueKeys().get(NEWTON).get(0).getLine()); // Check columns assertEquals(0, texResult.getUniqueKeys().get(EINSTEIN).get(0).getColStart()); @@ -169,14 +175,14 @@ void testTwoFiles() throws URISyntaxException, IOException { assertEquals(0, texResult.getUniqueKeys().get(DARWIN).get(1).getColStart()); assertEquals(17, texResult.getUniqueKeys().get(DARWIN).get(1).getColEnd()); - assertEquals(48, texResult.getUniqueKeys().get(EINSTEIN_A).get(0).getColStart()); - assertEquals(68, texResult.getUniqueKeys().get(EINSTEIN_A).get(0).getColEnd()); + assertEquals(48, texResult.getUniqueKeys().get(DARWIN).get(2).getColStart()); + assertEquals(65, texResult.getUniqueKeys().get(DARWIN).get(2).getColEnd()); - assertEquals(48, texResult.getUniqueKeys().get(EINSTEIN_B).get(0).getColStart()); - assertEquals(68, texResult.getUniqueKeys().get(EINSTEIN_B).get(0).getColEnd()); + assertEquals(48, texResult.getUniqueKeys().get(EINSTEIN).get(2).getColStart()); + assertEquals(67, texResult.getUniqueKeys().get(EINSTEIN).get(2).getColEnd()); - assertEquals(48, texResult.getUniqueKeys().get(EINSTEIN_C).get(0).getColStart()); - assertEquals(68, texResult.getUniqueKeys().get(EINSTEIN_C).get(0).getColEnd()); + assertEquals(48, texResult.getUniqueKeys().get(NEWTON).get(0).getColStart()); + assertEquals(65, texResult.getUniqueKeys().get(NEWTON).get(0).getColEnd()); // Check line texts assertEquals("\\cite{Einstein1920}", texResult.getUniqueKeys().get(EINSTEIN).get(0).getLineText()); @@ -186,12 +192,12 @@ void testTwoFiles() throws URISyntaxException, IOException { texResult.getUniqueKeys().get(EINSTEIN).get(1).getLineText()); assertEquals("\\cite{Darwin1888}.", texResult.getUniqueKeys().get(DARWIN).get(1).getLineText()); - assertEquals("This is some content trying to cite a bib file: \\cite{Einstein1920a}", - texResult.getUniqueKeys().get(EINSTEIN_A).get(0).getLineText()); - assertEquals("This is some content trying to cite a bib file: \\cite{Einstein1920b}", - texResult.getUniqueKeys().get(EINSTEIN_B).get(0).getLineText()); - assertEquals("This is some content trying to cite a bib file: \\cite{Einstein1920c}", - texResult.getUniqueKeys().get(EINSTEIN_C).get(0).getLineText()); + assertEquals("This is some content trying to cite a bib file: \\cite{Darwin1888}", + texResult.getUniqueKeys().get(DARWIN).get(2).getLineText()); + assertEquals("This is some content trying to cite a bib file: \\cite{Einstein1920}", + texResult.getUniqueKeys().get(EINSTEIN).get(2).getLineText()); + assertEquals("This is some content trying to cite a bib file: \\cite{Newton1999}", + texResult.getUniqueKeys().get(NEWTON).get(0).getLineText()); // Check contexts assertEquals("\\cite{Einstein1920}", texResult.getUniqueKeys().get(EINSTEIN).get(0).getContext()); @@ -201,12 +207,12 @@ void testTwoFiles() throws URISyntaxException, IOException { texResult.getUniqueKeys().get(EINSTEIN).get(1).getContext()); assertEquals("\\cite{Darwin1888}.", texResult.getUniqueKeys().get(DARWIN).get(1).getContext()); - assertEquals("nt trying to cite a bib file: \\cite{Einstein1920a}", - texResult.getUniqueKeys().get(EINSTEIN_A).get(0).getContext()); - assertEquals("nt trying to cite a bib file: \\cite{Einstein1920b}", - texResult.getUniqueKeys().get(EINSTEIN_B).get(0).getContext()); - assertEquals("nt trying to cite a bib file: \\cite{Einstein1920c}", - texResult.getUniqueKeys().get(EINSTEIN_C).get(0).getContext()); + assertEquals("ntent trying to cite a bib file: \\cite{Darwin1888}", + texResult.getUniqueKeys().get(DARWIN).get(2).getContext()); + assertEquals("ent trying to cite a bib file: \\cite{Einstein1920}", + texResult.getUniqueKeys().get(EINSTEIN).get(2).getContext()); + assertEquals("ntent trying to cite a bib file: \\cite{Newton1999}", + texResult.getUniqueKeys().get(NEWTON).get(0).getContext()); } } @@ -234,6 +240,10 @@ void testDuplicateFiles() throws URISyntaxException, IOException { assertEquals(2, texResult.getFoundKeysInTex()); assertEquals(0, texResult.getUnresolvedKeysCount()); assertEquals(2, texResult.getResolvedKeysCount()); + assertEquals(0, texResult.getNestedFilesCount()); + assertEquals(0, texResult.getCrossRefEntriesCount()); + assertEquals(texResult.getFoundKeysInTex() + texResult.getCrossRefEntriesCount(), + texResult.getResolvedKeysCount() + texResult.getUnresolvedKeysCount()); assertEquals(2, newDatabase.getEntries().size()); // Check paths @@ -293,6 +303,10 @@ void testUnknownKey() throws URISyntaxException, IOException { assertEquals(3, texResult.getFoundKeysInTex()); assertEquals(1, texResult.getUnresolvedKeysCount()); assertEquals(2, texResult.getResolvedKeysCount()); + assertEquals(0, texResult.getNestedFilesCount()); + assertEquals(0, texResult.getCrossRefEntriesCount()); + assertEquals(texResult.getFoundKeysInTex() + texResult.getCrossRefEntriesCount(), + texResult.getResolvedKeysCount() + texResult.getUnresolvedKeysCount()); assertEquals(2, newDatabase.getEntries().size()); } } @@ -307,6 +321,10 @@ void testFileNotFound() { assertEquals(0, texResult.getFoundKeysInTex()); assertEquals(0, texResult.getUnresolvedKeysCount()); assertEquals(0, texResult.getResolvedKeysCount()); + assertEquals(0, texResult.getNestedFilesCount()); + assertEquals(0, texResult.getCrossRefEntriesCount()); + assertEquals(texResult.getFoundKeysInTex() + texResult.getCrossRefEntriesCount(), + texResult.getResolvedKeysCount() + texResult.getUnresolvedKeysCount()); assertEquals(0, newDatabase.getEntries().size()); } @@ -326,4 +344,52 @@ void testDuplicateBibDatabaseConfiguration() throws URISyntaxException, IOExcept assertEquals(1, newDatabase.getStringCount()); } } + + @Test + void testNestedFiles() throws URISyntaxException, IOException { + InputStream originalStream = TexParserTest.class.getResourceAsStream("origin.bib"); + Path texFile = Paths.get(TexParserTest.class.getResource("nested.tex").toURI()); + try (InputStreamReader originalReader = new InputStreamReader(originalStream, StandardCharsets.UTF_8)) { + ParserResult result = new BibtexParser(importFormatPreferences, + new DummyFileUpdateMonitor()).parse(originalReader); + + TexParser texParser = new DefaultTexParser(result.getDatabase()); + TexParserResult texResult = texParser.parse(texFile); + BibDatabase newDatabase = texResult.getGeneratedBibDatabase(); + + assertTrue(texResult.getGeneratedBibDatabase().hasEntries()); + assertEquals(2, texResult.getFoundKeysInTex()); + assertEquals(0, texResult.getUnresolvedKeysCount()); + assertEquals(2, texResult.getResolvedKeysCount()); + assertEquals(1, texResult.getNestedFilesCount()); + assertEquals(0, texResult.getCrossRefEntriesCount()); + assertEquals(texResult.getFoundKeysInTex() + texResult.getCrossRefEntriesCount(), + texResult.getResolvedKeysCount() + texResult.getUnresolvedKeysCount()); + assertEquals(2, newDatabase.getEntries().size()); + } + } + + @Test + void testCrossRef() throws URISyntaxException, IOException { + InputStream originalStream = TexParserTest.class.getResourceAsStream("origin.bib"); + Path texFile = Paths.get(TexParserTest.class.getResource("crossref.tex").toURI()); + try (InputStreamReader originalReader = new InputStreamReader(originalStream, StandardCharsets.UTF_8)) { + ParserResult result = new BibtexParser(importFormatPreferences, + new DummyFileUpdateMonitor()).parse(originalReader); + + TexParser texParser = new DefaultTexParser(result.getDatabase()); + TexParserResult texResult = texParser.parse(texFile); + BibDatabase newDatabase = texResult.getGeneratedBibDatabase(); + + assertTrue(texResult.getGeneratedBibDatabase().hasEntries()); + assertEquals(4, texResult.getFoundKeysInTex()); + assertEquals(2, texResult.getUnresolvedKeysCount()); + assertEquals(3, texResult.getResolvedKeysCount()); + assertEquals(0, texResult.getNestedFilesCount()); + assertEquals(1, texResult.getCrossRefEntriesCount()); + assertEquals(texResult.getFoundKeysInTex() + texResult.getCrossRefEntriesCount(), + texResult.getResolvedKeysCount() + texResult.getUnresolvedKeysCount()); + assertEquals(4, newDatabase.getEntries().size()); + } + } } diff --git a/src/test/resources/org/jabref/logic/texparser/crossref.tex b/src/test/resources/org/jabref/logic/texparser/crossref.tex new file mode 100644 index 00000000000..4a2809c0d16 --- /dev/null +++ b/src/test/resources/org/jabref/logic/texparser/crossref.tex @@ -0,0 +1,11 @@ +\documentclass{article} + +\begin{document} +This is some content trying to cite a bib file: \cite{Einstein1920a} +This is some content trying to cite a bib file: \cite{Einstein1920b} +This is some content trying to cite a bib file: \cite{Einstein1920c} +This is some content trying to cite a bib file: \cite{UnresolvedKey} + +\bibliographystyle{plain} +\bibliography{origin} +\end{document} diff --git a/src/test/resources/org/jabref/logic/texparser/nested.tex b/src/test/resources/org/jabref/logic/texparser/nested.tex new file mode 100644 index 00000000000..c68348d3241 --- /dev/null +++ b/src/test/resources/org/jabref/logic/texparser/nested.tex @@ -0,0 +1,9 @@ +\documentclass{article} + +\begin{document} +This is some content trying to cite a bib file. + +\bibliographystyle{plain} +\bibliography{origin} +\include{paper.tex} +\end{document} diff --git a/src/test/resources/org/jabref/logic/texparser/paper2.tex b/src/test/resources/org/jabref/logic/texparser/paper2.tex index 4609cd6812c..1b730d244d7 100644 --- a/src/test/resources/org/jabref/logic/texparser/paper2.tex +++ b/src/test/resources/org/jabref/logic/texparser/paper2.tex @@ -1,9 +1,9 @@ \documentclass{article} \begin{document} -This is some content trying to cite a bib file: \cite{Einstein1920a} -This is some content trying to cite a bib file: \cite{Einstein1920b} -This is some content trying to cite a bib file: \cite{Einstein1920c} +This is some content trying to cite a bib file: \cite{Darwin1888} +This is some content trying to cite a bib file: \cite{Einstein1920} +This is some content trying to cite a bib file: \cite{Newton1999} \bibliographystyle{plain} \bibliography{origin} From f75e5111fda8d145cf7b6a94835dfaecfbcd14f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20M=C3=A9ndez?= Date: Tue, 4 Jun 2019 08:40:20 +0200 Subject: [PATCH 03/57] Fix several TEX parser issues --- .../logic/texparser/CrossReferences.java | 30 ++ .../logic/texparser/DefaultTexParser.java | 101 +++-- .../org/jabref/model/texparser/Citation.java | 11 +- .../org/jabref/model/texparser/TexParser.java | 16 +- .../model/texparser/TexParserResult.java | 47 ++ .../jabref/logic/texparser/TexParserTest.java | 404 +++++++----------- .../org/jabref/logic/texparser/nested.tex | 4 +- .../org/jabref/logic/texparser/nested2.tex | 9 + .../org/jabref/logic/texparser/paper.tex | 6 +- .../org/jabref/logic/texparser/paper2.tex | 2 +- .../jabref/logic/texparser/unknown_key.tex | 2 +- 11 files changed, 329 insertions(+), 303 deletions(-) create mode 100644 src/main/java/org/jabref/logic/texparser/CrossReferences.java create mode 100644 src/test/resources/org/jabref/logic/texparser/nested2.tex diff --git a/src/main/java/org/jabref/logic/texparser/CrossReferences.java b/src/main/java/org/jabref/logic/texparser/CrossReferences.java new file mode 100644 index 00000000000..443a5b00614 --- /dev/null +++ b/src/main/java/org/jabref/logic/texparser/CrossReferences.java @@ -0,0 +1,30 @@ +package org.jabref.logic.texparser; + +import java.util.Optional; + +import org.jabref.model.database.BibDatabase; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.FieldName; +import org.jabref.model.texparser.TexParserResult; + +public class CrossReferences { + + private CrossReferences() throws IllegalStateException { + throw new IllegalStateException("Utility class"); + } + + protected static void resolve(BibDatabase database, TexParserResult result, BibEntry entry) { + entry.getField(FieldName.CROSSREF).ifPresent(crossRef -> { + if (!result.getGeneratedBibDatabase().getEntryByKey(crossRef).isPresent()) { + Optional refEntry = database.getEntryByKey(crossRef); + + if (refEntry.isPresent()) { + result.getGeneratedBibDatabase().insertEntry((BibEntry) refEntry.get().clone()); + result.increaseCrossRefEntriesCounter(); + } else { + result.getUnresolvedKeys().add(crossRef); + } + } + }); + } +} diff --git a/src/main/java/org/jabref/logic/texparser/DefaultTexParser.java b/src/main/java/org/jabref/logic/texparser/DefaultTexParser.java index 005981f4b49..55e6a25d4c8 100644 --- a/src/main/java/org/jabref/logic/texparser/DefaultTexParser.java +++ b/src/main/java/org/jabref/logic/texparser/DefaultTexParser.java @@ -16,7 +16,6 @@ import org.jabref.model.database.BibDatabase; import org.jabref.model.entry.BibEntry; -import org.jabref.model.entry.FieldName; import org.jabref.model.texparser.Citation; import org.jabref.model.texparser.TexParser; import org.jabref.model.texparser.TexParserResult; @@ -30,14 +29,19 @@ public class DefaultTexParser implements TexParser { /** * It is allowed to add new cite commands for pattern matching. + * + *

Some valid examples: "citep", "[cC]ite", "[cC]ite(author|title|year|t|p)?" + * + *

TODO: Add support for multicite commands. */ - private static final String[] CITE_COMMANDS = new String[] {"cite", "citep", "citet"}; - private static final String CITE_REGEX = String.format("\\\\(?:%s)\\{([^\\}]*)\\}", - String.join("|", CITE_COMMANDS)); + private static final String CITE_REGEX = String.format("\\\\(?:%s)\\*?(?:\\[(?:[^\\]]*)\\]){0,2}\\{(?[^\\}]*)\\}", + String.join("|", new String[] { + "[cC]ite(alt|alp|author|authorfull|date|num|p|t|text|title|url|year|yearpar)?", + "([aA]|fnote|foot|footfull|full|no|[nN]ote|[pP]aren|[pP]note|[tT]ext|[sS]mart|super)cite", + "footcitetext" + })); - private static final String[] INCLUDE_COMMANDS = new String[] {"include", "includeonly", "input"}; - private static final String INCLUDE_REGEX = String.format("\\\\(?:%s)\\{([^\\}]*)\\}", - String.join("|", INCLUDE_COMMANDS)); + private static final String INCLUDE_REGEX = "\\\\(?:include|input)\\{(?[^\\}]*)\\}"; private final BibDatabase masterDatabase; @@ -45,22 +49,32 @@ public DefaultTexParser(BibDatabase database) { masterDatabase = database; } + @Override + public TexParserResult parse(String citeString) { + TexParserResult result = new TexParserResult(masterDatabase); + matchCitation(result, Paths.get("foo/bar"), 1, citeString); + resolveTags(result); + return result; + } + @Override public TexParserResult parse(Path texFile) { - return parseTexFiles(new ArrayList<>(Collections.singletonList(texFile))); + return parse(new TexParserResult(masterDatabase), Collections.singletonList(texFile)); } @Override public TexParserResult parse(List texFiles) { - return parseTexFiles(texFiles); + return parse(new TexParserResult(masterDatabase), texFiles); } - private TexParserResult parseTexFiles(List texFiles) { - TexParserResult result = new TexParserResult(masterDatabase); - int fileIndex = 0; + /** + * Parse a list of TEX files and, recursively, their referenced files. + */ + private TexParserResult parse(TexParserResult result, List texFiles) { + List referencedFiles = new ArrayList<>(); - while (fileIndex < texFiles.size()) { - Path file = texFiles.get(fileIndex++); + for (int fileIndex = 0; fileIndex < texFiles.size(); fileIndex++) { + Path file = texFiles.get(fileIndex); try (LineNumberReader lnr = new LineNumberReader(Files.newBufferedReader(file))) { for (String line = lnr.readLine(); line != null; line = lnr.readLine()) { @@ -70,25 +84,29 @@ private TexParserResult parseTexFiles(List texFiles) { } matchCitation(result, file, lnr.getLineNumber(), line); - matchNestedFile(result, file, texFiles, line); + matchNestedFile(result, file, texFiles, referencedFiles, line); } } catch (IOException e) { LOGGER.warn("Error opening the TEX file", e); } } + if (!referencedFiles.isEmpty()) { + parse(result, referencedFiles); + } + resolveTags(result); return result; } - /* + /** * Find cites along a specific line and store them. */ private void matchCitation(TexParserResult result, Path file, int lineNumber, String line) { Matcher citeMatch = Pattern.compile(CITE_REGEX).matcher(line); while (citeMatch.find()) { - String[] keys = citeMatch.group(1).split(","); + String[] keys = citeMatch.group("key").split(","); for (String key : keys) { addKey(result, key.trim(), new Citation(file, lineNumber, citeMatch.start(), citeMatch.end(), line)); @@ -96,26 +114,32 @@ private void matchCitation(TexParserResult result, Path file, int lineNumber, St } } - private void matchNestedFile(TexParserResult result, Path file, List fileList, String line) { + /** + * Find inputs and includes along a specific line and store them for parsing later. + */ + private void matchNestedFile(TexParserResult result, Path file, List texFiles, List referencedFiles, String line) { Matcher includeMatch = Pattern.compile(INCLUDE_REGEX).matcher(line); while (includeMatch.find()) { - String[] includes = includeMatch.group(1).split(","); + String include = includeMatch.group("file"); + + if (!include.endsWith(".tex")) { + include += ".tex"; + } - for (String include : includes) { - Path inputFile = (file.getParent() != null) - ? file.getParent().resolve(include) - : Paths.get(include); + Path folder = file.getParent(); + Path inputFile = (folder != null) + ? folder.resolve(include) + : Paths.get(include); - if (!fileList.contains(inputFile)) { - fileList.add(inputFile); - result.increaseNestedFilesCounter(); - } + if (!texFiles.contains(inputFile)) { + referencedFiles.add(inputFile); + result.increaseNestedFilesCounter(); } } } - /* + /** * Add a citation to the uniqueKeys map. */ private void addKey(TexParserResult result, String key, Citation citation) { @@ -130,7 +154,7 @@ private void addKey(TexParserResult result, String key, Citation citation) { } } - /* + /** * Look for an equivalent BibTeX entry within the reference database for all keys inside of the TEX files. */ private void resolveTags(TexParserResult result) { @@ -142,7 +166,7 @@ private void resolveTags(TexParserResult result) { if (entry.isPresent()) { insertEntry(result, entry.get()); - resolveCrossReferences(result, entry.get()); + CrossReferences.resolve(masterDatabase, result, entry.get()); } else { result.getUnresolvedKeys().add(key); } @@ -156,22 +180,7 @@ private void resolveTags(TexParserResult result) { } } - private void resolveCrossReferences(TexParserResult result, BibEntry entry) { - entry.getField(FieldName.CROSSREF).ifPresent(crossRef -> { - if (!result.getGeneratedBibDatabase().getEntryByKey(crossRef).isPresent()) { - Optional refEntry = masterDatabase.getEntryByKey(crossRef); - - if (refEntry.isPresent()) { - insertEntry(result, refEntry.get()); - result.increaseCrossRefEntriesCounter(); - } else { - result.getUnresolvedKeys().add(crossRef); - } - } - }); - } - - /* + /** * Insert into the database a clone of the given entry. The cloned entry has a new unique ID. */ private void insertEntry(TexParserResult result, BibEntry entry) { diff --git a/src/main/java/org/jabref/model/texparser/Citation.java b/src/main/java/org/jabref/model/texparser/Citation.java index 6a9206ddc77..9ea642e16e7 100644 --- a/src/main/java/org/jabref/model/texparser/Citation.java +++ b/src/main/java/org/jabref/model/texparser/Citation.java @@ -45,6 +45,8 @@ public String getLineText() { } /** + * Get a fixed-width string that shows the context of a citation. + * * @return String that contains a cite and the text that surrounds it along the same line. */ public String getContext() { @@ -61,7 +63,7 @@ public String getContext() { @Override public String toString() { - return String.format("%s\t%d:%d-%d\t%s", path.toString(), line, colStart, colEnd, getContext()); + return String.format("%s (%d:%d-%d) \"%s\"", path, line, colStart, colEnd, getContext()); } @Override @@ -76,11 +78,12 @@ public boolean equals(Object o) { Citation citation = (Citation) o; - return line == citation.line + return path.equals(citation.path) + && line == citation.line && colStart == citation.colStart && colEnd == citation.colEnd - && Objects.equals(path, citation.path) - && Objects.equals(lineText, citation.lineText); + && lineText.equals(citation.lineText) + && getContext().equals(citation.getContext()); } @Override diff --git a/src/main/java/org/jabref/model/texparser/TexParser.java b/src/main/java/org/jabref/model/texparser/TexParser.java index 6afa191564d..3a0eb0a9cc5 100644 --- a/src/main/java/org/jabref/model/texparser/TexParser.java +++ b/src/main/java/org/jabref/model/texparser/TexParser.java @@ -6,6 +6,16 @@ public interface TexParser { /** + * For testing purposes. + * + * @param citeString String that contains a citation + * @return a TexParserResult, where Path is /foo/bar and lineNumber is 1 + */ + TexParserResult parse(String citeString); + + /** + * Parse a single TEX file. + * * @param texFile Path to a TEX file * @return a TexParserResult, which contains the generated BibDatabase and all data related to the bibliographic * entries @@ -13,9 +23,11 @@ public interface TexParser { TexParserResult parse(Path texFile); /** + * Parse a list of TEX files. + * * @param texFiles List of Path objects linked to a TEX file - * @return a list of TexParserResult objects, which contains the generated BibDatabase and all data related to the - * bibliographic entries + * @return a TexParserResult, which contains the generated BibDatabase and all data related to the bibliographic + * entries */ TexParserResult parse(List texFiles); } diff --git a/src/main/java/org/jabref/model/texparser/TexParserResult.java b/src/main/java/org/jabref/model/texparser/TexParserResult.java index 4d702823f3b..61e83c80330 100644 --- a/src/main/java/org/jabref/model/texparser/TexParserResult.java +++ b/src/main/java/org/jabref/model/texparser/TexParserResult.java @@ -3,8 +3,10 @@ import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Objects; import org.jabref.model.database.BibDatabase; import org.jabref.model.entry.BibtexString; @@ -19,6 +21,13 @@ public class TexParserResult { private int nestedFilesCount = 0; private int crossRefEntriesCount = 0; + public TexParserResult(BibDatabase masterDatabase, int insertedStrings, int nestedFilesCount, int crossRefEntriesCount) { + this.masterDatabase = masterDatabase; + this.insertedStrings = insertedStrings; + this.nestedFilesCount = nestedFilesCount; + this.crossRefEntriesCount = crossRefEntriesCount; + } + public TexParserResult(BibDatabase masterDatabase) { this.masterDatabase = masterDatabase; } @@ -81,4 +90,42 @@ public int getCrossRefEntriesCount() { public void increaseCrossRefEntriesCounter() { crossRefEntriesCount++; } + + @Override + public String toString() { + return String.format("%nTexParserResult{%n masterDatabase=%s,%n uniqueKeys=%s,%n unresolvedKeys=%s,%n texDatabase=%s,%n insertedStrings=%s,%n nestedFilesCount=%s,%n crossRefEntriesCount=%s%n}%n", + masterDatabase, + uniqueKeys, + unresolvedKeys, + texDatabase, + insertedStrings, + nestedFilesCount, + crossRefEntriesCount); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + + if (o == null || getClass() != o.getClass()) { + return false; + } + + TexParserResult result = (TexParserResult) o; + + return masterDatabase.equals(result.masterDatabase) + && uniqueKeys.equals(result.uniqueKeys) + && unresolvedKeys.equals(result.unresolvedKeys) + && (new HashSet<>(texDatabase.getEntries())).equals(new HashSet<>(result.texDatabase.getEntries())) + && insertedStrings == result.insertedStrings + && nestedFilesCount == result.nestedFilesCount + && crossRefEntriesCount == result.crossRefEntriesCount; + } + + @Override + public int hashCode() { + return Objects.hash(masterDatabase, uniqueKeys, unresolvedKeys, texDatabase, insertedStrings, nestedFilesCount, crossRefEntriesCount); + } } diff --git a/src/test/java/org/jabref/logic/texparser/TexParserTest.java b/src/test/java/org/jabref/logic/texparser/TexParserTest.java index f32d6b100aa..8bf4f5e4196 100644 --- a/src/test/java/org/jabref/logic/texparser/TexParserTest.java +++ b/src/test/java/org/jabref/logic/texparser/TexParserTest.java @@ -7,6 +7,7 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.Arrays; import java.util.Optional; @@ -14,6 +15,8 @@ import org.jabref.logic.importer.ParserResult; import org.jabref.logic.importer.fileformat.BibtexParser; import org.jabref.model.database.BibDatabase; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.texparser.Citation; import org.jabref.model.texparser.TexParser; import org.jabref.model.texparser.TexParserResult; import org.jabref.model.util.DummyFileUpdateMonitor; @@ -31,6 +34,10 @@ class TexParserTest { private ImportFormatPreferences importFormatPreferences; + private final String DARWIN = "Darwin1888"; + private final String EINSTEIN = "Einstein1920"; + private final String UNRESOLVED = "UnresolvedKey"; + @BeforeEach void setUp() { importFormatPreferences = mock(ImportFormatPreferences.class, Answers.RETURNS_DEEP_STUBS); @@ -41,301 +48,189 @@ void tearDown() { importFormatPreferences = null; } + private void addCite(TexParserResult expectedResult, String key, Path texFile, int line, int colStart, int colEnd, String lineText, int resolved) { + if (!expectedResult.getUniqueKeys().containsKey(key)) { + expectedResult.getUniqueKeys().put(key, new ArrayList<>()); + } else { + resolved = 0; + } + + Citation citation = new Citation(texFile, line, colStart, colEnd, lineText); + if (!expectedResult.getUniqueKeys().get(key).contains(citation)) { + expectedResult.getUniqueKeys().get(key).add(citation); + } + + if (resolved > 0) { + BibEntry clonedEntry = (BibEntry) expectedResult.getMasterDatabase().getEntryByKey(key).get().clone(); + expectedResult.getGeneratedBibDatabase().insertEntry(clonedEntry); + } else if (resolved < 0) { + expectedResult.getUnresolvedKeys().add(key); + } + } + + private void testCite(String citeString) throws IOException { + InputStream originalStream = TexParserTest.class.getResourceAsStream("origin.bib"); + + try (InputStreamReader originalReader = new InputStreamReader(originalStream, StandardCharsets.UTF_8)) { + ParserResult result = new BibtexParser(importFormatPreferences, new DummyFileUpdateMonitor()).parse(originalReader); + TexParser texParser = new DefaultTexParser(result.getDatabase()); + TexParserResult texResult = texParser.parse(citeString); + TexParserResult expectedResult = new TexParserResult(result.getDatabase()); + BibDatabase newDatabase = texResult.getGeneratedBibDatabase(); + + addCite(expectedResult, UNRESOLVED, Paths.get("foo/bar"), 1, 0, citeString.length(), citeString, -1); + + assertEquals(result.getDatabase(), texResult.getMasterDatabase()); + assertFalse(texResult.getGeneratedBibDatabase().hasEntries()); + assertEquals(texResult.getFoundKeysInTex() + texResult.getCrossRefEntriesCount(), texResult.getResolvedKeysCount() + texResult.getUnresolvedKeysCount()); + assertEquals(expectedResult.getResolvedKeysCount() + expectedResult.getCrossRefEntriesCount(), newDatabase.getEntries().size()); + assertEquals(expectedResult, texResult); + } + } + @Test - void testSingleFile() throws URISyntaxException, IOException { - final String DARWIN = "Darwin1888"; - final String EINSTEIN = "Einstein1920"; + void testCiteCommands() throws IOException { + testCite("\\cite[pre][post]{UnresolvedKey}"); + testCite("\\cite*{UnresolvedKey}"); + testCite("\\parencite[post]{UnresolvedKey}"); + testCite("\\cite[pre][post]{UnresolvedKey}"); + testCite("\\citep{UnresolvedKey}"); + testCite("\\citet{UnresolvedKey}"); + } + @Test + void testSingleFile() throws URISyntaxException, IOException { InputStream originalStream = TexParserTest.class.getResourceAsStream("origin.bib"); Path texFile = Paths.get(TexParserTest.class.getResource("paper.tex").toURI()); try (InputStreamReader originalReader = new InputStreamReader(originalStream, StandardCharsets.UTF_8)) { - ParserResult result = new BibtexParser(importFormatPreferences, - new DummyFileUpdateMonitor()).parse(originalReader); + ParserResult result = new BibtexParser(importFormatPreferences, new DummyFileUpdateMonitor()).parse(originalReader); TexParser texParser = new DefaultTexParser(result.getDatabase()); TexParserResult texResult = texParser.parse(texFile); + TexParserResult expectedResult = new TexParserResult(result.getDatabase()); BibDatabase newDatabase = texResult.getGeneratedBibDatabase(); - assertEquals(result.getDatabase(), texResult.getMasterDatabase()); + addCite(expectedResult, EINSTEIN, texFile, 4, 0, 19, "\\cite{Einstein1920}", 1); + addCite(expectedResult, EINSTEIN, texFile, 6, 14, 33, "Einstein said \\cite{Einstein1920} that lorem impsum, consectetur adipiscing elit.", 1); - // Check entries + addCite(expectedResult, DARWIN, texFile, 5, 0, 17, "\\cite{Darwin1888}.", 1); + addCite(expectedResult, DARWIN, texFile, 7, 67, 84, "Nunc ultricies leo nec libero rhoncus, eu vehicula enim efficitur. \\cite{Darwin1888}", 1); + + assertEquals(result.getDatabase(), texResult.getMasterDatabase()); assertTrue(texResult.getGeneratedBibDatabase().hasEntries()); - assertEquals(2, texResult.getCitationsCountByKey(DARWIN)); - assertEquals(2, texResult.getCitationsCountByKey(EINSTEIN)); - assertEquals(2, texResult.getFoundKeysInTex()); - assertEquals(0, texResult.getUnresolvedKeysCount()); - assertEquals(2, texResult.getResolvedKeysCount()); - assertEquals(0, texResult.getNestedFilesCount()); - assertEquals(0, texResult.getCrossRefEntriesCount()); - assertEquals(texResult.getFoundKeysInTex() + texResult.getCrossRefEntriesCount(), - texResult.getResolvedKeysCount() + texResult.getUnresolvedKeysCount()); - assertEquals(2, newDatabase.getEntries().size()); - - // Check paths - assertEquals(texFile, texResult.getUniqueKeys().get(DARWIN).get(0).getPath()); - assertEquals(texFile, texResult.getUniqueKeys().get(EINSTEIN).get(0).getPath()); - - // Check lines - assertEquals(4, texResult.getUniqueKeys().get(EINSTEIN).get(0).getLine()); - assertEquals(5, texResult.getUniqueKeys().get(DARWIN).get(0).getLine()); - assertEquals(6, texResult.getUniqueKeys().get(EINSTEIN).get(1).getLine()); - assertEquals(7, texResult.getUniqueKeys().get(DARWIN).get(1).getLine()); - - // Check columns - assertEquals(0, texResult.getUniqueKeys().get(EINSTEIN).get(0).getColStart()); - assertEquals(19, texResult.getUniqueKeys().get(EINSTEIN).get(0).getColEnd()); - - assertEquals(67, texResult.getUniqueKeys().get(DARWIN).get(0).getColStart()); - assertEquals(84, texResult.getUniqueKeys().get(DARWIN).get(0).getColEnd()); - - assertEquals(14, texResult.getUniqueKeys().get(EINSTEIN).get(1).getColStart()); - assertEquals(33, texResult.getUniqueKeys().get(EINSTEIN).get(1).getColEnd()); - - assertEquals(0, texResult.getUniqueKeys().get(DARWIN).get(1).getColStart()); - assertEquals(17, texResult.getUniqueKeys().get(DARWIN).get(1).getColEnd()); - - // Check line texts - assertEquals("\\cite{Einstein1920}", texResult.getUniqueKeys().get(EINSTEIN).get(0).getLineText()); - assertEquals("Nunc ultricies leo nec libero rhoncus, eu vehicula enim efficitur.~\\cite{Darwin1888}", - texResult.getUniqueKeys().get(DARWIN).get(0).getLineText()); - assertEquals("Einstein said~\\cite{Einstein1920} that lorem impsum, consectetur adipiscing elit. Integer eros tortor, dictum nec aliquet in, pharetra nec justo.", - texResult.getUniqueKeys().get(EINSTEIN).get(1).getLineText()); - assertEquals("\\cite{Darwin1888}.", texResult.getUniqueKeys().get(DARWIN).get(1).getLineText()); - - // Check contexts - assertEquals("\\cite{Einstein1920}", texResult.getUniqueKeys().get(EINSTEIN).get(0).getContext()); - assertEquals("cus, eu vehicula enim efficitur.~\\cite{Darwin1888}", - texResult.getUniqueKeys().get(DARWIN).get(0).getContext()); - assertEquals("Einstein said~\\cite{Einstein1920} that lorem impsu", - texResult.getUniqueKeys().get(EINSTEIN).get(1).getContext()); - assertEquals("\\cite{Darwin1888}.", texResult.getUniqueKeys().get(DARWIN).get(1).getContext()); + assertEquals(texResult.getFoundKeysInTex() + texResult.getCrossRefEntriesCount(), texResult.getResolvedKeysCount() + texResult.getUnresolvedKeysCount()); + assertEquals(expectedResult.getResolvedKeysCount() + expectedResult.getCrossRefEntriesCount(), newDatabase.getEntries().size()); + assertEquals(expectedResult, texResult); } } @Test void testTwoFiles() throws URISyntaxException, IOException { - final String DARWIN = "Darwin1888"; - final String EINSTEIN = "Einstein1920"; - final String NEWTON = "Newton1999"; + String NEWTON = "Newton1999"; InputStream originalStream = TexParserTest.class.getResourceAsStream("origin.bib"); Path texFile = Paths.get(TexParserTest.class.getResource("paper.tex").toURI()); Path texFile2 = Paths.get(TexParserTest.class.getResource("paper2.tex").toURI()); try (InputStreamReader originalReader = new InputStreamReader(originalStream, StandardCharsets.UTF_8)) { - ParserResult result = new BibtexParser(importFormatPreferences, - new DummyFileUpdateMonitor()).parse(originalReader); + ParserResult result = new BibtexParser(importFormatPreferences, new DummyFileUpdateMonitor()).parse(originalReader); TexParser texParser = new DefaultTexParser(result.getDatabase()); TexParserResult texResult = texParser.parse(Arrays.asList(texFile, texFile2)); + TexParserResult expectedResult = new TexParserResult(result.getDatabase()); BibDatabase newDatabase = texResult.getGeneratedBibDatabase(); - assertEquals(result.getDatabase(), texResult.getMasterDatabase()); + addCite(expectedResult, EINSTEIN, texFile, 4, 0, 19, "\\cite{Einstein1920}", 1); + addCite(expectedResult, EINSTEIN, texFile, 6, 14, 33, "Einstein said \\cite{Einstein1920} that lorem impsum, consectetur adipiscing elit.", 1); + addCite(expectedResult, EINSTEIN, texFile2, 5, 48, 67, "This is some content trying to cite a bib file: \\cite{Einstein1920}", 1); - // Check entries + addCite(expectedResult, DARWIN, texFile, 5, 0, 17, "\\cite{Darwin1888}.", 1); + addCite(expectedResult, DARWIN, texFile, 7, 67, 84, "Nunc ultricies leo nec libero rhoncus, eu vehicula enim efficitur. \\cite{Darwin1888}", 1); + addCite(expectedResult, DARWIN, texFile2, 4, 48, 65, "This is some content trying to cite a bib file: \\cite{Darwin1888}", 1); + + addCite(expectedResult, NEWTON, texFile2, 6, 48, 65, "This is some content trying to cite a bib file: \\cite{Newton1999}", 1); + + assertEquals(result.getDatabase(), texResult.getMasterDatabase()); assertTrue(texResult.getGeneratedBibDatabase().hasEntries()); - assertEquals(3, texResult.getCitationsCountByKey(DARWIN)); - assertEquals(3, texResult.getCitationsCountByKey(EINSTEIN)); - assertEquals(3, texResult.getFoundKeysInTex()); - assertEquals(0, texResult.getUnresolvedKeysCount()); - assertEquals(3, texResult.getResolvedKeysCount()); - assertEquals(0, texResult.getNestedFilesCount()); - assertEquals(0, texResult.getCrossRefEntriesCount()); - assertEquals(texResult.getFoundKeysInTex() + texResult.getCrossRefEntriesCount(), - texResult.getResolvedKeysCount() + texResult.getUnresolvedKeysCount()); - assertEquals(3, newDatabase.getEntries().size()); - - // Check paths - assertEquals(texFile, texResult.getUniqueKeys().get(DARWIN).get(0).getPath()); - assertEquals(texFile, texResult.getUniqueKeys().get(EINSTEIN).get(0).getPath()); - - assertEquals(texFile2, texResult.getUniqueKeys().get(DARWIN).get(2).getPath()); - assertEquals(texFile2, texResult.getUniqueKeys().get(EINSTEIN).get(2).getPath()); - assertEquals(texFile2, texResult.getUniqueKeys().get(NEWTON).get(0).getPath()); - - // Check lines - assertEquals(4, texResult.getUniqueKeys().get(EINSTEIN).get(0).getLine()); - assertEquals(5, texResult.getUniqueKeys().get(DARWIN).get(0).getLine()); - assertEquals(6, texResult.getUniqueKeys().get(EINSTEIN).get(1).getLine()); - assertEquals(7, texResult.getUniqueKeys().get(DARWIN).get(1).getLine()); - - assertEquals(4, texResult.getUniqueKeys().get(DARWIN).get(2).getLine()); - assertEquals(5, texResult.getUniqueKeys().get(EINSTEIN).get(2).getLine()); - assertEquals(6, texResult.getUniqueKeys().get(NEWTON).get(0).getLine()); - - // Check columns - assertEquals(0, texResult.getUniqueKeys().get(EINSTEIN).get(0).getColStart()); - assertEquals(19, texResult.getUniqueKeys().get(EINSTEIN).get(0).getColEnd()); - - assertEquals(67, texResult.getUniqueKeys().get(DARWIN).get(0).getColStart()); - assertEquals(84, texResult.getUniqueKeys().get(DARWIN).get(0).getColEnd()); - - assertEquals(14, texResult.getUniqueKeys().get(EINSTEIN).get(1).getColStart()); - assertEquals(33, texResult.getUniqueKeys().get(EINSTEIN).get(1).getColEnd()); - - assertEquals(0, texResult.getUniqueKeys().get(DARWIN).get(1).getColStart()); - assertEquals(17, texResult.getUniqueKeys().get(DARWIN).get(1).getColEnd()); - - assertEquals(48, texResult.getUniqueKeys().get(DARWIN).get(2).getColStart()); - assertEquals(65, texResult.getUniqueKeys().get(DARWIN).get(2).getColEnd()); - - assertEquals(48, texResult.getUniqueKeys().get(EINSTEIN).get(2).getColStart()); - assertEquals(67, texResult.getUniqueKeys().get(EINSTEIN).get(2).getColEnd()); - - assertEquals(48, texResult.getUniqueKeys().get(NEWTON).get(0).getColStart()); - assertEquals(65, texResult.getUniqueKeys().get(NEWTON).get(0).getColEnd()); - - // Check line texts - assertEquals("\\cite{Einstein1920}", texResult.getUniqueKeys().get(EINSTEIN).get(0).getLineText()); - assertEquals("Nunc ultricies leo nec libero rhoncus, eu vehicula enim efficitur.~\\cite{Darwin1888}", - texResult.getUniqueKeys().get(DARWIN).get(0).getLineText()); - assertEquals("Einstein said~\\cite{Einstein1920} that lorem impsum, consectetur adipiscing elit. Integer eros tortor, dictum nec aliquet in, pharetra nec justo.", - texResult.getUniqueKeys().get(EINSTEIN).get(1).getLineText()); - assertEquals("\\cite{Darwin1888}.", texResult.getUniqueKeys().get(DARWIN).get(1).getLineText()); - - assertEquals("This is some content trying to cite a bib file: \\cite{Darwin1888}", - texResult.getUniqueKeys().get(DARWIN).get(2).getLineText()); - assertEquals("This is some content trying to cite a bib file: \\cite{Einstein1920}", - texResult.getUniqueKeys().get(EINSTEIN).get(2).getLineText()); - assertEquals("This is some content trying to cite a bib file: \\cite{Newton1999}", - texResult.getUniqueKeys().get(NEWTON).get(0).getLineText()); - - // Check contexts - assertEquals("\\cite{Einstein1920}", texResult.getUniqueKeys().get(EINSTEIN).get(0).getContext()); - assertEquals("cus, eu vehicula enim efficitur.~\\cite{Darwin1888}", - texResult.getUniqueKeys().get(DARWIN).get(0).getContext()); - assertEquals("Einstein said~\\cite{Einstein1920} that lorem impsu", - texResult.getUniqueKeys().get(EINSTEIN).get(1).getContext()); - assertEquals("\\cite{Darwin1888}.", texResult.getUniqueKeys().get(DARWIN).get(1).getContext()); - - assertEquals("ntent trying to cite a bib file: \\cite{Darwin1888}", - texResult.getUniqueKeys().get(DARWIN).get(2).getContext()); - assertEquals("ent trying to cite a bib file: \\cite{Einstein1920}", - texResult.getUniqueKeys().get(EINSTEIN).get(2).getContext()); - assertEquals("ntent trying to cite a bib file: \\cite{Newton1999}", - texResult.getUniqueKeys().get(NEWTON).get(0).getContext()); + assertEquals(texResult.getFoundKeysInTex() + texResult.getCrossRefEntriesCount(), texResult.getResolvedKeysCount() + texResult.getUnresolvedKeysCount()); + assertEquals(expectedResult.getResolvedKeysCount() + expectedResult.getCrossRefEntriesCount(), newDatabase.getEntries().size()); + assertEquals(expectedResult, texResult); } } @Test void testDuplicateFiles() throws URISyntaxException, IOException { - final String DARWIN = "Darwin1888"; - final String EINSTEIN = "Einstein1920"; - InputStream originalStream = TexParserTest.class.getResourceAsStream("origin.bib"); Path texFile = Paths.get(TexParserTest.class.getResource("paper.tex").toURI()); try (InputStreamReader originalReader = new InputStreamReader(originalStream, StandardCharsets.UTF_8)) { - ParserResult result = new BibtexParser(importFormatPreferences, - new DummyFileUpdateMonitor()).parse(originalReader); + ParserResult result = new BibtexParser(importFormatPreferences, new DummyFileUpdateMonitor()).parse(originalReader); TexParser texParser = new DefaultTexParser(result.getDatabase()); TexParserResult texResult = texParser.parse(Arrays.asList(texFile, texFile)); + TexParserResult expectedResult = new TexParserResult(result.getDatabase()); BibDatabase newDatabase = texResult.getGeneratedBibDatabase(); - assertEquals(result.getDatabase(), texResult.getMasterDatabase()); + addCite(expectedResult, EINSTEIN, texFile, 4, 0, 19, "\\cite{Einstein1920}", 1); + addCite(expectedResult, EINSTEIN, texFile, 6, 14, 33, "Einstein said \\cite{Einstein1920} that lorem impsum, consectetur adipiscing elit.", 1); + + addCite(expectedResult, DARWIN, texFile, 5, 0, 17, "\\cite{Darwin1888}.", 1); + addCite(expectedResult, DARWIN, texFile, 7, 67, 84, "Nunc ultricies leo nec libero rhoncus, eu vehicula enim efficitur. \\cite{Darwin1888}", 1); - // Check entries + assertEquals(result.getDatabase(), texResult.getMasterDatabase()); assertTrue(texResult.getGeneratedBibDatabase().hasEntries()); - assertEquals(2, texResult.getCitationsCountByKey(DARWIN)); - assertEquals(2, texResult.getCitationsCountByKey(EINSTEIN)); - assertEquals(2, texResult.getFoundKeysInTex()); - assertEquals(0, texResult.getUnresolvedKeysCount()); - assertEquals(2, texResult.getResolvedKeysCount()); - assertEquals(0, texResult.getNestedFilesCount()); - assertEquals(0, texResult.getCrossRefEntriesCount()); - assertEquals(texResult.getFoundKeysInTex() + texResult.getCrossRefEntriesCount(), - texResult.getResolvedKeysCount() + texResult.getUnresolvedKeysCount()); - assertEquals(2, newDatabase.getEntries().size()); - - // Check paths - assertEquals(texFile, texResult.getUniqueKeys().get(DARWIN).get(0).getPath()); - assertEquals(texFile, texResult.getUniqueKeys().get(EINSTEIN).get(0).getPath()); - - // Check lines - assertEquals(4, texResult.getUniqueKeys().get(EINSTEIN).get(0).getLine()); - assertEquals(5, texResult.getUniqueKeys().get(DARWIN).get(0).getLine()); - assertEquals(6, texResult.getUniqueKeys().get(EINSTEIN).get(1).getLine()); - assertEquals(7, texResult.getUniqueKeys().get(DARWIN).get(1).getLine()); - - // Check columns - assertEquals(0, texResult.getUniqueKeys().get(EINSTEIN).get(0).getColStart()); - assertEquals(19, texResult.getUniqueKeys().get(EINSTEIN).get(0).getColEnd()); - - assertEquals(67, texResult.getUniqueKeys().get(DARWIN).get(0).getColStart()); - assertEquals(84, texResult.getUniqueKeys().get(DARWIN).get(0).getColEnd()); - - assertEquals(14, texResult.getUniqueKeys().get(EINSTEIN).get(1).getColStart()); - assertEquals(33, texResult.getUniqueKeys().get(EINSTEIN).get(1).getColEnd()); - - assertEquals(0, texResult.getUniqueKeys().get(DARWIN).get(1).getColStart()); - assertEquals(17, texResult.getUniqueKeys().get(DARWIN).get(1).getColEnd()); - - // Check line texts - assertEquals("\\cite{Einstein1920}", texResult.getUniqueKeys().get(EINSTEIN).get(0).getLineText()); - assertEquals("Nunc ultricies leo nec libero rhoncus, eu vehicula enim efficitur.~\\cite{Darwin1888}", - texResult.getUniqueKeys().get(DARWIN).get(0).getLineText()); - assertEquals("Einstein said~\\cite{Einstein1920} that lorem impsum, consectetur adipiscing elit. Integer eros tortor, dictum nec aliquet in, pharetra nec justo.", - texResult.getUniqueKeys().get(EINSTEIN).get(1).getLineText()); - assertEquals("\\cite{Darwin1888}.", texResult.getUniqueKeys().get(DARWIN).get(1).getLineText()); - - // Check contexts - assertEquals("\\cite{Einstein1920}", texResult.getUniqueKeys().get(EINSTEIN).get(0).getContext()); - assertEquals("cus, eu vehicula enim efficitur.~\\cite{Darwin1888}", - texResult.getUniqueKeys().get(DARWIN).get(0).getContext()); - assertEquals("Einstein said~\\cite{Einstein1920} that lorem impsu", - texResult.getUniqueKeys().get(EINSTEIN).get(1).getContext()); - assertEquals("\\cite{Darwin1888}.", texResult.getUniqueKeys().get(DARWIN).get(1).getContext()); + assertEquals(texResult.getFoundKeysInTex() + texResult.getCrossRefEntriesCount(), texResult.getResolvedKeysCount() + texResult.getUnresolvedKeysCount()); + assertEquals(expectedResult.getResolvedKeysCount() + expectedResult.getCrossRefEntriesCount(), newDatabase.getEntries().size()); + assertEquals(expectedResult, texResult); } } @Test void testUnknownKey() throws URISyntaxException, IOException { + String UNKNOWN = "UnknownKey"; + InputStream originalStream = TexParserTest.class.getResourceAsStream("origin.bib"); Path texFile = Paths.get(TexParserTest.class.getResource("unknown_key.tex").toURI()); - try (InputStreamReader originalReader = new InputStreamReader(originalStream, StandardCharsets.UTF_8)) { - ParserResult result = new BibtexParser(importFormatPreferences, - new DummyFileUpdateMonitor()).parse(originalReader); + try (InputStreamReader originalReader = new InputStreamReader(originalStream, StandardCharsets.UTF_8)) { + ParserResult result = new BibtexParser(importFormatPreferences, new DummyFileUpdateMonitor()).parse(originalReader); TexParser texParser = new DefaultTexParser(result.getDatabase()); TexParserResult texResult = texParser.parse(texFile); + TexParserResult expectedResult = new TexParserResult(result.getDatabase()); BibDatabase newDatabase = texResult.getGeneratedBibDatabase(); + addCite(expectedResult, EINSTEIN, texFile, 5, 48, 67, "This is some content trying to cite a bib file: \\cite{Einstein1920}", 1); + addCite(expectedResult, DARWIN, texFile, 4, 48, 65, "This is some content trying to cite a bib file: \\cite{Darwin1888}", 1); + addCite(expectedResult, UNKNOWN, texFile, 6, 48, 65, "This is some content trying to cite a bib file: \\cite{UnknownKey}", -1); + + assertEquals(result.getDatabase(), texResult.getMasterDatabase()); assertTrue(texResult.getGeneratedBibDatabase().hasEntries()); - assertEquals(3, texResult.getFoundKeysInTex()); - assertEquals(1, texResult.getUnresolvedKeysCount()); - assertEquals(2, texResult.getResolvedKeysCount()); - assertEquals(0, texResult.getNestedFilesCount()); - assertEquals(0, texResult.getCrossRefEntriesCount()); - assertEquals(texResult.getFoundKeysInTex() + texResult.getCrossRefEntriesCount(), - texResult.getResolvedKeysCount() + texResult.getUnresolvedKeysCount()); - assertEquals(2, newDatabase.getEntries().size()); + assertEquals(texResult.getFoundKeysInTex() + texResult.getCrossRefEntriesCount(), texResult.getResolvedKeysCount() + texResult.getUnresolvedKeysCount()); + assertEquals(expectedResult.getResolvedKeysCount() + expectedResult.getCrossRefEntriesCount(), newDatabase.getEntries().size()); + assertEquals(expectedResult, texResult); } } @Test void testFileNotFound() { - TexParser texParser = new DefaultTexParser(new BibDatabase()); + BibDatabase masterDatabase = new BibDatabase(); + TexParser texParser = new DefaultTexParser(masterDatabase); TexParserResult texResult = texParser.parse(Paths.get("file_not_found.tex")); + TexParserResult expectedResult = new TexParserResult(masterDatabase); BibDatabase newDatabase = texResult.getGeneratedBibDatabase(); + assertEquals(masterDatabase, texResult.getMasterDatabase()); assertFalse(texResult.getGeneratedBibDatabase().hasEntries()); - assertEquals(0, texResult.getFoundKeysInTex()); - assertEquals(0, texResult.getUnresolvedKeysCount()); - assertEquals(0, texResult.getResolvedKeysCount()); - assertEquals(0, texResult.getNestedFilesCount()); - assertEquals(0, texResult.getCrossRefEntriesCount()); - assertEquals(texResult.getFoundKeysInTex() + texResult.getCrossRefEntriesCount(), - texResult.getResolvedKeysCount() + texResult.getUnresolvedKeysCount()); - assertEquals(0, newDatabase.getEntries().size()); + assertEquals(texResult.getFoundKeysInTex() + texResult.getCrossRefEntriesCount(), texResult.getResolvedKeysCount() + texResult.getUnresolvedKeysCount()); + assertEquals(expectedResult.getResolvedKeysCount() + expectedResult.getCrossRefEntriesCount(), newDatabase.getEntries().size()); + assertEquals(expectedResult, texResult); } @Test void testDuplicateBibDatabaseConfiguration() throws URISyntaxException, IOException { InputStream originalStream = TexParserTest.class.getResourceAsStream("config.bib"); Path texFile = Paths.get(TexParserTest.class.getResource("paper.tex").toURI()); - try (InputStreamReader originalReader = new InputStreamReader(originalStream, StandardCharsets.UTF_8)) { - ParserResult result = new BibtexParser(importFormatPreferences, - new DummyFileUpdateMonitor()).parse(originalReader); + try (InputStreamReader originalReader = new InputStreamReader(originalStream, StandardCharsets.UTF_8)) { + ParserResult result = new BibtexParser(importFormatPreferences, new DummyFileUpdateMonitor()).parse(originalReader); TexParser texParser = new DefaultTexParser(result.getDatabase()); TexParserResult texResult = texParser.parse(texFile); BibDatabase newDatabase = texResult.getGeneratedBibDatabase(); @@ -349,47 +244,68 @@ void testDuplicateBibDatabaseConfiguration() throws URISyntaxException, IOExcept void testNestedFiles() throws URISyntaxException, IOException { InputStream originalStream = TexParserTest.class.getResourceAsStream("origin.bib"); Path texFile = Paths.get(TexParserTest.class.getResource("nested.tex").toURI()); - try (InputStreamReader originalReader = new InputStreamReader(originalStream, StandardCharsets.UTF_8)) { - ParserResult result = new BibtexParser(importFormatPreferences, - new DummyFileUpdateMonitor()).parse(originalReader); + try (InputStreamReader originalReader = new InputStreamReader(originalStream, StandardCharsets.UTF_8)) { + ParserResult result = new BibtexParser(importFormatPreferences, new DummyFileUpdateMonitor()).parse(originalReader); TexParser texParser = new DefaultTexParser(result.getDatabase()); TexParserResult texResult = texParser.parse(texFile); + TexParserResult expectedResult = new TexParserResult(result.getDatabase(), 0, 2, 0); BibDatabase newDatabase = texResult.getGeneratedBibDatabase(); + addCite(expectedResult, EINSTEIN, texFile.getParent().resolve("paper.tex"), 4, 0, 19, "\\cite{Einstein1920}", 1); + addCite(expectedResult, EINSTEIN, texFile.getParent().resolve("paper.tex"), 6, 14, 33, "Einstein said \\cite{Einstein1920} that lorem impsum, consectetur adipiscing elit.", 1); + + addCite(expectedResult, DARWIN, texFile.getParent().resolve("paper.tex"), 5, 0, 17, "\\cite{Darwin1888}.", 1); + addCite(expectedResult, DARWIN, texFile.getParent().resolve("paper.tex"), 7, 67, 84, "Nunc ultricies leo nec libero rhoncus, eu vehicula enim efficitur. \\cite{Darwin1888}", 1); + + assertEquals(result.getDatabase(), texResult.getMasterDatabase()); assertTrue(texResult.getGeneratedBibDatabase().hasEntries()); - assertEquals(2, texResult.getFoundKeysInTex()); - assertEquals(0, texResult.getUnresolvedKeysCount()); - assertEquals(2, texResult.getResolvedKeysCount()); - assertEquals(1, texResult.getNestedFilesCount()); - assertEquals(0, texResult.getCrossRefEntriesCount()); - assertEquals(texResult.getFoundKeysInTex() + texResult.getCrossRefEntriesCount(), - texResult.getResolvedKeysCount() + texResult.getUnresolvedKeysCount()); - assertEquals(2, newDatabase.getEntries().size()); + assertEquals(texResult.getFoundKeysInTex() + texResult.getCrossRefEntriesCount(), texResult.getResolvedKeysCount() + texResult.getUnresolvedKeysCount()); + assertEquals(expectedResult.getResolvedKeysCount() + expectedResult.getCrossRefEntriesCount(), newDatabase.getEntries().size()); + assertEquals(expectedResult, texResult); } } @Test void testCrossRef() throws URISyntaxException, IOException { + String EINSTEIN_A = "Einstein1920a"; + String EINSTEIN_B = "Einstein1920b"; + String EINSTEIN21 = "Einstein1921"; + String EINSTEIN_C = "Einstein1920c"; + InputStream originalStream = TexParserTest.class.getResourceAsStream("origin.bib"); Path texFile = Paths.get(TexParserTest.class.getResource("crossref.tex").toURI()); - try (InputStreamReader originalReader = new InputStreamReader(originalStream, StandardCharsets.UTF_8)) { - ParserResult result = new BibtexParser(importFormatPreferences, - new DummyFileUpdateMonitor()).parse(originalReader); + try (InputStreamReader originalReader = new InputStreamReader(originalStream, StandardCharsets.UTF_8)) { + ParserResult result = new BibtexParser(importFormatPreferences, new DummyFileUpdateMonitor()).parse(originalReader); TexParser texParser = new DefaultTexParser(result.getDatabase()); TexParserResult texResult = texParser.parse(texFile); + TexParserResult expectedResult = new TexParserResult(result.getDatabase(), 0, 0, 1); BibDatabase newDatabase = texResult.getGeneratedBibDatabase(); + expectedResult.getUniqueKeys().put(EINSTEIN_A, new ArrayList<>()); + expectedResult.getUniqueKeys().get(EINSTEIN_A).add(new Citation(texFile, 4, 48, 68, "This is some content trying to cite a bib file: \\cite{Einstein1920a}")); + expectedResult.getGeneratedBibDatabase().insertEntry((BibEntry) expectedResult.getMasterDatabase().getEntryByKey(EINSTEIN_A).get().clone()); + expectedResult.getGeneratedBibDatabase().insertEntry((BibEntry) expectedResult.getMasterDatabase().getEntryByKey(EINSTEIN).get().clone()); + + expectedResult.getUniqueKeys().put(EINSTEIN_B, new ArrayList<>()); + expectedResult.getUniqueKeys().get(EINSTEIN_B).add(new Citation(texFile, 5, 48, 68, "This is some content trying to cite a bib file: \\cite{Einstein1920b}")); + expectedResult.getGeneratedBibDatabase().insertEntry((BibEntry) expectedResult.getMasterDatabase().getEntryByKey(EINSTEIN_B).get().clone()); + expectedResult.getUnresolvedKeys().add(EINSTEIN21); + + expectedResult.getUniqueKeys().put(EINSTEIN_C, new ArrayList<>()); + expectedResult.getUniqueKeys().get(EINSTEIN_C).add(new Citation(texFile, 6, 48, 68, "This is some content trying to cite a bib file: \\cite{Einstein1920c}")); + expectedResult.getGeneratedBibDatabase().insertEntry((BibEntry) expectedResult.getMasterDatabase().getEntryByKey(EINSTEIN_C).get().clone()); + + expectedResult.getUniqueKeys().put(UNRESOLVED, new ArrayList<>()); + expectedResult.getUniqueKeys().get(UNRESOLVED).add(new Citation(texFile, 7, 48, 68, "This is some content trying to cite a bib file: \\cite{UnresolvedKey}")); + expectedResult.getUnresolvedKeys().add(UNRESOLVED); + + assertEquals(result.getDatabase(), texResult.getMasterDatabase()); assertTrue(texResult.getGeneratedBibDatabase().hasEntries()); - assertEquals(4, texResult.getFoundKeysInTex()); - assertEquals(2, texResult.getUnresolvedKeysCount()); - assertEquals(3, texResult.getResolvedKeysCount()); - assertEquals(0, texResult.getNestedFilesCount()); - assertEquals(1, texResult.getCrossRefEntriesCount()); - assertEquals(texResult.getFoundKeysInTex() + texResult.getCrossRefEntriesCount(), - texResult.getResolvedKeysCount() + texResult.getUnresolvedKeysCount()); - assertEquals(4, newDatabase.getEntries().size()); + assertEquals(texResult.getFoundKeysInTex() + texResult.getCrossRefEntriesCount(), texResult.getResolvedKeysCount() + texResult.getUnresolvedKeysCount()); + assertEquals(expectedResult.getResolvedKeysCount() + expectedResult.getCrossRefEntriesCount(), newDatabase.getEntries().size()); + assertEquals(expectedResult, texResult); } } } diff --git a/src/test/resources/org/jabref/logic/texparser/nested.tex b/src/test/resources/org/jabref/logic/texparser/nested.tex index c68348d3241..e43fa1eaf14 100644 --- a/src/test/resources/org/jabref/logic/texparser/nested.tex +++ b/src/test/resources/org/jabref/logic/texparser/nested.tex @@ -1,9 +1,9 @@ \documentclass{article} - \begin{document} + This is some content trying to cite a bib file. +\include{nested2} \bibliographystyle{plain} \bibliography{origin} -\include{paper.tex} \end{document} diff --git a/src/test/resources/org/jabref/logic/texparser/nested2.tex b/src/test/resources/org/jabref/logic/texparser/nested2.tex new file mode 100644 index 00000000000..84e07c2ca67 --- /dev/null +++ b/src/test/resources/org/jabref/logic/texparser/nested2.tex @@ -0,0 +1,9 @@ +\documentclass{article} +\begin{document} + +This is some content trying to cite a bib file. +\input{paper} + +\bibliographystyle{plain} +\bibliography{origin} +\end{document} diff --git a/src/test/resources/org/jabref/logic/texparser/paper.tex b/src/test/resources/org/jabref/logic/texparser/paper.tex index 6379c6ee027..5a8afd4ab0e 100644 --- a/src/test/resources/org/jabref/logic/texparser/paper.tex +++ b/src/test/resources/org/jabref/logic/texparser/paper.tex @@ -1,10 +1,10 @@ \documentclass{article} - \begin{document} + \cite{Einstein1920} -Nunc ultricies leo nec libero rhoncus, eu vehicula enim efficitur.~\cite{Darwin1888} -Einstein said~\cite{Einstein1920} that lorem impsum, consectetur adipiscing elit. Integer eros tortor, dictum nec aliquet in, pharetra nec justo. \cite{Darwin1888}. +Einstein said \cite{Einstein1920} that lorem impsum, consectetur adipiscing elit. +Nunc ultricies leo nec libero rhoncus, eu vehicula enim efficitur. \cite{Darwin1888} % Comment with \cite{Darwin1888} diff --git a/src/test/resources/org/jabref/logic/texparser/paper2.tex b/src/test/resources/org/jabref/logic/texparser/paper2.tex index 1b730d244d7..6e3f81fee3c 100644 --- a/src/test/resources/org/jabref/logic/texparser/paper2.tex +++ b/src/test/resources/org/jabref/logic/texparser/paper2.tex @@ -1,6 +1,6 @@ \documentclass{article} - \begin{document} + This is some content trying to cite a bib file: \cite{Darwin1888} This is some content trying to cite a bib file: \cite{Einstein1920} This is some content trying to cite a bib file: \cite{Newton1999} diff --git a/src/test/resources/org/jabref/logic/texparser/unknown_key.tex b/src/test/resources/org/jabref/logic/texparser/unknown_key.tex index 68f72ed2edc..3543179b4c4 100644 --- a/src/test/resources/org/jabref/logic/texparser/unknown_key.tex +++ b/src/test/resources/org/jabref/logic/texparser/unknown_key.tex @@ -1,6 +1,6 @@ \documentclass{article} - \begin{document} + This is some content trying to cite a bib file: \cite{Darwin1888} This is some content trying to cite a bib file: \cite{Einstein1920} This is some content trying to cite a bib file: \cite{UnknownKey} From 0e47a8bebaafcf82f5fc4033f8fa6b75b497b9e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20M=C3=A9ndez?= Date: Tue, 4 Jun 2019 22:46:05 +0200 Subject: [PATCH 04/57] Move database-related methods to the CrossReferences class --- .../logic/texparser/CrossReferences.java | 42 ++++++++++- .../logic/texparser/DefaultTexParser.java | 74 ++++--------------- .../org/jabref/model/texparser/TexParser.java | 2 +- .../jabref/logic/texparser/TexParserTest.java | 2 +- 4 files changed, 57 insertions(+), 63 deletions(-) diff --git a/src/main/java/org/jabref/logic/texparser/CrossReferences.java b/src/main/java/org/jabref/logic/texparser/CrossReferences.java index 443a5b00614..c46d5f578c4 100644 --- a/src/main/java/org/jabref/logic/texparser/CrossReferences.java +++ b/src/main/java/org/jabref/logic/texparser/CrossReferences.java @@ -1,22 +1,50 @@ package org.jabref.logic.texparser; import java.util.Optional; +import java.util.Set; import org.jabref.model.database.BibDatabase; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.FieldName; import org.jabref.model.texparser.TexParserResult; -public class CrossReferences { +class CrossReferences { private CrossReferences() throws IllegalStateException { throw new IllegalStateException("Utility class"); } - protected static void resolve(BibDatabase database, TexParserResult result, BibEntry entry) { + /** + * Look for an equivalent BibTeX entry within the reference database for all keys inside of the TEX files. + */ + static void resolveKeys(TexParserResult result) { + BibDatabase masterDatabase = result.getMasterDatabase(); + Set keySet = result.getUniqueKeys().keySet(); + + for (String key : keySet) { + if (!result.getGeneratedBibDatabase().getEntryByKey(key).isPresent()) { + Optional entry = masterDatabase.getEntryByKey(key); + + if (entry.isPresent()) { + insertEntry(result, entry.get()); + resolveCrossReferences(result, masterDatabase, entry.get()); + } else { + result.getUnresolvedKeys().add(key); + } + } + } + + // Copy database definitions + if (result.getGeneratedBibDatabase().hasEntries()) { + result.getGeneratedBibDatabase().copyPreamble(masterDatabase); + result.insertStrings(masterDatabase.getUsedStrings(result.getGeneratedBibDatabase().getEntries())); + } + } + + private static void resolveCrossReferences(TexParserResult result, BibDatabase masterDatabase, BibEntry entry) { entry.getField(FieldName.CROSSREF).ifPresent(crossRef -> { if (!result.getGeneratedBibDatabase().getEntryByKey(crossRef).isPresent()) { - Optional refEntry = database.getEntryByKey(crossRef); + Optional refEntry = masterDatabase.getEntryByKey(crossRef); if (refEntry.isPresent()) { result.getGeneratedBibDatabase().insertEntry((BibEntry) refEntry.get().clone()); @@ -27,4 +55,12 @@ protected static void resolve(BibDatabase database, TexParserResult result, BibE } }); } + + /** + * Insert into the database a clone of the given entry. The cloned entry has a new unique ID. + */ + private static void insertEntry(TexParserResult result, BibEntry entry) { + BibEntry clonedEntry = (BibEntry) entry.clone(); + result.getGeneratedBibDatabase().insertEntry(clonedEntry); + } } diff --git a/src/main/java/org/jabref/logic/texparser/DefaultTexParser.java b/src/main/java/org/jabref/logic/texparser/DefaultTexParser.java index 55e6a25d4c8..3707c5916e1 100644 --- a/src/main/java/org/jabref/logic/texparser/DefaultTexParser.java +++ b/src/main/java/org/jabref/logic/texparser/DefaultTexParser.java @@ -9,13 +9,10 @@ import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Optional; -import java.util.Set; import java.util.regex.Matcher; import java.util.regex.Pattern; import org.jabref.model.database.BibDatabase; -import org.jabref.model.entry.BibEntry; import org.jabref.model.texparser.Citation; import org.jabref.model.texparser.TexParser; import org.jabref.model.texparser.TexParserResult; @@ -43,34 +40,29 @@ public class DefaultTexParser implements TexParser { private static final String INCLUDE_REGEX = "\\\\(?:include|input)\\{(?[^\\}]*)\\}"; - private final BibDatabase masterDatabase; + private final TexParserResult result; - public DefaultTexParser(BibDatabase database) { - masterDatabase = database; + public DefaultTexParser(BibDatabase masterDatabase) { + this.result = new TexParserResult(masterDatabase); } @Override public TexParserResult parse(String citeString) { - TexParserResult result = new TexParserResult(masterDatabase); - matchCitation(result, Paths.get("foo/bar"), 1, citeString); - resolveTags(result); + matchCitation(Paths.get("foo/bar"), 1, citeString); + CrossReferences.resolveKeys(result); return result; } @Override public TexParserResult parse(Path texFile) { - return parse(new TexParserResult(masterDatabase), Collections.singletonList(texFile)); - } - - @Override - public TexParserResult parse(List texFiles) { - return parse(new TexParserResult(masterDatabase), texFiles); + return parse(Collections.singletonList(texFile)); } /** * Parse a list of TEX files and, recursively, their referenced files. */ - private TexParserResult parse(TexParserResult result, List texFiles) { + @Override + public TexParserResult parse(List texFiles) { List referencedFiles = new ArrayList<>(); for (int fileIndex = 0; fileIndex < texFiles.size(); fileIndex++) { @@ -83,8 +75,8 @@ private TexParserResult parse(TexParserResult result, List texFiles) { continue; } - matchCitation(result, file, lnr.getLineNumber(), line); - matchNestedFile(result, file, texFiles, referencedFiles, line); + matchCitation(file, lnr.getLineNumber(), line); + matchNestedFile(file, texFiles, referencedFiles, line); } } catch (IOException e) { LOGGER.warn("Error opening the TEX file", e); @@ -92,24 +84,24 @@ private TexParserResult parse(TexParserResult result, List texFiles) { } if (!referencedFiles.isEmpty()) { - parse(result, referencedFiles); + parse(referencedFiles); } - resolveTags(result); + CrossReferences.resolveKeys(result); return result; } /** * Find cites along a specific line and store them. */ - private void matchCitation(TexParserResult result, Path file, int lineNumber, String line) { + private void matchCitation(Path file, int lineNumber, String line) { Matcher citeMatch = Pattern.compile(CITE_REGEX).matcher(line); while (citeMatch.find()) { String[] keys = citeMatch.group("key").split(","); for (String key : keys) { - addKey(result, key.trim(), new Citation(file, lineNumber, citeMatch.start(), citeMatch.end(), line)); + addKey(key.trim(), new Citation(file, lineNumber, citeMatch.start(), citeMatch.end(), line)); } } } @@ -117,7 +109,7 @@ private void matchCitation(TexParserResult result, Path file, int lineNumber, St /** * Find inputs and includes along a specific line and store them for parsing later. */ - private void matchNestedFile(TexParserResult result, Path file, List texFiles, List referencedFiles, String line) { + private void matchNestedFile(Path file, List texFiles, List referencedFiles, String line) { Matcher includeMatch = Pattern.compile(INCLUDE_REGEX).matcher(line); while (includeMatch.find()) { @@ -142,7 +134,7 @@ private void matchNestedFile(TexParserResult result, Path file, List texFi /** * Add a citation to the uniqueKeys map. */ - private void addKey(TexParserResult result, String key, Citation citation) { + private void addKey(String key, Citation citation) { Map> uniqueKeys = result.getUniqueKeys(); if (!uniqueKeys.containsKey(key)) { @@ -153,38 +145,4 @@ private void addKey(TexParserResult result, String key, Citation citation) { uniqueKeys.get(key).add(citation); } } - - /** - * Look for an equivalent BibTeX entry within the reference database for all keys inside of the TEX files. - */ - private void resolveTags(TexParserResult result) { - Set keySet = result.getUniqueKeys().keySet(); - - for (String key : keySet) { - if (!result.getGeneratedBibDatabase().getEntryByKey(key).isPresent()) { - Optional entry = masterDatabase.getEntryByKey(key); - - if (entry.isPresent()) { - insertEntry(result, entry.get()); - CrossReferences.resolve(masterDatabase, result, entry.get()); - } else { - result.getUnresolvedKeys().add(key); - } - } - } - - // Copy database definitions - if (result.getGeneratedBibDatabase().hasEntries()) { - result.getGeneratedBibDatabase().copyPreamble(masterDatabase); - result.insertStrings(masterDatabase.getUsedStrings(result.getGeneratedBibDatabase().getEntries())); - } - } - - /** - * Insert into the database a clone of the given entry. The cloned entry has a new unique ID. - */ - private void insertEntry(TexParserResult result, BibEntry entry) { - BibEntry clonedEntry = (BibEntry) entry.clone(); - result.getGeneratedBibDatabase().insertEntry(clonedEntry); - } } diff --git a/src/main/java/org/jabref/model/texparser/TexParser.java b/src/main/java/org/jabref/model/texparser/TexParser.java index 3a0eb0a9cc5..5b7b72be308 100644 --- a/src/main/java/org/jabref/model/texparser/TexParser.java +++ b/src/main/java/org/jabref/model/texparser/TexParser.java @@ -9,7 +9,7 @@ public interface TexParser { * For testing purposes. * * @param citeString String that contains a citation - * @return a TexParserResult, where Path is /foo/bar and lineNumber is 1 + * @return a TexParserResult, where Path is foo/bar and lineNumber is 1 */ TexParserResult parse(String citeString); diff --git a/src/test/java/org/jabref/logic/texparser/TexParserTest.java b/src/test/java/org/jabref/logic/texparser/TexParserTest.java index 8bf4f5e4196..aedd47ca48d 100644 --- a/src/test/java/org/jabref/logic/texparser/TexParserTest.java +++ b/src/test/java/org/jabref/logic/texparser/TexParserTest.java @@ -26,8 +26,8 @@ import org.junit.jupiter.api.Test; import org.mockito.Answers; -import static org.junit.Assert.assertFalse; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; From deb44d63dfb3cc154c6abe9be7d5c61aa9acdd4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20M=C3=A9ndez?= Date: Wed, 5 Jun 2019 15:18:04 +0200 Subject: [PATCH 05/57] Refactor code --- .../logic/texparser/CrossReferences.java | 66 ---- .../jabref/logic/texparser/CrossingKeys.java | 73 ++++ .../logic/texparser/DefaultTexParser.java | 59 ++-- .../org/jabref/model/texparser/Citation.java | 3 +- .../model/texparser/CrossingKeysResult.java | 103 ++++++ .../model/texparser/TexParserResult.java | 110 ++---- .../jabref/logic/texparser/TexParserTest.java | 325 +++++++++--------- 7 files changed, 377 insertions(+), 362 deletions(-) delete mode 100644 src/main/java/org/jabref/logic/texparser/CrossReferences.java create mode 100644 src/main/java/org/jabref/logic/texparser/CrossingKeys.java create mode 100644 src/main/java/org/jabref/model/texparser/CrossingKeysResult.java diff --git a/src/main/java/org/jabref/logic/texparser/CrossReferences.java b/src/main/java/org/jabref/logic/texparser/CrossReferences.java deleted file mode 100644 index c46d5f578c4..00000000000 --- a/src/main/java/org/jabref/logic/texparser/CrossReferences.java +++ /dev/null @@ -1,66 +0,0 @@ -package org.jabref.logic.texparser; - -import java.util.Optional; -import java.util.Set; - -import org.jabref.model.database.BibDatabase; -import org.jabref.model.entry.BibEntry; -import org.jabref.model.entry.FieldName; -import org.jabref.model.texparser.TexParserResult; - -class CrossReferences { - - private CrossReferences() throws IllegalStateException { - throw new IllegalStateException("Utility class"); - } - - /** - * Look for an equivalent BibTeX entry within the reference database for all keys inside of the TEX files. - */ - static void resolveKeys(TexParserResult result) { - BibDatabase masterDatabase = result.getMasterDatabase(); - Set keySet = result.getUniqueKeys().keySet(); - - for (String key : keySet) { - if (!result.getGeneratedBibDatabase().getEntryByKey(key).isPresent()) { - Optional entry = masterDatabase.getEntryByKey(key); - - if (entry.isPresent()) { - insertEntry(result, entry.get()); - resolveCrossReferences(result, masterDatabase, entry.get()); - } else { - result.getUnresolvedKeys().add(key); - } - } - } - - // Copy database definitions - if (result.getGeneratedBibDatabase().hasEntries()) { - result.getGeneratedBibDatabase().copyPreamble(masterDatabase); - result.insertStrings(masterDatabase.getUsedStrings(result.getGeneratedBibDatabase().getEntries())); - } - } - - private static void resolveCrossReferences(TexParserResult result, BibDatabase masterDatabase, BibEntry entry) { - entry.getField(FieldName.CROSSREF).ifPresent(crossRef -> { - if (!result.getGeneratedBibDatabase().getEntryByKey(crossRef).isPresent()) { - Optional refEntry = masterDatabase.getEntryByKey(crossRef); - - if (refEntry.isPresent()) { - result.getGeneratedBibDatabase().insertEntry((BibEntry) refEntry.get().clone()); - result.increaseCrossRefEntriesCounter(); - } else { - result.getUnresolvedKeys().add(crossRef); - } - } - }); - } - - /** - * Insert into the database a clone of the given entry. The cloned entry has a new unique ID. - */ - private static void insertEntry(TexParserResult result, BibEntry entry) { - BibEntry clonedEntry = (BibEntry) entry.clone(); - result.getGeneratedBibDatabase().insertEntry(clonedEntry); - } -} diff --git a/src/main/java/org/jabref/logic/texparser/CrossingKeys.java b/src/main/java/org/jabref/logic/texparser/CrossingKeys.java new file mode 100644 index 00000000000..457c71acc01 --- /dev/null +++ b/src/main/java/org/jabref/logic/texparser/CrossingKeys.java @@ -0,0 +1,73 @@ +package org.jabref.logic.texparser; + +import java.util.Optional; +import java.util.Set; + +import org.jabref.model.database.BibDatabase; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.FieldName; +import org.jabref.model.texparser.CrossingKeysResult; +import org.jabref.model.texparser.TexParserResult; + +class CrossingKeys { + + private final CrossingKeysResult result; + + public CrossingKeys(TexParserResult texParserResult, BibDatabase masterDatabase) { + this.result = new CrossingKeysResult(texParserResult, masterDatabase); + } + + /** + * Look for an equivalent BibTeX entry within the reference database for all keys inside of the TEX files. + */ + public CrossingKeysResult resolveKeys() { + Set keySet = result.getParserResult().getCitations().keySet(); + + for (String key : keySet) { + if (!result.getNewDatabase().getEntryByKey(key).isPresent()) { + Optional entry = result.getMasterDatabase().getEntryByKey(key); + + if (entry.isPresent()) { + insertEntry(entry.get()); + resolveCrossReferences(entry.get()); + } else { + result.getUnresolvedKeys().add(key); + } + } + } + + // Copy database definitions. + if (result.getNewDatabase().hasEntries()) { + result.getNewDatabase().copyPreamble(result.getMasterDatabase()); + result.insertStrings(result.getMasterDatabase().getUsedStrings(result.getNewDatabase().getEntries())); + } + + return result; + } + + /** + * Find cross references for inserting into the new database. + */ + private void resolveCrossReferences(BibEntry entry) { + entry.getField(FieldName.CROSSREF).ifPresent(crossRef -> { + if (!result.getNewDatabase().getEntryByKey(crossRef).isPresent()) { + Optional refEntry = result.getMasterDatabase().getEntryByKey(crossRef); + + if (refEntry.isPresent()) { + insertEntry(refEntry.get()); + result.increaseCrossRefEntriesCounter(); + } else { + result.getUnresolvedKeys().add(crossRef); + } + } + }); + } + + /** + * Insert into the database a clone of the given entry. The cloned entry has a new unique ID. + */ + private void insertEntry(BibEntry entry) { + BibEntry clonedEntry = (BibEntry) entry.clone(); + result.getNewDatabase().insertEntry(clonedEntry); + } +} diff --git a/src/main/java/org/jabref/logic/texparser/DefaultTexParser.java b/src/main/java/org/jabref/logic/texparser/DefaultTexParser.java index 3707c5916e1..97f6c4406f3 100644 --- a/src/main/java/org/jabref/logic/texparser/DefaultTexParser.java +++ b/src/main/java/org/jabref/logic/texparser/DefaultTexParser.java @@ -8,11 +8,9 @@ import java.util.ArrayList; import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.jabref.model.database.BibDatabase; import org.jabref.model.texparser.Citation; import org.jabref.model.texparser.TexParser; import org.jabref.model.texparser.TexParserResult; @@ -31,25 +29,25 @@ public class DefaultTexParser implements TexParser { * *

TODO: Add support for multicite commands. */ + private static final String[] CITE_COMMANDS = new String[] { + "[cC]ite(alt|alp|author|authorfull|date|num|p|t|text|title|url|year|yearpar)?", + "([aA]|fnote|foot|footfull|full|no|[nN]ote|[pP]aren|[pP]note|[tT]ext|[sS]mart|super)cite", + "footcitetext" + }; private static final String CITE_REGEX = String.format("\\\\(?:%s)\\*?(?:\\[(?:[^\\]]*)\\]){0,2}\\{(?[^\\}]*)\\}", - String.join("|", new String[] { - "[cC]ite(alt|alp|author|authorfull|date|num|p|t|text|title|url|year|yearpar)?", - "([aA]|fnote|foot|footfull|full|no|[nN]ote|[pP]aren|[pP]note|[tT]ext|[sS]mart|super)cite", - "footcitetext" - })); + String.join("|", CITE_COMMANDS)); private static final String INCLUDE_REGEX = "\\\\(?:include|input)\\{(?[^\\}]*)\\}"; private final TexParserResult result; - public DefaultTexParser(BibDatabase masterDatabase) { - this.result = new TexParserResult(masterDatabase); + public DefaultTexParser() { + this.result = new TexParserResult(); } @Override public TexParserResult parse(String citeString) { matchCitation(Paths.get("foo/bar"), 1, citeString); - CrossReferences.resolveKeys(result); return result; } @@ -58,20 +56,23 @@ public TexParserResult parse(Path texFile) { return parse(Collections.singletonList(texFile)); } - /** - * Parse a list of TEX files and, recursively, their referenced files. - */ @Override public TexParserResult parse(List texFiles) { List referencedFiles = new ArrayList<>(); + if (result.getFileList().isEmpty()) { + result.getFileList().addAll(texFiles); + } else { + result.getNestedFiles().addAll(texFiles); + } + for (int fileIndex = 0; fileIndex < texFiles.size(); fileIndex++) { Path file = texFiles.get(fileIndex); try (LineNumberReader lnr = new LineNumberReader(Files.newBufferedReader(file))) { for (String line = lnr.readLine(); line != null; line = lnr.readLine()) { - // Skip comment lines. if (line.startsWith("%")) { + // Skip comment lines. continue; } @@ -83,16 +84,16 @@ public TexParserResult parse(List texFiles) { } } + // Parse all files referenced by TEX files, recursively. if (!referencedFiles.isEmpty()) { parse(referencedFiles); } - CrossReferences.resolveKeys(result); return result; } /** - * Find cites along a specific line and store them. + * Find cites along a specific line and add them to a map. */ private void matchCitation(Path file, int lineNumber, String line) { Matcher citeMatch = Pattern.compile(CITE_REGEX).matcher(line); @@ -101,7 +102,15 @@ private void matchCitation(Path file, int lineNumber, String line) { String[] keys = citeMatch.group("key").split(","); for (String key : keys) { - addKey(key.trim(), new Citation(file, lineNumber, citeMatch.start(), citeMatch.end(), line)); + Citation citation = new Citation(file, lineNumber, citeMatch.start(), citeMatch.end(), line); + + if (!result.getCitations().containsKey(key)) { + result.getCitations().put(key, new ArrayList<>()); + } + + if (!result.getCitations().get(key).contains(citation)) { + result.getCitations().get(key).add(citation); + } } } } @@ -126,23 +135,7 @@ private void matchNestedFile(Path file, List texFiles, List referenc if (!texFiles.contains(inputFile)) { referencedFiles.add(inputFile); - result.increaseNestedFilesCounter(); } } } - - /** - * Add a citation to the uniqueKeys map. - */ - private void addKey(String key, Citation citation) { - Map> uniqueKeys = result.getUniqueKeys(); - - if (!uniqueKeys.containsKey(key)) { - uniqueKeys.put(key, new ArrayList<>()); - } - - if (!uniqueKeys.get(key).contains(citation)) { - uniqueKeys.get(key).add(citation); - } - } } diff --git a/src/main/java/org/jabref/model/texparser/Citation.java b/src/main/java/org/jabref/model/texparser/Citation.java index 9ea642e16e7..003c0704e80 100644 --- a/src/main/java/org/jabref/model/texparser/Citation.java +++ b/src/main/java/org/jabref/model/texparser/Citation.java @@ -82,8 +82,7 @@ public boolean equals(Object o) { && line == citation.line && colStart == citation.colStart && colEnd == citation.colEnd - && lineText.equals(citation.lineText) - && getContext().equals(citation.getContext()); + && lineText.equals(citation.lineText); } @Override diff --git a/src/main/java/org/jabref/model/texparser/CrossingKeysResult.java b/src/main/java/org/jabref/model/texparser/CrossingKeysResult.java new file mode 100644 index 00000000000..4db2853ce22 --- /dev/null +++ b/src/main/java/org/jabref/model/texparser/CrossingKeysResult.java @@ -0,0 +1,103 @@ +package org.jabref.model.texparser; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Objects; + +import org.jabref.model.database.BibDatabase; +import org.jabref.model.entry.BibtexString; + +public class CrossingKeysResult { + + private final TexParserResult texParserResult; + private final BibDatabase masterDatabase; + private final List unresolvedKeys; + private final BibDatabase newDatabase; + private int insertedStrings; + private int crossRefEntriesCount; + + public CrossingKeysResult(TexParserResult texParserResult, BibDatabase masterDatabase, int insertedStrings, int crossRefEntriesCount) { + this.texParserResult = texParserResult; + this.masterDatabase = masterDatabase; + this.unresolvedKeys = new ArrayList<>(); + this.newDatabase = new BibDatabase(); + this.insertedStrings = insertedStrings; + this.crossRefEntriesCount = crossRefEntriesCount; + } + + public CrossingKeysResult(TexParserResult texParserResult, BibDatabase masterDatabase) { + this(texParserResult, masterDatabase, 0, 0); + } + + public TexParserResult getParserResult() { + return texParserResult; + } + + public BibDatabase getMasterDatabase() { + return masterDatabase; + } + + public List getUnresolvedKeys() { + return unresolvedKeys; + } + + public BibDatabase getNewDatabase() { + return newDatabase; + } + + public int getInsertedStrings() { + return insertedStrings; + } + + public int getResolvedKeysCount() { + return newDatabase.getEntryCount() - crossRefEntriesCount; + } + + public int getCrossRefEntriesCount() { + return crossRefEntriesCount; + } + + public void increaseCrossRefEntriesCounter() { + crossRefEntriesCount++; + } + + public void insertStrings(Collection usedStrings) { + for (BibtexString string : usedStrings) { + newDatabase.addString(string); + insertedStrings++; + } + } + + @Override + public String toString() { + return String.format("%nCrossReferencesResult{texParserResult=%s // masterDatabase=%s // unresolvedKeys=%s // newDatabase=%s // insertedStrings=%s // crossRefEntriesCount=%s}%n", + texParserResult, masterDatabase, unresolvedKeys, newDatabase, insertedStrings, crossRefEntriesCount); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + + if (o == null || getClass() != o.getClass()) { + return false; + } + + CrossingKeysResult result = (CrossingKeysResult) o; + + return texParserResult.equals(result.texParserResult) + && masterDatabase.equals(result.masterDatabase) + && unresolvedKeys.equals(result.unresolvedKeys) + && (new HashSet<>(newDatabase.getEntries())).equals(new HashSet<>(result.newDatabase.getEntries())) + && insertedStrings == result.insertedStrings + && crossRefEntriesCount == result.crossRefEntriesCount; + } + + @Override + public int hashCode() { + return Objects.hash(masterDatabase, unresolvedKeys, newDatabase, insertedStrings, crossRefEntriesCount); + } +} diff --git a/src/main/java/org/jabref/model/texparser/TexParserResult.java b/src/main/java/org/jabref/model/texparser/TexParserResult.java index 61e83c80330..4c8549ecf4c 100644 --- a/src/main/java/org/jabref/model/texparser/TexParserResult.java +++ b/src/main/java/org/jabref/model/texparser/TexParserResult.java @@ -1,106 +1,40 @@ package org.jabref.model.texparser; +import java.nio.file.Path; import java.util.ArrayList; -import java.util.Collection; import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; -import org.jabref.model.database.BibDatabase; -import org.jabref.model.entry.BibtexString; - public class TexParserResult { - private final BibDatabase masterDatabase; - private final Map> uniqueKeys = new HashMap<>(); - private final List unresolvedKeys = new ArrayList<>(); - private final BibDatabase texDatabase = new BibDatabase(); - private int insertedStrings = 0; - private int nestedFilesCount = 0; - private int crossRefEntriesCount = 0; - - public TexParserResult(BibDatabase masterDatabase, int insertedStrings, int nestedFilesCount, int crossRefEntriesCount) { - this.masterDatabase = masterDatabase; - this.insertedStrings = insertedStrings; - this.nestedFilesCount = nestedFilesCount; - this.crossRefEntriesCount = crossRefEntriesCount; - } - - public TexParserResult(BibDatabase masterDatabase) { - this.masterDatabase = masterDatabase; - } - - public BibDatabase getMasterDatabase() { - return masterDatabase; - } - - public Map> getUniqueKeys() { - return uniqueKeys; - } - - public int getFoundKeysInTex() { - return uniqueKeys.size(); - } - - public int getCitationsCountByKey(String key) { - return uniqueKeys.get(key).size(); - } - - public List getUnresolvedKeys() { - return unresolvedKeys; - } - - public int getUnresolvedKeysCount() { - return unresolvedKeys.size(); - } - - public BibDatabase getGeneratedBibDatabase() { - return texDatabase; - } - - public int getResolvedKeysCount() { - return texDatabase.getEntryCount() - crossRefEntriesCount; - } - - public int getInsertedStrings() { - return insertedStrings; - } - - public void insertStrings(Collection usedStrings) { - for (BibtexString string : usedStrings) { - texDatabase.addString(string); - insertedStrings++; - } - } + private final List fileList; + private final List nestedFiles; + private final Map> citations; - public int getNestedFilesCount() { - return nestedFilesCount; + public TexParserResult() { + this.fileList = new ArrayList<>(); + this.nestedFiles = new ArrayList<>(); + this.citations = new HashMap<>(); } - public void increaseNestedFilesCounter() { - nestedFilesCount++; + public List getFileList() { + return fileList; } - public int getCrossRefEntriesCount() { - return crossRefEntriesCount; + public List getNestedFiles() { + return nestedFiles; } - public void increaseCrossRefEntriesCounter() { - crossRefEntriesCount++; + public Map> getCitations() { + return citations; } @Override public String toString() { - return String.format("%nTexParserResult{%n masterDatabase=%s,%n uniqueKeys=%s,%n unresolvedKeys=%s,%n texDatabase=%s,%n insertedStrings=%s,%n nestedFilesCount=%s,%n crossRefEntriesCount=%s%n}%n", - masterDatabase, - uniqueKeys, - unresolvedKeys, - texDatabase, - insertedStrings, - nestedFilesCount, - crossRefEntriesCount); + return String.format("%nTexParserResult{fileList=%s // nestedFiles=%s // citations=%s}%n", + fileList, nestedFiles, citations); } @Override @@ -115,17 +49,13 @@ public boolean equals(Object o) { TexParserResult result = (TexParserResult) o; - return masterDatabase.equals(result.masterDatabase) - && uniqueKeys.equals(result.uniqueKeys) - && unresolvedKeys.equals(result.unresolvedKeys) - && (new HashSet<>(texDatabase.getEntries())).equals(new HashSet<>(result.texDatabase.getEntries())) - && insertedStrings == result.insertedStrings - && nestedFilesCount == result.nestedFilesCount - && crossRefEntriesCount == result.crossRefEntriesCount; + return fileList.equals(result.fileList) + && nestedFiles.equals(result.nestedFiles) + && citations.equals(result.citations); } @Override public int hashCode() { - return Objects.hash(masterDatabase, uniqueKeys, unresolvedKeys, texDatabase, insertedStrings, nestedFilesCount, crossRefEntriesCount); + return Objects.hash(fileList, nestedFiles, citations); } } diff --git a/src/test/java/org/jabref/logic/texparser/TexParserTest.java b/src/test/java/org/jabref/logic/texparser/TexParserTest.java index aedd47ca48d..f4544845d91 100644 --- a/src/test/java/org/jabref/logic/texparser/TexParserTest.java +++ b/src/test/java/org/jabref/logic/texparser/TexParserTest.java @@ -14,10 +14,9 @@ import org.jabref.logic.importer.ImportFormatPreferences; import org.jabref.logic.importer.ParserResult; import org.jabref.logic.importer.fileformat.BibtexParser; -import org.jabref.model.database.BibDatabase; import org.jabref.model.entry.BibEntry; import org.jabref.model.texparser.Citation; -import org.jabref.model.texparser.TexParser; +import org.jabref.model.texparser.CrossingKeysResult; import org.jabref.model.texparser.TexParserResult; import org.jabref.model.util.DummyFileUpdateMonitor; @@ -27,16 +26,13 @@ import org.mockito.Answers; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; class TexParserTest { - private ImportFormatPreferences importFormatPreferences; - private final String DARWIN = "Darwin1888"; private final String EINSTEIN = "Einstein1920"; private final String UNRESOLVED = "UnresolvedKey"; + private ImportFormatPreferences importFormatPreferences; @BeforeEach void setUp() { @@ -48,54 +44,58 @@ void tearDown() { importFormatPreferences = null; } - private void addCite(TexParserResult expectedResult, String key, Path texFile, int line, int colStart, int colEnd, String lineText, int resolved) { - if (!expectedResult.getUniqueKeys().containsKey(key)) { - expectedResult.getUniqueKeys().put(key, new ArrayList<>()); - } else { - resolved = 0; - } + private void testCite(String key, String citeString, boolean match) { + TexParserResult texParserResult = new DefaultTexParser().parse(citeString); + TexParserResult expectedResult = new TexParserResult(); - Citation citation = new Citation(texFile, line, colStart, colEnd, lineText); - if (!expectedResult.getUniqueKeys().get(key).contains(citation)) { - expectedResult.getUniqueKeys().get(key).add(citation); + if (match) { + expectedResult.getCitations().put(key, new ArrayList<>()); + expectedResult.getCitations().get(key).add( + new Citation(Paths.get("foo/bar"), 1, 0, citeString.length(), citeString)); } - if (resolved > 0) { - BibEntry clonedEntry = (BibEntry) expectedResult.getMasterDatabase().getEntryByKey(key).get().clone(); - expectedResult.getGeneratedBibDatabase().insertEntry(clonedEntry); - } else if (resolved < 0) { - expectedResult.getUnresolvedKeys().add(key); - } + assertEquals(expectedResult, texParserResult); } - private void testCite(String citeString) throws IOException { - InputStream originalStream = TexParserTest.class.getResourceAsStream("origin.bib"); + private void testMatchCite(String citeString) { + testCite(UNRESOLVED, citeString, true); + } - try (InputStreamReader originalReader = new InputStreamReader(originalStream, StandardCharsets.UTF_8)) { - ParserResult result = new BibtexParser(importFormatPreferences, new DummyFileUpdateMonitor()).parse(originalReader); - TexParser texParser = new DefaultTexParser(result.getDatabase()); - TexParserResult texResult = texParser.parse(citeString); - TexParserResult expectedResult = new TexParserResult(result.getDatabase()); - BibDatabase newDatabase = texResult.getGeneratedBibDatabase(); - - addCite(expectedResult, UNRESOLVED, Paths.get("foo/bar"), 1, 0, citeString.length(), citeString, -1); - - assertEquals(result.getDatabase(), texResult.getMasterDatabase()); - assertFalse(texResult.getGeneratedBibDatabase().hasEntries()); - assertEquals(texResult.getFoundKeysInTex() + texResult.getCrossRefEntriesCount(), texResult.getResolvedKeysCount() + texResult.getUnresolvedKeysCount()); - assertEquals(expectedResult.getResolvedKeysCount() + expectedResult.getCrossRefEntriesCount(), newDatabase.getEntries().size()); - assertEquals(expectedResult, texResult); - } + private void testNonMatchCite(String citeString) { + testCite(UNRESOLVED, citeString, false); } @Test - void testCiteCommands() throws IOException { - testCite("\\cite[pre][post]{UnresolvedKey}"); - testCite("\\cite*{UnresolvedKey}"); - testCite("\\parencite[post]{UnresolvedKey}"); - testCite("\\cite[pre][post]{UnresolvedKey}"); - testCite("\\citep{UnresolvedKey}"); - testCite("\\citet{UnresolvedKey}"); + void testCiteCommands() { + testMatchCite("\\cite[pre][post]{UnresolvedKey}"); + testMatchCite("\\cite*{UnresolvedKey}"); + testMatchCite("\\parencite[post]{UnresolvedKey}"); + testMatchCite("\\cite[pre][post]{UnresolvedKey}"); + testMatchCite("\\citep{UnresolvedKey}"); + + testNonMatchCite("\\citet21312{123U123n123resolvedKey}"); + testNonMatchCite("\\1cite[pr234e][post]{UnresolvedKey}"); + testNonMatchCite("\\citep55{5}UnresolvedKey}"); + testNonMatchCite("\\cit2et{UnresolvedKey}"); + } + + private void addCite(CrossingKeysResult expectedResult, String key, Path texFile, int line, int colStart, int colEnd, String lineText, boolean insert) { + Citation citation = new Citation(texFile, line, colStart, colEnd, lineText); + + if (!expectedResult.getParserResult().getCitations().containsKey(key)) { + expectedResult.getParserResult().getCitations().put(key, new ArrayList<>()); + } + + if (!expectedResult.getParserResult().getCitations().get(key).contains(citation)) { + expectedResult.getParserResult().getCitations().get(key).add(citation); + } + + if (insert) { + BibEntry clonedEntry = (BibEntry) expectedResult.getMasterDatabase().getEntryByKey(key).get().clone(); + expectedResult.getNewDatabase().insertEntry(clonedEntry); + } else { + expectedResult.getUnresolvedKeys().add(key); + } } @Test @@ -105,22 +105,19 @@ void testSingleFile() throws URISyntaxException, IOException { try (InputStreamReader originalReader = new InputStreamReader(originalStream, StandardCharsets.UTF_8)) { ParserResult result = new BibtexParser(importFormatPreferences, new DummyFileUpdateMonitor()).parse(originalReader); - TexParser texParser = new DefaultTexParser(result.getDatabase()); - TexParserResult texResult = texParser.parse(texFile); - TexParserResult expectedResult = new TexParserResult(result.getDatabase()); - BibDatabase newDatabase = texResult.getGeneratedBibDatabase(); - - addCite(expectedResult, EINSTEIN, texFile, 4, 0, 19, "\\cite{Einstein1920}", 1); - addCite(expectedResult, EINSTEIN, texFile, 6, 14, 33, "Einstein said \\cite{Einstein1920} that lorem impsum, consectetur adipiscing elit.", 1); - - addCite(expectedResult, DARWIN, texFile, 5, 0, 17, "\\cite{Darwin1888}.", 1); - addCite(expectedResult, DARWIN, texFile, 7, 67, 84, "Nunc ultricies leo nec libero rhoncus, eu vehicula enim efficitur. \\cite{Darwin1888}", 1); - - assertEquals(result.getDatabase(), texResult.getMasterDatabase()); - assertTrue(texResult.getGeneratedBibDatabase().hasEntries()); - assertEquals(texResult.getFoundKeysInTex() + texResult.getCrossRefEntriesCount(), texResult.getResolvedKeysCount() + texResult.getUnresolvedKeysCount()); - assertEquals(expectedResult.getResolvedKeysCount() + expectedResult.getCrossRefEntriesCount(), newDatabase.getEntries().size()); - assertEquals(expectedResult, texResult); + + TexParserResult parserResult = new DefaultTexParser().parse(texFile); + TexParserResult expectedParserResult = new TexParserResult(); + expectedParserResult.getFileList().add(texFile); + + CrossingKeysResult crossingResult = new CrossingKeys(parserResult, result.getDatabase()).resolveKeys(); + CrossingKeysResult expectedCrossingResult = new CrossingKeysResult(expectedParserResult, result.getDatabase()); + addCite(expectedCrossingResult, EINSTEIN, texFile, 4, 0, 19, "\\cite{Einstein1920}", true); + addCite(expectedCrossingResult, EINSTEIN, texFile, 6, 14, 33, "Einstein said \\cite{Einstein1920} that lorem impsum, consectetur adipiscing elit.", true); + addCite(expectedCrossingResult, DARWIN, texFile, 5, 0, 17, "\\cite{Darwin1888}.", true); + addCite(expectedCrossingResult, DARWIN, texFile, 7, 67, 84, "Nunc ultricies leo nec libero rhoncus, eu vehicula enim efficitur. \\cite{Darwin1888}", true); + + assertEquals(expectedCrossingResult, crossingResult); } } @@ -134,26 +131,22 @@ void testTwoFiles() throws URISyntaxException, IOException { try (InputStreamReader originalReader = new InputStreamReader(originalStream, StandardCharsets.UTF_8)) { ParserResult result = new BibtexParser(importFormatPreferences, new DummyFileUpdateMonitor()).parse(originalReader); - TexParser texParser = new DefaultTexParser(result.getDatabase()); - TexParserResult texResult = texParser.parse(Arrays.asList(texFile, texFile2)); - TexParserResult expectedResult = new TexParserResult(result.getDatabase()); - BibDatabase newDatabase = texResult.getGeneratedBibDatabase(); - - addCite(expectedResult, EINSTEIN, texFile, 4, 0, 19, "\\cite{Einstein1920}", 1); - addCite(expectedResult, EINSTEIN, texFile, 6, 14, 33, "Einstein said \\cite{Einstein1920} that lorem impsum, consectetur adipiscing elit.", 1); - addCite(expectedResult, EINSTEIN, texFile2, 5, 48, 67, "This is some content trying to cite a bib file: \\cite{Einstein1920}", 1); - - addCite(expectedResult, DARWIN, texFile, 5, 0, 17, "\\cite{Darwin1888}.", 1); - addCite(expectedResult, DARWIN, texFile, 7, 67, 84, "Nunc ultricies leo nec libero rhoncus, eu vehicula enim efficitur. \\cite{Darwin1888}", 1); - addCite(expectedResult, DARWIN, texFile2, 4, 48, 65, "This is some content trying to cite a bib file: \\cite{Darwin1888}", 1); - - addCite(expectedResult, NEWTON, texFile2, 6, 48, 65, "This is some content trying to cite a bib file: \\cite{Newton1999}", 1); - - assertEquals(result.getDatabase(), texResult.getMasterDatabase()); - assertTrue(texResult.getGeneratedBibDatabase().hasEntries()); - assertEquals(texResult.getFoundKeysInTex() + texResult.getCrossRefEntriesCount(), texResult.getResolvedKeysCount() + texResult.getUnresolvedKeysCount()); - assertEquals(expectedResult.getResolvedKeysCount() + expectedResult.getCrossRefEntriesCount(), newDatabase.getEntries().size()); - assertEquals(expectedResult, texResult); + + TexParserResult parserResult = new DefaultTexParser().parse(Arrays.asList(texFile, texFile2)); + TexParserResult expectedParserResult = new TexParserResult(); + expectedParserResult.getFileList().addAll(Arrays.asList(texFile, texFile2)); + + CrossingKeysResult crossingResult = new CrossingKeys(parserResult, result.getDatabase()).resolveKeys(); + CrossingKeysResult expectedCrossingResult = new CrossingKeysResult(expectedParserResult, result.getDatabase()); + addCite(expectedCrossingResult, EINSTEIN, texFile, 4, 0, 19, "\\cite{Einstein1920}", true); + addCite(expectedCrossingResult, EINSTEIN, texFile, 6, 14, 33, "Einstein said \\cite{Einstein1920} that lorem impsum, consectetur adipiscing elit.", true); + addCite(expectedCrossingResult, EINSTEIN, texFile2, 5, 48, 67, "This is some content trying to cite a bib file: \\cite{Einstein1920}", true); + addCite(expectedCrossingResult, DARWIN, texFile, 5, 0, 17, "\\cite{Darwin1888}.", true); + addCite(expectedCrossingResult, DARWIN, texFile, 7, 67, 84, "Nunc ultricies leo nec libero rhoncus, eu vehicula enim efficitur. \\cite{Darwin1888}", true); + addCite(expectedCrossingResult, DARWIN, texFile2, 4, 48, 65, "This is some content trying to cite a bib file: \\cite{Darwin1888}", true); + addCite(expectedCrossingResult, NEWTON, texFile2, 6, 48, 65, "This is some content trying to cite a bib file: \\cite{Newton1999}", true); + + assertEquals(expectedCrossingResult, crossingResult); } } @@ -164,22 +157,19 @@ void testDuplicateFiles() throws URISyntaxException, IOException { try (InputStreamReader originalReader = new InputStreamReader(originalStream, StandardCharsets.UTF_8)) { ParserResult result = new BibtexParser(importFormatPreferences, new DummyFileUpdateMonitor()).parse(originalReader); - TexParser texParser = new DefaultTexParser(result.getDatabase()); - TexParserResult texResult = texParser.parse(Arrays.asList(texFile, texFile)); - TexParserResult expectedResult = new TexParserResult(result.getDatabase()); - BibDatabase newDatabase = texResult.getGeneratedBibDatabase(); - - addCite(expectedResult, EINSTEIN, texFile, 4, 0, 19, "\\cite{Einstein1920}", 1); - addCite(expectedResult, EINSTEIN, texFile, 6, 14, 33, "Einstein said \\cite{Einstein1920} that lorem impsum, consectetur adipiscing elit.", 1); - - addCite(expectedResult, DARWIN, texFile, 5, 0, 17, "\\cite{Darwin1888}.", 1); - addCite(expectedResult, DARWIN, texFile, 7, 67, 84, "Nunc ultricies leo nec libero rhoncus, eu vehicula enim efficitur. \\cite{Darwin1888}", 1); - - assertEquals(result.getDatabase(), texResult.getMasterDatabase()); - assertTrue(texResult.getGeneratedBibDatabase().hasEntries()); - assertEquals(texResult.getFoundKeysInTex() + texResult.getCrossRefEntriesCount(), texResult.getResolvedKeysCount() + texResult.getUnresolvedKeysCount()); - assertEquals(expectedResult.getResolvedKeysCount() + expectedResult.getCrossRefEntriesCount(), newDatabase.getEntries().size()); - assertEquals(expectedResult, texResult); + + TexParserResult parserResult = new DefaultTexParser().parse(Arrays.asList(texFile, texFile)); + TexParserResult expectedParserResult = new TexParserResult(); + expectedParserResult.getFileList().addAll(Arrays.asList(texFile, texFile)); + + CrossingKeysResult crossingResult = new CrossingKeys(parserResult, result.getDatabase()).resolveKeys(); + CrossingKeysResult expectedCrossingResult = new CrossingKeysResult(expectedParserResult, result.getDatabase()); + addCite(expectedCrossingResult, EINSTEIN, texFile, 4, 0, 19, "\\cite{Einstein1920}", true); + addCite(expectedCrossingResult, EINSTEIN, texFile, 6, 14, 33, "Einstein said \\cite{Einstein1920} that lorem impsum, consectetur adipiscing elit.", true); + addCite(expectedCrossingResult, DARWIN, texFile, 5, 0, 17, "\\cite{Darwin1888}.", true); + addCite(expectedCrossingResult, DARWIN, texFile, 7, 67, 84, "Nunc ultricies leo nec libero rhoncus, eu vehicula enim efficitur. \\cite{Darwin1888}", true); + + assertEquals(expectedCrossingResult, crossingResult); } } @@ -192,36 +182,29 @@ void testUnknownKey() throws URISyntaxException, IOException { try (InputStreamReader originalReader = new InputStreamReader(originalStream, StandardCharsets.UTF_8)) { ParserResult result = new BibtexParser(importFormatPreferences, new DummyFileUpdateMonitor()).parse(originalReader); - TexParser texParser = new DefaultTexParser(result.getDatabase()); - TexParserResult texResult = texParser.parse(texFile); - TexParserResult expectedResult = new TexParserResult(result.getDatabase()); - BibDatabase newDatabase = texResult.getGeneratedBibDatabase(); - - addCite(expectedResult, EINSTEIN, texFile, 5, 48, 67, "This is some content trying to cite a bib file: \\cite{Einstein1920}", 1); - addCite(expectedResult, DARWIN, texFile, 4, 48, 65, "This is some content trying to cite a bib file: \\cite{Darwin1888}", 1); - addCite(expectedResult, UNKNOWN, texFile, 6, 48, 65, "This is some content trying to cite a bib file: \\cite{UnknownKey}", -1); - - assertEquals(result.getDatabase(), texResult.getMasterDatabase()); - assertTrue(texResult.getGeneratedBibDatabase().hasEntries()); - assertEquals(texResult.getFoundKeysInTex() + texResult.getCrossRefEntriesCount(), texResult.getResolvedKeysCount() + texResult.getUnresolvedKeysCount()); - assertEquals(expectedResult.getResolvedKeysCount() + expectedResult.getCrossRefEntriesCount(), newDatabase.getEntries().size()); - assertEquals(expectedResult, texResult); + + TexParserResult parserResult = new DefaultTexParser().parse(texFile); + TexParserResult expectedParserResult = new TexParserResult(); + expectedParserResult.getFileList().add(texFile); + + CrossingKeysResult crossingResult = new CrossingKeys(parserResult, result.getDatabase()).resolveKeys(); + CrossingKeysResult expectedCrossingResult = new CrossingKeysResult(expectedParserResult, result.getDatabase()); + addCite(expectedCrossingResult, EINSTEIN, texFile, 5, 48, 67, "This is some content trying to cite a bib file: \\cite{Einstein1920}", true); + addCite(expectedCrossingResult, DARWIN, texFile, 4, 48, 65, "This is some content trying to cite a bib file: \\cite{Darwin1888}", true); + addCite(expectedCrossingResult, UNKNOWN, texFile, 6, 48, 65, "This is some content trying to cite a bib file: \\cite{UnknownKey}", false); + + assertEquals(expectedCrossingResult, crossingResult); } } @Test void testFileNotFound() { - BibDatabase masterDatabase = new BibDatabase(); - TexParser texParser = new DefaultTexParser(masterDatabase); - TexParserResult texResult = texParser.parse(Paths.get("file_not_found.tex")); - TexParserResult expectedResult = new TexParserResult(masterDatabase); - BibDatabase newDatabase = texResult.getGeneratedBibDatabase(); - - assertEquals(masterDatabase, texResult.getMasterDatabase()); - assertFalse(texResult.getGeneratedBibDatabase().hasEntries()); - assertEquals(texResult.getFoundKeysInTex() + texResult.getCrossRefEntriesCount(), texResult.getResolvedKeysCount() + texResult.getUnresolvedKeysCount()); - assertEquals(expectedResult.getResolvedKeysCount() + expectedResult.getCrossRefEntriesCount(), newDatabase.getEntries().size()); - assertEquals(expectedResult, texResult); + TexParserResult parserResult = new DefaultTexParser().parse(Paths.get("file_not_found.tex")); + TexParserResult expectedParserResult = new TexParserResult(); + + expectedParserResult.getFileList().add(Paths.get("file_not_found.tex")); + + assertEquals(expectedParserResult, parserResult); } @Test @@ -231,12 +214,12 @@ void testDuplicateBibDatabaseConfiguration() throws URISyntaxException, IOExcept try (InputStreamReader originalReader = new InputStreamReader(originalStream, StandardCharsets.UTF_8)) { ParserResult result = new BibtexParser(importFormatPreferences, new DummyFileUpdateMonitor()).parse(originalReader); - TexParser texParser = new DefaultTexParser(result.getDatabase()); - TexParserResult texResult = texParser.parse(texFile); - BibDatabase newDatabase = texResult.getGeneratedBibDatabase(); - assertEquals(Optional.of("\"Maintained by \" # maintainer"), newDatabase.getPreamble()); - assertEquals(1, newDatabase.getStringCount()); + TexParserResult parserResult = new DefaultTexParser().parse(texFile); + CrossingKeysResult crossingResult = new CrossingKeys(parserResult, result.getDatabase()).resolveKeys(); + + assertEquals(Optional.of("\"Maintained by \" # maintainer"), crossingResult.getNewDatabase().getPreamble()); + assertEquals(1, crossingResult.getNewDatabase().getStringCount()); } } @@ -244,25 +227,26 @@ void testDuplicateBibDatabaseConfiguration() throws URISyntaxException, IOExcept void testNestedFiles() throws URISyntaxException, IOException { InputStream originalStream = TexParserTest.class.getResourceAsStream("origin.bib"); Path texFile = Paths.get(TexParserTest.class.getResource("nested.tex").toURI()); + Path texFile2 = Paths.get(TexParserTest.class.getResource("nested2.tex").toURI()); + Path texFile3 = Paths.get(TexParserTest.class.getResource("paper.tex").toURI()); try (InputStreamReader originalReader = new InputStreamReader(originalStream, StandardCharsets.UTF_8)) { ParserResult result = new BibtexParser(importFormatPreferences, new DummyFileUpdateMonitor()).parse(originalReader); - TexParser texParser = new DefaultTexParser(result.getDatabase()); - TexParserResult texResult = texParser.parse(texFile); - TexParserResult expectedResult = new TexParserResult(result.getDatabase(), 0, 2, 0); - BibDatabase newDatabase = texResult.getGeneratedBibDatabase(); - - addCite(expectedResult, EINSTEIN, texFile.getParent().resolve("paper.tex"), 4, 0, 19, "\\cite{Einstein1920}", 1); - addCite(expectedResult, EINSTEIN, texFile.getParent().resolve("paper.tex"), 6, 14, 33, "Einstein said \\cite{Einstein1920} that lorem impsum, consectetur adipiscing elit.", 1); - - addCite(expectedResult, DARWIN, texFile.getParent().resolve("paper.tex"), 5, 0, 17, "\\cite{Darwin1888}.", 1); - addCite(expectedResult, DARWIN, texFile.getParent().resolve("paper.tex"), 7, 67, 84, "Nunc ultricies leo nec libero rhoncus, eu vehicula enim efficitur. \\cite{Darwin1888}", 1); - - assertEquals(result.getDatabase(), texResult.getMasterDatabase()); - assertTrue(texResult.getGeneratedBibDatabase().hasEntries()); - assertEquals(texResult.getFoundKeysInTex() + texResult.getCrossRefEntriesCount(), texResult.getResolvedKeysCount() + texResult.getUnresolvedKeysCount()); - assertEquals(expectedResult.getResolvedKeysCount() + expectedResult.getCrossRefEntriesCount(), newDatabase.getEntries().size()); - assertEquals(expectedResult, texResult); + + TexParserResult parserResult = new DefaultTexParser().parse(texFile); + TexParserResult expectedParserResult = new TexParserResult(); + expectedParserResult.getFileList().add(texFile); + expectedParserResult.getNestedFiles().addAll(Arrays.asList(texFile2, texFile3)); + + CrossingKeysResult crossingResult = new CrossingKeys(parserResult, result.getDatabase()).resolveKeys(); + CrossingKeysResult expectedCrossingResult = new CrossingKeysResult(expectedParserResult, result.getDatabase()); + + addCite(expectedCrossingResult, EINSTEIN, texFile.getParent().resolve("paper.tex"), 4, 0, 19, "\\cite{Einstein1920}", true); + addCite(expectedCrossingResult, EINSTEIN, texFile.getParent().resolve("paper.tex"), 6, 14, 33, "Einstein said \\cite{Einstein1920} that lorem impsum, consectetur adipiscing elit.", true); + addCite(expectedCrossingResult, DARWIN, texFile.getParent().resolve("paper.tex"), 5, 0, 17, "\\cite{Darwin1888}.", true); + addCite(expectedCrossingResult, DARWIN, texFile.getParent().resolve("paper.tex"), 7, 67, 84, "Nunc ultricies leo nec libero rhoncus, eu vehicula enim efficitur. \\cite{Darwin1888}", true); + + assertEquals(expectedCrossingResult, crossingResult); } } @@ -278,34 +262,33 @@ void testCrossRef() throws URISyntaxException, IOException { try (InputStreamReader originalReader = new InputStreamReader(originalStream, StandardCharsets.UTF_8)) { ParserResult result = new BibtexParser(importFormatPreferences, new DummyFileUpdateMonitor()).parse(originalReader); - TexParser texParser = new DefaultTexParser(result.getDatabase()); - TexParserResult texResult = texParser.parse(texFile); - TexParserResult expectedResult = new TexParserResult(result.getDatabase(), 0, 0, 1); - BibDatabase newDatabase = texResult.getGeneratedBibDatabase(); - - expectedResult.getUniqueKeys().put(EINSTEIN_A, new ArrayList<>()); - expectedResult.getUniqueKeys().get(EINSTEIN_A).add(new Citation(texFile, 4, 48, 68, "This is some content trying to cite a bib file: \\cite{Einstein1920a}")); - expectedResult.getGeneratedBibDatabase().insertEntry((BibEntry) expectedResult.getMasterDatabase().getEntryByKey(EINSTEIN_A).get().clone()); - expectedResult.getGeneratedBibDatabase().insertEntry((BibEntry) expectedResult.getMasterDatabase().getEntryByKey(EINSTEIN).get().clone()); - - expectedResult.getUniqueKeys().put(EINSTEIN_B, new ArrayList<>()); - expectedResult.getUniqueKeys().get(EINSTEIN_B).add(new Citation(texFile, 5, 48, 68, "This is some content trying to cite a bib file: \\cite{Einstein1920b}")); - expectedResult.getGeneratedBibDatabase().insertEntry((BibEntry) expectedResult.getMasterDatabase().getEntryByKey(EINSTEIN_B).get().clone()); - expectedResult.getUnresolvedKeys().add(EINSTEIN21); - - expectedResult.getUniqueKeys().put(EINSTEIN_C, new ArrayList<>()); - expectedResult.getUniqueKeys().get(EINSTEIN_C).add(new Citation(texFile, 6, 48, 68, "This is some content trying to cite a bib file: \\cite{Einstein1920c}")); - expectedResult.getGeneratedBibDatabase().insertEntry((BibEntry) expectedResult.getMasterDatabase().getEntryByKey(EINSTEIN_C).get().clone()); - - expectedResult.getUniqueKeys().put(UNRESOLVED, new ArrayList<>()); - expectedResult.getUniqueKeys().get(UNRESOLVED).add(new Citation(texFile, 7, 48, 68, "This is some content trying to cite a bib file: \\cite{UnresolvedKey}")); - expectedResult.getUnresolvedKeys().add(UNRESOLVED); - - assertEquals(result.getDatabase(), texResult.getMasterDatabase()); - assertTrue(texResult.getGeneratedBibDatabase().hasEntries()); - assertEquals(texResult.getFoundKeysInTex() + texResult.getCrossRefEntriesCount(), texResult.getResolvedKeysCount() + texResult.getUnresolvedKeysCount()); - assertEquals(expectedResult.getResolvedKeysCount() + expectedResult.getCrossRefEntriesCount(), newDatabase.getEntries().size()); - assertEquals(expectedResult, texResult); + + TexParserResult parserResult = new DefaultTexParser().parse(texFile); + TexParserResult expectedParserResult = new TexParserResult(); + expectedParserResult.getFileList().add(texFile); + + CrossingKeysResult crossingResult = new CrossingKeys(parserResult, result.getDatabase()).resolveKeys(); + CrossingKeysResult expectedCrossingResult = new CrossingKeysResult(expectedParserResult, result.getDatabase(), 0, 1); + + expectedParserResult.getCitations().put(EINSTEIN_A, new ArrayList<>()); + expectedParserResult.getCitations().get(EINSTEIN_A).add(new Citation(texFile, 4, 48, 68, "This is some content trying to cite a bib file: \\cite{Einstein1920a}")); + expectedCrossingResult.getNewDatabase().insertEntry((BibEntry) expectedCrossingResult.getMasterDatabase().getEntryByKey(EINSTEIN_A).get().clone()); + expectedCrossingResult.getNewDatabase().insertEntry((BibEntry) expectedCrossingResult.getMasterDatabase().getEntryByKey(EINSTEIN).get().clone()); + + expectedParserResult.getCitations().put(EINSTEIN_B, new ArrayList<>()); + expectedParserResult.getCitations().get(EINSTEIN_B).add(new Citation(texFile, 5, 48, 68, "This is some content trying to cite a bib file: \\cite{Einstein1920b}")); + expectedCrossingResult.getNewDatabase().insertEntry((BibEntry) expectedCrossingResult.getMasterDatabase().getEntryByKey(EINSTEIN_B).get().clone()); + expectedCrossingResult.getUnresolvedKeys().add(EINSTEIN21); + + expectedParserResult.getCitations().put(EINSTEIN_C, new ArrayList<>()); + expectedParserResult.getCitations().get(EINSTEIN_C).add(new Citation(texFile, 6, 48, 68, "This is some content trying to cite a bib file: \\cite{Einstein1920c}")); + expectedCrossingResult.getNewDatabase().insertEntry((BibEntry) expectedCrossingResult.getMasterDatabase().getEntryByKey(EINSTEIN_C).get().clone()); + + expectedParserResult.getCitations().put(UNRESOLVED, new ArrayList<>()); + expectedParserResult.getCitations().get(UNRESOLVED).add(new Citation(texFile, 7, 48, 68, "This is some content trying to cite a bib file: \\cite{UnresolvedKey}")); + expectedCrossingResult.getUnresolvedKeys().add(UNRESOLVED); + + assertEquals(expectedCrossingResult, crossingResult); } } } From 2dfbfdbf8778c2ad91e695c1218a6091037f007c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20M=C3=A9ndez?= Date: Mon, 10 Jun 2019 07:36:20 +0200 Subject: [PATCH 06/57] Update tests for an easier understand and fix general issues --- .../jabref/logic/texparser/CrossingKeys.java | 40 +- .../logic/texparser/DefaultTexParser.java | 64 ++-- .../org/jabref/model/texparser/Citation.java | 41 +- .../model/texparser/CrossingKeysResult.java | 124 +++++-- .../model/texparser/TexParserResult.java | 42 ++- .../logic/texparser/CrossingKeysTest.java | 170 +++++++++ .../logic/texparser/DefaultTexParserTest.java | 152 ++++++++ .../jabref/logic/texparser/TexParserTest.java | 351 +++++------------- .../org/jabref/logic/texparser/config.bib | 26 -- .../org/jabref/logic/texparser/origin.bib | 39 -- 10 files changed, 623 insertions(+), 426 deletions(-) create mode 100644 src/test/java/org/jabref/logic/texparser/CrossingKeysTest.java create mode 100644 src/test/java/org/jabref/logic/texparser/DefaultTexParserTest.java delete mode 100644 src/test/resources/org/jabref/logic/texparser/config.bib delete mode 100644 src/test/resources/org/jabref/logic/texparser/origin.bib diff --git a/src/main/java/org/jabref/logic/texparser/CrossingKeys.java b/src/main/java/org/jabref/logic/texparser/CrossingKeys.java index 457c71acc01..a08f15c0908 100644 --- a/src/main/java/org/jabref/logic/texparser/CrossingKeys.java +++ b/src/main/java/org/jabref/logic/texparser/CrossingKeys.java @@ -9,7 +9,7 @@ import org.jabref.model.texparser.CrossingKeysResult; import org.jabref.model.texparser.TexParserResult; -class CrossingKeys { +public class CrossingKeys { private final CrossingKeysResult result; @@ -17,31 +17,29 @@ public CrossingKeys(TexParserResult texParserResult, BibDatabase masterDatabase) this.result = new CrossingKeysResult(texParserResult, masterDatabase); } + public CrossingKeysResult getResult() { + return result; + } + /** * Look for an equivalent BibTeX entry within the reference database for all keys inside of the TEX files. */ public CrossingKeysResult resolveKeys() { - Set keySet = result.getParserResult().getCitations().keySet(); + Set keySet = result.getCitationsKeySet(); for (String key : keySet) { - if (!result.getNewDatabase().getEntryByKey(key).isPresent()) { - Optional entry = result.getMasterDatabase().getEntryByKey(key); + if (!result.checkEntryNewDatabase(key)) { + Optional entry = result.getEntryMasterDatabase(key); if (entry.isPresent()) { - insertEntry(entry.get()); + result.insertEntry(entry.get()); resolveCrossReferences(entry.get()); } else { - result.getUnresolvedKeys().add(key); + result.addUnresolvedKey(key); } } } - // Copy database definitions. - if (result.getNewDatabase().hasEntries()) { - result.getNewDatabase().copyPreamble(result.getMasterDatabase()); - result.insertStrings(result.getMasterDatabase().getUsedStrings(result.getNewDatabase().getEntries())); - } - return result; } @@ -50,24 +48,16 @@ public CrossingKeysResult resolveKeys() { */ private void resolveCrossReferences(BibEntry entry) { entry.getField(FieldName.CROSSREF).ifPresent(crossRef -> { - if (!result.getNewDatabase().getEntryByKey(crossRef).isPresent()) { - Optional refEntry = result.getMasterDatabase().getEntryByKey(crossRef); + if (!result.checkEntryNewDatabase(crossRef)) { + Optional refEntry = result.getEntryMasterDatabase(crossRef); if (refEntry.isPresent()) { - insertEntry(refEntry.get()); - result.increaseCrossRefEntriesCounter(); + result.insertEntry(refEntry.get()); + result.increaseCrossRefsCount(); } else { - result.getUnresolvedKeys().add(crossRef); + result.addUnresolvedKey(crossRef); } } }); } - - /** - * Insert into the database a clone of the given entry. The cloned entry has a new unique ID. - */ - private void insertEntry(BibEntry entry) { - BibEntry clonedEntry = (BibEntry) entry.clone(); - result.getNewDatabase().insertEntry(clonedEntry); - } } diff --git a/src/main/java/org/jabref/logic/texparser/DefaultTexParser.java b/src/main/java/org/jabref/logic/texparser/DefaultTexParser.java index 97f6c4406f3..6b103d97243 100644 --- a/src/main/java/org/jabref/logic/texparser/DefaultTexParser.java +++ b/src/main/java/org/jabref/logic/texparser/DefaultTexParser.java @@ -11,7 +11,6 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.jabref.model.texparser.Citation; import org.jabref.model.texparser.TexParser; import org.jabref.model.texparser.TexParserResult; @@ -29,15 +28,21 @@ public class DefaultTexParser implements TexParser { * *

TODO: Add support for multicite commands. */ - private static final String[] CITE_COMMANDS = new String[] { + private static final String[] CITE_COMMANDS = { "[cC]ite(alt|alp|author|authorfull|date|num|p|t|text|title|url|year|yearpar)?", "([aA]|fnote|foot|footfull|full|no|[nN]ote|[pP]aren|[pP]note|[tT]ext|[sS]mart|super)cite", "footcitetext" }; - private static final String CITE_REGEX = String.format("\\\\(?:%s)\\*?(?:\\[(?:[^\\]]*)\\]){0,2}\\{(?[^\\}]*)\\}", - String.join("|", CITE_COMMANDS)); + private static final String CITE_GROUP = "key"; + private static final Pattern CITE_PATTERN = Pattern.compile( + String.format("\\\\(%s)\\*?(?:\\[(?:[^\\]]*)\\]){0,2}\\{(?<%s>[^\\}]*)\\}", + String.join("|", CITE_COMMANDS), CITE_GROUP)); - private static final String INCLUDE_REGEX = "\\\\(?:include|input)\\{(?[^\\}]*)\\}"; + private static final String INCLUDE_GROUP = "file"; + private static final Pattern INCLUDE_PATTERN = Pattern.compile( + String.format("\\\\(?:include|input)\\{(?<%s>[^\\}]*)\\}", INCLUDE_GROUP)); + + private static final String TEX_EXT = ".tex"; private final TexParserResult result; @@ -45,6 +50,10 @@ public DefaultTexParser() { this.result = new TexParserResult(); } + public TexParserResult getResult() { + return result; + } + @Override public TexParserResult parse(String citeString) { matchCitation(Paths.get("foo/bar"), 1, citeString); @@ -60,19 +69,13 @@ public TexParserResult parse(Path texFile) { public TexParserResult parse(List texFiles) { List referencedFiles = new ArrayList<>(); - if (result.getFileList().isEmpty()) { - result.getFileList().addAll(texFiles); - } else { - result.getNestedFiles().addAll(texFiles); - } - - for (int fileIndex = 0; fileIndex < texFiles.size(); fileIndex++) { - Path file = texFiles.get(fileIndex); + result.addFiles(texFiles); + for (Path file : texFiles) { try (LineNumberReader lnr = new LineNumberReader(Files.newBufferedReader(file))) { for (String line = lnr.readLine(); line != null; line = lnr.readLine()) { - if (line.startsWith("%")) { - // Skip comment lines. + if (line.isEmpty() || line.charAt(0) == '%') { + // Skip comments and blank lines. continue; } @@ -93,24 +96,16 @@ public TexParserResult parse(List texFiles) { } /** - * Find cites along a specific line and add them to a map. + * Find cites along a specific line and store them. */ private void matchCitation(Path file, int lineNumber, String line) { - Matcher citeMatch = Pattern.compile(CITE_REGEX).matcher(line); + Matcher citeMatch = CITE_PATTERN.matcher(line); while (citeMatch.find()) { - String[] keys = citeMatch.group("key").split(","); + String[] keys = citeMatch.group(CITE_GROUP).split(","); for (String key : keys) { - Citation citation = new Citation(file, lineNumber, citeMatch.start(), citeMatch.end(), line); - - if (!result.getCitations().containsKey(key)) { - result.getCitations().put(key, new ArrayList<>()); - } - - if (!result.getCitations().get(key).contains(citation)) { - result.getCitations().get(key).add(citation); - } + result.addKey(key, file, lineNumber, citeMatch.start(), citeMatch.end(), line); } } } @@ -119,19 +114,20 @@ private void matchCitation(Path file, int lineNumber, String line) { * Find inputs and includes along a specific line and store them for parsing later. */ private void matchNestedFile(Path file, List texFiles, List referencedFiles, String line) { - Matcher includeMatch = Pattern.compile(INCLUDE_REGEX).matcher(line); + Matcher includeMatch = INCLUDE_PATTERN.matcher(line); + StringBuilder include; while (includeMatch.find()) { - String include = includeMatch.group("file"); + include = new StringBuilder(includeMatch.group(INCLUDE_GROUP)); - if (!include.endsWith(".tex")) { - include += ".tex"; + if (!include.toString().endsWith(TEX_EXT)) { + include.append(TEX_EXT); } Path folder = file.getParent(); - Path inputFile = (folder != null) - ? folder.resolve(include) - : Paths.get(include); + Path inputFile = (folder == null) + ? Paths.get(include.toString()) + : folder.resolve(include.toString()); if (!texFiles.contains(inputFile)) { referencedFiles.add(inputFile); diff --git a/src/main/java/org/jabref/model/texparser/Citation.java b/src/main/java/org/jabref/model/texparser/Citation.java index 003c0704e80..9905b4f9174 100644 --- a/src/main/java/org/jabref/model/texparser/Citation.java +++ b/src/main/java/org/jabref/model/texparser/Citation.java @@ -2,6 +2,7 @@ import java.nio.file.Path; import java.util.Objects; +import java.util.StringJoiner; public class Citation { @@ -17,6 +18,18 @@ public class Citation { private final String lineText; public Citation(Path path, int line, int colStart, int colEnd, String lineText) { + if (path == null) { + throw new IllegalArgumentException("Path cannot be null."); + } + + if (line <= 0) { + throw new IllegalArgumentException("Line has to be greater than 0."); + } + + if (colStart < 0 || colEnd > lineText.length()) { + throw new IllegalArgumentException("Citation has to be between 0 and line length."); + } + this.path = path; this.line = line; this.colStart = colStart; @@ -44,10 +57,12 @@ public String getLineText() { return lineText; } + public int getContextWidth() { + return CONTEXT_WIDTH; + } + /** - * Get a fixed-width string that shows the context of a citation. - * - * @return String that contains a cite and the text that surrounds it along the same line. + * Get a fixed-width string that contains a cite and the text that surrounds it along the same line. */ public String getContext() { int center = (colStart + colEnd) / 2; @@ -63,7 +78,13 @@ public String getContext() { @Override public String toString() { - return String.format("%s (%d:%d-%d) \"%s\"", path, line, colStart, colEnd, getContext()); + return new StringJoiner(", ", getClass().getSimpleName() + "[", "]") + .add("path = " + path) + .add("line = " + line) + .add("colStart = " + colStart) + .add("colEnd = " + colEnd) + .add("lineText = " + lineText) + .toString(); } @Override @@ -76,13 +97,13 @@ public boolean equals(Object o) { return false; } - Citation citation = (Citation) o; + Citation that = (Citation) o; - return path.equals(citation.path) - && line == citation.line - && colStart == citation.colStart - && colEnd == citation.colEnd - && lineText.equals(citation.lineText); + return Objects.equals(path, that.path) + && Objects.equals(line, that.line) + && Objects.equals(colStart, that.colStart) + && Objects.equals(colEnd, that.colEnd) + && Objects.equals(lineText, that.lineText); } @Override diff --git a/src/main/java/org/jabref/model/texparser/CrossingKeysResult.java b/src/main/java/org/jabref/model/texparser/CrossingKeysResult.java index 4db2853ce22..1472cdad494 100644 --- a/src/main/java/org/jabref/model/texparser/CrossingKeysResult.java +++ b/src/main/java/org/jabref/model/texparser/CrossingKeysResult.java @@ -1,13 +1,16 @@ package org.jabref.model.texparser; import java.util.ArrayList; -import java.util.Collection; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.StringJoiner; import org.jabref.model.database.BibDatabase; -import org.jabref.model.entry.BibtexString; +import org.jabref.model.entry.BibEntry; public class CrossingKeysResult { @@ -15,23 +18,17 @@ public class CrossingKeysResult { private final BibDatabase masterDatabase; private final List unresolvedKeys; private final BibDatabase newDatabase; - private int insertedStrings; - private int crossRefEntriesCount; + private int crossRefsCount; - public CrossingKeysResult(TexParserResult texParserResult, BibDatabase masterDatabase, int insertedStrings, int crossRefEntriesCount) { + public CrossingKeysResult(TexParserResult texParserResult, BibDatabase masterDatabase) { this.texParserResult = texParserResult; this.masterDatabase = masterDatabase; this.unresolvedKeys = new ArrayList<>(); this.newDatabase = new BibDatabase(); - this.insertedStrings = insertedStrings; - this.crossRefEntriesCount = crossRefEntriesCount; + this.crossRefsCount = 0; } - public CrossingKeysResult(TexParserResult texParserResult, BibDatabase masterDatabase) { - this(texParserResult, masterDatabase, 0, 0); - } - - public TexParserResult getParserResult() { + public TexParserResult getTexParserResult() { return texParserResult; } @@ -47,33 +44,90 @@ public BibDatabase getNewDatabase() { return newDatabase; } - public int getInsertedStrings() { - return insertedStrings; + public int getCrossRefsCount() { + return crossRefsCount; } - public int getResolvedKeysCount() { - return newDatabase.getEntryCount() - crossRefEntriesCount; + /** + * Return the citations map from the TexParserResult object. + */ + public Map> getCitations() { + return texParserResult.getCitations(); } - public int getCrossRefEntriesCount() { - return crossRefEntriesCount; + /** + * Return a set of strings with the keys of the citations map from the TexParserResult object. + */ + public Set getCitationsKeySet() { + return texParserResult.getCitations().keySet(); } - public void increaseCrossRefEntriesCounter() { - crossRefEntriesCount++; + /** + * Return the master database in a set, for comparing two objects. + */ + public Set getMasterDatabaseSet() { + return new HashSet<>(masterDatabase.getEntries()); } - public void insertStrings(Collection usedStrings) { - for (BibtexString string : usedStrings) { - newDatabase.addString(string); - insertedStrings++; - } + /** + * Get if an entry with the given key is present in the master database. + */ + public Optional getEntryMasterDatabase(String key) { + return masterDatabase.getEntryByKey(key); + } + + /** + * Add an unresolved key to the list. + */ + public void addUnresolvedKey(String key) { + unresolvedKeys.add(key); + } + + /** + * Return the new database in a set, for comparing two objects. + */ + public Set getNewDatabaseSet() { + return new HashSet<>(newDatabase.getEntries()); + } + + /** + * Check if an entry with the given key is present in the new database. + */ + public boolean checkEntryNewDatabase(String key) { + return newDatabase.getEntryByKey(key).isPresent(); + } + + /** + * Add 1 to cross references counter. + */ + public void increaseCrossRefsCount() { + crossRefsCount++; + } + + /** + * Insert into the database a clone of an entry with the given key. The cloned entry has a new unique ID. + */ + public void insertEntry(String key) { + insertEntry(masterDatabase.getEntryByKey(key).get()); + } + + /** + * Insert into the database a clone of the given entry. The cloned entry has a new unique ID. + */ + public void insertEntry(BibEntry entry) { + BibEntry clonedEntry = (BibEntry) entry.clone(); + newDatabase.insertEntry(clonedEntry); } @Override public String toString() { - return String.format("%nCrossReferencesResult{texParserResult=%s // masterDatabase=%s // unresolvedKeys=%s // newDatabase=%s // insertedStrings=%s // crossRefEntriesCount=%s}%n", - texParserResult, masterDatabase, unresolvedKeys, newDatabase, insertedStrings, crossRefEntriesCount); + return new StringJoiner(", ", this.getClass().getSimpleName() + "[", "]") + .add("texParserResult = " + texParserResult) + .add("masterDatabase = " + masterDatabase) + .add("unresolvedKeys = " + unresolvedKeys) + .add("newDatabase = " + newDatabase) + .add("crossRefsCount = " + crossRefsCount) + .toString(); } @Override @@ -86,18 +140,18 @@ public boolean equals(Object o) { return false; } - CrossingKeysResult result = (CrossingKeysResult) o; + CrossingKeysResult that = (CrossingKeysResult) o; - return texParserResult.equals(result.texParserResult) - && masterDatabase.equals(result.masterDatabase) - && unresolvedKeys.equals(result.unresolvedKeys) - && (new HashSet<>(newDatabase.getEntries())).equals(new HashSet<>(result.newDatabase.getEntries())) - && insertedStrings == result.insertedStrings - && crossRefEntriesCount == result.crossRefEntriesCount; + return Objects.equals(texParserResult, that.texParserResult) + && Objects.equals(getMasterDatabaseSet(), that.getMasterDatabaseSet()) + && Objects.equals(masterDatabase, that.masterDatabase) + && Objects.equals(unresolvedKeys, that.unresolvedKeys) + && Objects.equals(getNewDatabaseSet(), that.getNewDatabaseSet()) + && Objects.equals(crossRefsCount, that.crossRefsCount); } @Override public int hashCode() { - return Objects.hash(masterDatabase, unresolvedKeys, newDatabase, insertedStrings, crossRefEntriesCount); + return Objects.hash(texParserResult, masterDatabase, unresolvedKeys, newDatabase, crossRefsCount); } } diff --git a/src/main/java/org/jabref/model/texparser/TexParserResult.java b/src/main/java/org/jabref/model/texparser/TexParserResult.java index 4c8549ecf4c..923b24c4e3a 100644 --- a/src/main/java/org/jabref/model/texparser/TexParserResult.java +++ b/src/main/java/org/jabref/model/texparser/TexParserResult.java @@ -3,15 +3,18 @@ import java.nio.file.Path; import java.util.ArrayList; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Set; +import java.util.StringJoiner; public class TexParserResult { private final List fileList; private final List nestedFiles; - private final Map> citations; + private final Map> citations; public TexParserResult() { this.fileList = new ArrayList<>(); @@ -27,14 +30,37 @@ public List getNestedFiles() { return nestedFiles; } - public Map> getCitations() { + public Map> getCitations() { return citations; } + /** + * Add a list of files to fileList or nestedFiles, depending on whether this is the first list. + */ + public void addFiles(List texFiles) { + if (fileList.isEmpty()) { + fileList.addAll(texFiles); + } else { + nestedFiles.addAll(texFiles); + } + } + + /** + * Add a citation to the citations map. It puts a new key into the map if does not exist yet. + */ + public void addKey(String key, Path file, int lineNumber, int start, int end, String line) { + Citation citation = new Citation(file, lineNumber, start, end, line); + citations.computeIfAbsent(key, value -> new HashSet<>()) + .add(citation); + } + @Override public String toString() { - return String.format("%nTexParserResult{fileList=%s // nestedFiles=%s // citations=%s}%n", - fileList, nestedFiles, citations); + return new StringJoiner(", ", getClass().getSimpleName() + "[", "]") + .add("fileList = " + fileList) + .add("nestedFiles = " + nestedFiles) + .add("citations = " + citations) + .toString(); } @Override @@ -47,11 +73,11 @@ public boolean equals(Object o) { return false; } - TexParserResult result = (TexParserResult) o; + TexParserResult that = (TexParserResult) o; - return fileList.equals(result.fileList) - && nestedFiles.equals(result.nestedFiles) - && citations.equals(result.citations); + return Objects.equals(fileList, that.fileList) + && Objects.equals(nestedFiles, that.nestedFiles) + && Objects.equals(citations, that.citations); } @Override diff --git a/src/test/java/org/jabref/logic/texparser/CrossingKeysTest.java b/src/test/java/org/jabref/logic/texparser/CrossingKeysTest.java new file mode 100644 index 00000000000..56dbd6864fc --- /dev/null +++ b/src/test/java/org/jabref/logic/texparser/CrossingKeysTest.java @@ -0,0 +1,170 @@ +package org.jabref.logic.texparser; + +import java.net.URISyntaxException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; + +import org.jabref.model.database.BibDatabase; +import org.jabref.model.entry.BibEntry; +import org.jabref.model.entry.BibtexEntryTypes; +import org.jabref.model.texparser.CrossingKeysResult; +import org.jabref.model.texparser.TexParserResult; + +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class CrossingKeysTest { + private final static String DARWIN = "Darwin1888"; + private final static String EINSTEIN = "Einstein1920"; + private final static String NEWTON = "Newton1999"; + private final static String EINSTEIN_A = "Einstein1920a"; + private final static String EINSTEIN_B = "Einstein1920b"; + private final static String EINSTEIN_C = "Einstein1920c"; + private final static String EINSTEIN_21 = "Einstein1921"; + private final static String UNRESOLVED = "UnresolvedKey"; + private final static String UNKNOWN = "UnknownKey"; + + private static BibDatabase database; + + @BeforeEach + private void setUp() { + database = new BibDatabase(); + + BibEntry darwin = new BibEntry(BibtexEntryTypes.BOOK) + .withField("bibtexkey", DARWIN) + .withField("title", "The descent of man, and selection in relation to sex") + .withField("publisher", "J. Murray") + .withField("year", "1888") + .withField("author", "Darwin, Charles"); + database.insertEntry(darwin); + + BibEntry einstein = new BibEntry(BibtexEntryTypes.BOOK) + .withField("bibtexkey", EINSTEIN) + .withField("title", "Relativity: The special and general theory") + .withField("publisher", "Penguin") + .withField("year", "1920") + .withField("author", "Einstein, Albert"); + database.insertEntry(einstein); + + BibEntry newton = new BibEntry(BibtexEntryTypes.BOOK) + .withField("bibtexkey", NEWTON) + .withField("title", "The Principia: mathematical principles of natural philosophy") + .withField("publisher", "Univ of California Press") + .withField("year", "1999") + .withField("author", "Newton, Isaac"); + database.insertEntry(newton); + + BibEntry einsteinA = new BibEntry(BibtexEntryTypes.BOOK) + .withField("bibtexkey", EINSTEIN_A) + .withField("crossref", "Einstein1920") + .withField("pages", "22--23"); + database.insertEntry(einsteinA); + + BibEntry einsteinB = new BibEntry(BibtexEntryTypes.BOOK) + .withField("bibtexkey", EINSTEIN_B) + .withField("crossref", "Einstein1921") + .withField("pages", "22--23"); + database.insertEntry(einsteinB); + + BibEntry einsteinC = new BibEntry(BibtexEntryTypes.BOOK) + .withField("bibtexkey", EINSTEIN_C) + .withField("crossref", "Einstein1920") + .withField("pages", "25--33"); + database.insertEntry(einsteinC); + } + + @Test + public void testSingleFile() throws URISyntaxException { + Path texFile = Paths.get(CrossingKeysTest.class.getResource("paper.tex").toURI()); + TexParserResult parserResult = new DefaultTexParser().parse(texFile); + + CrossingKeysResult crossingResult = new CrossingKeys(parserResult, database).resolveKeys(); + CrossingKeysResult expectedCrossingResult = new CrossingKeysResult(parserResult, database); + + expectedCrossingResult.insertEntry(DARWIN); + expectedCrossingResult.insertEntry(EINSTEIN); + + assertEquals(expectedCrossingResult, crossingResult); + } + + @Test + public void testTwoFiles() throws URISyntaxException { + Path texFile = Paths.get(CrossingKeysTest.class.getResource("paper.tex").toURI()); + Path texFile2 = Paths.get(CrossingKeysTest.class.getResource("paper2.tex").toURI()); + TexParserResult parserResult = new DefaultTexParser().parse(Arrays.asList(texFile, texFile2)); + + CrossingKeysResult crossingResult = new CrossingKeys(parserResult, database).resolveKeys(); + CrossingKeysResult expectedCrossingResult = new CrossingKeysResult(parserResult, database); + + expectedCrossingResult.insertEntry(DARWIN); + expectedCrossingResult.insertEntry(EINSTEIN); + expectedCrossingResult.insertEntry(NEWTON); + + assertEquals(expectedCrossingResult, crossingResult); + } + + @Test + public void testDuplicateFiles() throws URISyntaxException { + Path texFile = Paths.get(CrossingKeysTest.class.getResource("paper.tex").toURI()); + TexParserResult parserResult = new DefaultTexParser().parse(texFile); + + CrossingKeysResult crossingResult = new CrossingKeys(parserResult, database).resolveKeys(); + CrossingKeysResult expectedCrossingResult = new CrossingKeysResult(parserResult, database); + + expectedCrossingResult.insertEntry(DARWIN); + expectedCrossingResult.insertEntry(EINSTEIN); + + assertEquals(expectedCrossingResult, crossingResult); + } + + @Test + public void testUnknownKey() throws URISyntaxException { + Path texFile = Paths.get(CrossingKeysTest.class.getResource("unknown_key.tex").toURI()); + TexParserResult parserResult = new DefaultTexParser().parse(texFile); + + CrossingKeysResult crossingResult = new CrossingKeys(parserResult, database).resolveKeys(); + CrossingKeysResult expectedCrossingResult = new CrossingKeysResult(parserResult, database); + + expectedCrossingResult.insertEntry(DARWIN); + expectedCrossingResult.insertEntry(EINSTEIN); + expectedCrossingResult.addUnresolvedKey(UNKNOWN); + + assertEquals(expectedCrossingResult, crossingResult); + } + + @Test + public void testNestedFiles() throws URISyntaxException { + Path texFile = Paths.get(CrossingKeysTest.class.getResource("nested.tex").toURI()); + TexParserResult parserResult = new DefaultTexParser().parse(texFile); + + CrossingKeysResult crossingResult = new CrossingKeys(parserResult, database).resolveKeys(); + CrossingKeysResult expectedCrossingResult = new CrossingKeysResult(parserResult, database); + + expectedCrossingResult.insertEntry(DARWIN); + expectedCrossingResult.insertEntry(EINSTEIN); + + assertEquals(expectedCrossingResult, crossingResult); + } + + @Test + public void testCrossRef() throws URISyntaxException { + Path texFile = Paths.get(CrossingKeysTest.class.getResource("crossref.tex").toURI()); + TexParserResult parserResult = new DefaultTexParser().parse(texFile); + + CrossingKeysResult crossingResult = new CrossingKeys(parserResult, database).resolveKeys(); + CrossingKeysResult expectedCrossingResult = new CrossingKeysResult(parserResult, database); + + expectedCrossingResult.insertEntry(EINSTEIN); + expectedCrossingResult.insertEntry(EINSTEIN_A); + expectedCrossingResult.insertEntry(EINSTEIN_B); + expectedCrossingResult.insertEntry(EINSTEIN_C); + expectedCrossingResult.addUnresolvedKey(EINSTEIN_21); + expectedCrossingResult.addUnresolvedKey(UNRESOLVED); + expectedCrossingResult.increaseCrossRefsCount(); + + assertEquals(expectedCrossingResult, crossingResult); + } +} diff --git a/src/test/java/org/jabref/logic/texparser/DefaultTexParserTest.java b/src/test/java/org/jabref/logic/texparser/DefaultTexParserTest.java new file mode 100644 index 00000000000..23f5c1ee7de --- /dev/null +++ b/src/test/java/org/jabref/logic/texparser/DefaultTexParserTest.java @@ -0,0 +1,152 @@ +package org.jabref.logic.texparser; + +import java.net.URISyntaxException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; + +import org.jabref.model.texparser.TexParserResult; + +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +public class DefaultTexParserTest { + private final static String DARWIN = "Darwin1888"; + private final static String EINSTEIN = "Einstein1920"; + private final static String NEWTON = "Newton1999"; + private final static String EINSTEIN_A = "Einstein1920a"; + private final static String EINSTEIN_B = "Einstein1920b"; + private final static String EINSTEIN_C = "Einstein1920c"; + private final static String EINSTEIN_21 = "Einstein1921"; + private final static String UNRESOLVED = "UnresolvedKey"; + private final static String UNKNOWN = "UnknownKey"; + + private void testMatchCite(String key, String citeString) { + TexParserResult texParserResult = new DefaultTexParser().parse(citeString); + TexParserResult expectedParserResult = new TexParserResult(); + + expectedParserResult.addKey(key, Paths.get("foo/bar"), 1, 0, citeString.length(), citeString); + + assertEquals(expectedParserResult, texParserResult); + } + + private void testNonMatchCite(String citeString) { + TexParserResult texParserResult = new DefaultTexParser().parse(citeString); + TexParserResult expectedParserResult = new TexParserResult(); + + assertEquals(expectedParserResult, texParserResult); + } + + @Test + public void testCiteCommands() { + testMatchCite(UNRESOLVED, "\\cite[pre][post]{UnresolvedKey}"); + testMatchCite(UNRESOLVED, "\\cite*{UnresolvedKey}"); + testMatchCite(UNRESOLVED, "\\parencite[post]{UnresolvedKey}"); + testMatchCite(UNRESOLVED, "\\cite[pre][post]{UnresolvedKey}"); + testMatchCite(EINSTEIN_C, "\\citep{Einstein1920c}"); + + testNonMatchCite("\\citet21312{123U123n123resolvedKey}"); + testNonMatchCite("\\1cite[pr234e][post]{UnresolvedKey}"); + testNonMatchCite("\\citep55{5}UnresolvedKey}"); + testNonMatchCite("\\cit2et{UnresolvedKey}"); + } + + @Test + public void testSingleFile() throws URISyntaxException { + Path texFile = Paths.get(DefaultTexParserTest.class.getResource("paper.tex").toURI()); + + TexParserResult parserResult = new DefaultTexParser().parse(texFile); + TexParserResult expectedParserResult = new TexParserResult(); + + expectedParserResult.getFileList().add(texFile); + expectedParserResult.addKey(EINSTEIN, texFile, 4, 0, 19, "\\cite{Einstein1920}"); + expectedParserResult.addKey(DARWIN, texFile, 5, 0, 17, "\\cite{Darwin1888}."); + expectedParserResult.addKey(EINSTEIN, texFile, 6, 14, 33, "Einstein said \\cite{Einstein1920} that lorem impsum, consectetur adipiscing elit."); + expectedParserResult.addKey(DARWIN, texFile, 7, 67, 84, "Nunc ultricies leo nec libero rhoncus, eu vehicula enim efficitur. \\cite{Darwin1888}"); + + assertEquals(expectedParserResult, parserResult); + } + + @Test + public void testTwoFiles() throws URISyntaxException { + Path texFile = Paths.get(DefaultTexParserTest.class.getResource("paper.tex").toURI()); + Path texFile2 = Paths.get(DefaultTexParserTest.class.getResource("paper2.tex").toURI()); + + TexParserResult parserResult = new DefaultTexParser().parse(Arrays.asList(texFile, texFile2)); + TexParserResult expectedParserResult = new TexParserResult(); + + expectedParserResult.getFileList().addAll(Arrays.asList(texFile, texFile2)); + expectedParserResult.addKey(EINSTEIN, texFile, 4, 0, 19, "\\cite{Einstein1920}"); + expectedParserResult.addKey(DARWIN, texFile, 5, 0, 17, "\\cite{Darwin1888}."); + expectedParserResult.addKey(EINSTEIN, texFile, 6, 14, 33, "Einstein said \\cite{Einstein1920} that lorem impsum, consectetur adipiscing elit."); + expectedParserResult.addKey(DARWIN, texFile, 7, 67, 84, "Nunc ultricies leo nec libero rhoncus, eu vehicula enim efficitur. \\cite{Darwin1888}"); + expectedParserResult.addKey(DARWIN, texFile2, 4, 48, 65, "This is some content trying to cite a bib file: \\cite{Darwin1888}"); + expectedParserResult.addKey(EINSTEIN, texFile2, 5, 48, 67, "This is some content trying to cite a bib file: \\cite{Einstein1920}"); + expectedParserResult.addKey(NEWTON, texFile2, 6, 48, 65, "This is some content trying to cite a bib file: \\cite{Newton1999}"); + + assertEquals(expectedParserResult, parserResult); + } + + @Test + public void testDuplicateFiles() throws URISyntaxException { + Path texFile = Paths.get(DefaultTexParserTest.class.getResource("paper.tex").toURI()); + + TexParserResult parserResult = new DefaultTexParser().parse(Arrays.asList(texFile, texFile)); + TexParserResult expectedParserResult = new TexParserResult(); + + expectedParserResult.getFileList().addAll(Arrays.asList(texFile, texFile)); + expectedParserResult.addKey(EINSTEIN, texFile, 4, 0, 19, "\\cite{Einstein1920}"); + expectedParserResult.addKey(DARWIN, texFile, 5, 0, 17, "\\cite{Darwin1888}."); + expectedParserResult.addKey(EINSTEIN, texFile, 6, 14, 33, "Einstein said \\cite{Einstein1920} that lorem impsum, consectetur adipiscing elit."); + expectedParserResult.addKey(DARWIN, texFile, 7, 67, 84, "Nunc ultricies leo nec libero rhoncus, eu vehicula enim efficitur. \\cite{Darwin1888}"); + + assertEquals(expectedParserResult, parserResult); + } + + @Test + public void testUnknownKey() throws URISyntaxException { + Path texFile = Paths.get(DefaultTexParserTest.class.getResource("unknown_key.tex").toURI()); + + TexParserResult parserResult = new DefaultTexParser().parse(texFile); + TexParserResult expectedParserResult = new TexParserResult(); + + expectedParserResult.getFileList().add(texFile); + expectedParserResult.addKey(DARWIN, texFile, 4, 48, 65, "This is some content trying to cite a bib file: \\cite{Darwin1888}"); + expectedParserResult.addKey(EINSTEIN, texFile, 5, 48, 67, "This is some content trying to cite a bib file: \\cite{Einstein1920}"); + expectedParserResult.addKey(UNKNOWN, texFile, 6, 48, 65, "This is some content trying to cite a bib file: \\cite{UnknownKey}"); + + assertEquals(expectedParserResult, parserResult); + } + + @Test + public void testFileNotFound() { + Path texFile = Paths.get("file_not_found.tex"); + + TexParserResult parserResult = new DefaultTexParser().parse(texFile); + TexParserResult expectedParserResult = new TexParserResult(); + + expectedParserResult.getFileList().add(texFile); + + assertEquals(expectedParserResult, parserResult); + } + + @Test + public void testNestedFiles() throws URISyntaxException { + Path texFile = Paths.get(DefaultTexParserTest.class.getResource("nested.tex").toURI()); + Path texFile2 = Paths.get(DefaultTexParserTest.class.getResource("nested2.tex").toURI()); + Path texFile3 = Paths.get(DefaultTexParserTest.class.getResource("paper.tex").toURI()); + + TexParserResult parserResult = new DefaultTexParser().parse(texFile); + TexParserResult expectedParserResult = new TexParserResult(); + + expectedParserResult.getFileList().add(texFile); + expectedParserResult.getNestedFiles().addAll(Arrays.asList(texFile2, texFile3)); + expectedParserResult.addKey(EINSTEIN, texFile3, 4, 0, 19, "\\cite{Einstein1920}"); + expectedParserResult.addKey(DARWIN, texFile3, 5, 0, 17, "\\cite{Darwin1888}."); + expectedParserResult.addKey(EINSTEIN, texFile3, 6, 14, 33, "Einstein said \\cite{Einstein1920} that lorem impsum, consectetur adipiscing elit."); + expectedParserResult.addKey(DARWIN, texFile3, 7, 67, 84, "Nunc ultricies leo nec libero rhoncus, eu vehicula enim efficitur. \\cite{Darwin1888}"); + + assertEquals(expectedParserResult, parserResult); + } +} diff --git a/src/test/java/org/jabref/logic/texparser/TexParserTest.java b/src/test/java/org/jabref/logic/texparser/TexParserTest.java index f4544845d91..4437f6cc32b 100644 --- a/src/test/java/org/jabref/logic/texparser/TexParserTest.java +++ b/src/test/java/org/jabref/logic/texparser/TexParserTest.java @@ -1,294 +1,147 @@ package org.jabref.logic.texparser; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; import java.net.URISyntaxException; -import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.ArrayList; import java.util.Arrays; -import java.util.Optional; -import org.jabref.logic.importer.ImportFormatPreferences; -import org.jabref.logic.importer.ParserResult; -import org.jabref.logic.importer.fileformat.BibtexParser; +import org.jabref.model.database.BibDatabase; import org.jabref.model.entry.BibEntry; -import org.jabref.model.texparser.Citation; +import org.jabref.model.entry.BibtexEntryTypes; import org.jabref.model.texparser.CrossingKeysResult; import org.jabref.model.texparser.TexParserResult; -import org.jabref.model.util.DummyFileUpdateMonitor; -import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.mockito.Answers; import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.Mockito.mock; -class TexParserTest { - private final String DARWIN = "Darwin1888"; - private final String EINSTEIN = "Einstein1920"; - private final String UNRESOLVED = "UnresolvedKey"; - private ImportFormatPreferences importFormatPreferences; +public class TexParserTest { + private final static String DARWIN = "Darwin1888"; + private final static String EINSTEIN = "Einstein1920"; + private final static String NEWTON = "Newton1999"; + private final static String EINSTEIN_A = "Einstein1920a"; + private final static String EINSTEIN_B = "Einstein1920b"; + private final static String EINSTEIN_C = "Einstein1920c"; + private final static String EINSTEIN_21 = "Einstein1921"; + private final static String UNRESOLVED = "UnresolvedKey"; + private final static String UNKNOWN = "UnknownKey"; - @BeforeEach - void setUp() { - importFormatPreferences = mock(ImportFormatPreferences.class, Answers.RETURNS_DEEP_STUBS); - } - - @AfterEach - void tearDown() { - importFormatPreferences = null; - } - - private void testCite(String key, String citeString, boolean match) { - TexParserResult texParserResult = new DefaultTexParser().parse(citeString); - TexParserResult expectedResult = new TexParserResult(); - - if (match) { - expectedResult.getCitations().put(key, new ArrayList<>()); - expectedResult.getCitations().get(key).add( - new Citation(Paths.get("foo/bar"), 1, 0, citeString.length(), citeString)); - } - - assertEquals(expectedResult, texParserResult); - } - - private void testMatchCite(String citeString) { - testCite(UNRESOLVED, citeString, true); - } - - private void testNonMatchCite(String citeString) { - testCite(UNRESOLVED, citeString, false); - } - - @Test - void testCiteCommands() { - testMatchCite("\\cite[pre][post]{UnresolvedKey}"); - testMatchCite("\\cite*{UnresolvedKey}"); - testMatchCite("\\parencite[post]{UnresolvedKey}"); - testMatchCite("\\cite[pre][post]{UnresolvedKey}"); - testMatchCite("\\citep{UnresolvedKey}"); - - testNonMatchCite("\\citet21312{123U123n123resolvedKey}"); - testNonMatchCite("\\1cite[pr234e][post]{UnresolvedKey}"); - testNonMatchCite("\\citep55{5}UnresolvedKey}"); - testNonMatchCite("\\cit2et{UnresolvedKey}"); - } + private static BibDatabase database; + private static BibDatabase database2; - private void addCite(CrossingKeysResult expectedResult, String key, Path texFile, int line, int colStart, int colEnd, String lineText, boolean insert) { - Citation citation = new Citation(texFile, line, colStart, colEnd, lineText); - - if (!expectedResult.getParserResult().getCitations().containsKey(key)) { - expectedResult.getParserResult().getCitations().put(key, new ArrayList<>()); - } - - if (!expectedResult.getParserResult().getCitations().get(key).contains(citation)) { - expectedResult.getParserResult().getCitations().get(key).add(citation); - } - - if (insert) { - BibEntry clonedEntry = (BibEntry) expectedResult.getMasterDatabase().getEntryByKey(key).get().clone(); - expectedResult.getNewDatabase().insertEntry(clonedEntry); - } else { - expectedResult.getUnresolvedKeys().add(key); - } + @BeforeEach + private void setUp() { + database = new BibDatabase(); + database2 = new BibDatabase(); + + BibEntry darwin = new BibEntry(BibtexEntryTypes.BOOK) + .withField("bibtexkey", DARWIN) + .withField("title", "The descent of man, and selection in relation to sex") + .withField("publisher", "J. Murray") + .withField("year", "1888") + .withField("author", "Darwin, Charles"); + database.insertEntry(darwin); + + BibEntry einstein = new BibEntry(BibtexEntryTypes.BOOK) + .withField("bibtexkey", EINSTEIN) + .withField("title", "Relativity: The special and general theory") + .withField("publisher", "Penguin") + .withField("year", "1920") + .withField("author", "Einstein, Albert"); + database.insertEntry(einstein); + database2.insertEntry(einstein); + + BibEntry newton = new BibEntry(BibtexEntryTypes.BOOK) + .withField("bibtexkey", NEWTON) + .withField("title", "The Principia: mathematical principles of natural philosophy") + .withField("publisher", "Univ of California Press") + .withField("year", "1999") + .withField("author", "Newton, Isaac"); + database.insertEntry(newton); + + BibEntry einsteinA = new BibEntry(BibtexEntryTypes.BOOK) + .withField("bibtexkey", EINSTEIN_A) + .withField("crossref", "Einstein1920") + .withField("pages", "22--23"); + database.insertEntry(einsteinA); + + BibEntry einsteinB = new BibEntry(BibtexEntryTypes.BOOK) + .withField("bibtexkey", EINSTEIN_B) + .withField("crossref", "Einstein1921") + .withField("pages", "22--23"); + database.insertEntry(einsteinB); + + BibEntry einsteinC = new BibEntry(BibtexEntryTypes.BOOK) + .withField("bibtexkey", EINSTEIN_C) + .withField("crossref", "Einstein1920") + .withField("pages", "25--33"); + database.insertEntry(einsteinC); } @Test - void testSingleFile() throws URISyntaxException, IOException { - InputStream originalStream = TexParserTest.class.getResourceAsStream("origin.bib"); + public void testSameFileDifferentDatabases() throws URISyntaxException { Path texFile = Paths.get(TexParserTest.class.getResource("paper.tex").toURI()); - try (InputStreamReader originalReader = new InputStreamReader(originalStream, StandardCharsets.UTF_8)) { - ParserResult result = new BibtexParser(importFormatPreferences, new DummyFileUpdateMonitor()).parse(originalReader); - - TexParserResult parserResult = new DefaultTexParser().parse(texFile); - TexParserResult expectedParserResult = new TexParserResult(); - expectedParserResult.getFileList().add(texFile); - - CrossingKeysResult crossingResult = new CrossingKeys(parserResult, result.getDatabase()).resolveKeys(); - CrossingKeysResult expectedCrossingResult = new CrossingKeysResult(expectedParserResult, result.getDatabase()); - addCite(expectedCrossingResult, EINSTEIN, texFile, 4, 0, 19, "\\cite{Einstein1920}", true); - addCite(expectedCrossingResult, EINSTEIN, texFile, 6, 14, 33, "Einstein said \\cite{Einstein1920} that lorem impsum, consectetur adipiscing elit.", true); - addCite(expectedCrossingResult, DARWIN, texFile, 5, 0, 17, "\\cite{Darwin1888}.", true); - addCite(expectedCrossingResult, DARWIN, texFile, 7, 67, 84, "Nunc ultricies leo nec libero rhoncus, eu vehicula enim efficitur. \\cite{Darwin1888}", true); + TexParserResult parserResult = new DefaultTexParser().parse(texFile); + TexParserResult expectedParserResult = new TexParserResult(); - assertEquals(expectedCrossingResult, crossingResult); - } - } + expectedParserResult.getFileList().add(texFile); + expectedParserResult.addKey(EINSTEIN, texFile, 4, 0, 19, "\\cite{Einstein1920}"); + expectedParserResult.addKey(DARWIN, texFile, 5, 0, 17, "\\cite{Darwin1888}."); + expectedParserResult.addKey(EINSTEIN, texFile, 6, 14, 33, "Einstein said \\cite{Einstein1920} that lorem impsum, consectetur adipiscing elit."); + expectedParserResult.addKey(DARWIN, texFile, 7, 67, 84, "Nunc ultricies leo nec libero rhoncus, eu vehicula enim efficitur. \\cite{Darwin1888}"); - @Test - void testTwoFiles() throws URISyntaxException, IOException { - String NEWTON = "Newton1999"; + CrossingKeysResult crossingResult = new CrossingKeys(parserResult, database).resolveKeys(); + CrossingKeysResult expectedCrossingResult = new CrossingKeysResult(expectedParserResult, database); - InputStream originalStream = TexParserTest.class.getResourceAsStream("origin.bib"); - Path texFile = Paths.get(TexParserTest.class.getResource("paper.tex").toURI()); - Path texFile2 = Paths.get(TexParserTest.class.getResource("paper2.tex").toURI()); + expectedCrossingResult.insertEntry(DARWIN); + expectedCrossingResult.insertEntry(EINSTEIN); - try (InputStreamReader originalReader = new InputStreamReader(originalStream, StandardCharsets.UTF_8)) { - ParserResult result = new BibtexParser(importFormatPreferences, new DummyFileUpdateMonitor()).parse(originalReader); + assertEquals(expectedCrossingResult, crossingResult); - TexParserResult parserResult = new DefaultTexParser().parse(Arrays.asList(texFile, texFile2)); - TexParserResult expectedParserResult = new TexParserResult(); - expectedParserResult.getFileList().addAll(Arrays.asList(texFile, texFile2)); + CrossingKeysResult crossingResult2 = new CrossingKeys(parserResult, database2).resolveKeys(); + CrossingKeysResult expectedCrossingResult2 = new CrossingKeysResult(expectedParserResult, database2); - CrossingKeysResult crossingResult = new CrossingKeys(parserResult, result.getDatabase()).resolveKeys(); - CrossingKeysResult expectedCrossingResult = new CrossingKeysResult(expectedParserResult, result.getDatabase()); - addCite(expectedCrossingResult, EINSTEIN, texFile, 4, 0, 19, "\\cite{Einstein1920}", true); - addCite(expectedCrossingResult, EINSTEIN, texFile, 6, 14, 33, "Einstein said \\cite{Einstein1920} that lorem impsum, consectetur adipiscing elit.", true); - addCite(expectedCrossingResult, EINSTEIN, texFile2, 5, 48, 67, "This is some content trying to cite a bib file: \\cite{Einstein1920}", true); - addCite(expectedCrossingResult, DARWIN, texFile, 5, 0, 17, "\\cite{Darwin1888}.", true); - addCite(expectedCrossingResult, DARWIN, texFile, 7, 67, 84, "Nunc ultricies leo nec libero rhoncus, eu vehicula enim efficitur. \\cite{Darwin1888}", true); - addCite(expectedCrossingResult, DARWIN, texFile2, 4, 48, 65, "This is some content trying to cite a bib file: \\cite{Darwin1888}", true); - addCite(expectedCrossingResult, NEWTON, texFile2, 6, 48, 65, "This is some content trying to cite a bib file: \\cite{Newton1999}", true); + expectedCrossingResult2.insertEntry(EINSTEIN); + expectedCrossingResult2.addUnresolvedKey(DARWIN); - assertEquals(expectedCrossingResult, crossingResult); - } + assertEquals(expectedCrossingResult2, crossingResult2); } @Test - void testDuplicateFiles() throws URISyntaxException, IOException { - InputStream originalStream = TexParserTest.class.getResourceAsStream("origin.bib"); + public void testTwoFilesDifferentDatabases() throws URISyntaxException { Path texFile = Paths.get(TexParserTest.class.getResource("paper.tex").toURI()); + Path texFile2 = Paths.get(TexParserTest.class.getResource("paper2.tex").toURI()); - try (InputStreamReader originalReader = new InputStreamReader(originalStream, StandardCharsets.UTF_8)) { - ParserResult result = new BibtexParser(importFormatPreferences, new DummyFileUpdateMonitor()).parse(originalReader); - - TexParserResult parserResult = new DefaultTexParser().parse(Arrays.asList(texFile, texFile)); - TexParserResult expectedParserResult = new TexParserResult(); - expectedParserResult.getFileList().addAll(Arrays.asList(texFile, texFile)); - - CrossingKeysResult crossingResult = new CrossingKeys(parserResult, result.getDatabase()).resolveKeys(); - CrossingKeysResult expectedCrossingResult = new CrossingKeysResult(expectedParserResult, result.getDatabase()); - addCite(expectedCrossingResult, EINSTEIN, texFile, 4, 0, 19, "\\cite{Einstein1920}", true); - addCite(expectedCrossingResult, EINSTEIN, texFile, 6, 14, 33, "Einstein said \\cite{Einstein1920} that lorem impsum, consectetur adipiscing elit.", true); - addCite(expectedCrossingResult, DARWIN, texFile, 5, 0, 17, "\\cite{Darwin1888}.", true); - addCite(expectedCrossingResult, DARWIN, texFile, 7, 67, 84, "Nunc ultricies leo nec libero rhoncus, eu vehicula enim efficitur. \\cite{Darwin1888}", true); - - assertEquals(expectedCrossingResult, crossingResult); - } - } - - @Test - void testUnknownKey() throws URISyntaxException, IOException { - String UNKNOWN = "UnknownKey"; - - InputStream originalStream = TexParserTest.class.getResourceAsStream("origin.bib"); - Path texFile = Paths.get(TexParserTest.class.getResource("unknown_key.tex").toURI()); - - try (InputStreamReader originalReader = new InputStreamReader(originalStream, StandardCharsets.UTF_8)) { - ParserResult result = new BibtexParser(importFormatPreferences, new DummyFileUpdateMonitor()).parse(originalReader); - - TexParserResult parserResult = new DefaultTexParser().parse(texFile); - TexParserResult expectedParserResult = new TexParserResult(); - expectedParserResult.getFileList().add(texFile); - - CrossingKeysResult crossingResult = new CrossingKeys(parserResult, result.getDatabase()).resolveKeys(); - CrossingKeysResult expectedCrossingResult = new CrossingKeysResult(expectedParserResult, result.getDatabase()); - addCite(expectedCrossingResult, EINSTEIN, texFile, 5, 48, 67, "This is some content trying to cite a bib file: \\cite{Einstein1920}", true); - addCite(expectedCrossingResult, DARWIN, texFile, 4, 48, 65, "This is some content trying to cite a bib file: \\cite{Darwin1888}", true); - addCite(expectedCrossingResult, UNKNOWN, texFile, 6, 48, 65, "This is some content trying to cite a bib file: \\cite{UnknownKey}", false); - - assertEquals(expectedCrossingResult, crossingResult); - } - } - - @Test - void testFileNotFound() { - TexParserResult parserResult = new DefaultTexParser().parse(Paths.get("file_not_found.tex")); + TexParserResult parserResult = new DefaultTexParser().parse(Arrays.asList(texFile, texFile2)); TexParserResult expectedParserResult = new TexParserResult(); - expectedParserResult.getFileList().add(Paths.get("file_not_found.tex")); - - assertEquals(expectedParserResult, parserResult); - } - - @Test - void testDuplicateBibDatabaseConfiguration() throws URISyntaxException, IOException { - InputStream originalStream = TexParserTest.class.getResourceAsStream("config.bib"); - Path texFile = Paths.get(TexParserTest.class.getResource("paper.tex").toURI()); - - try (InputStreamReader originalReader = new InputStreamReader(originalStream, StandardCharsets.UTF_8)) { - ParserResult result = new BibtexParser(importFormatPreferences, new DummyFileUpdateMonitor()).parse(originalReader); - - TexParserResult parserResult = new DefaultTexParser().parse(texFile); - CrossingKeysResult crossingResult = new CrossingKeys(parserResult, result.getDatabase()).resolveKeys(); - - assertEquals(Optional.of("\"Maintained by \" # maintainer"), crossingResult.getNewDatabase().getPreamble()); - assertEquals(1, crossingResult.getNewDatabase().getStringCount()); - } - } - - @Test - void testNestedFiles() throws URISyntaxException, IOException { - InputStream originalStream = TexParserTest.class.getResourceAsStream("origin.bib"); - Path texFile = Paths.get(TexParserTest.class.getResource("nested.tex").toURI()); - Path texFile2 = Paths.get(TexParserTest.class.getResource("nested2.tex").toURI()); - Path texFile3 = Paths.get(TexParserTest.class.getResource("paper.tex").toURI()); - - try (InputStreamReader originalReader = new InputStreamReader(originalStream, StandardCharsets.UTF_8)) { - ParserResult result = new BibtexParser(importFormatPreferences, new DummyFileUpdateMonitor()).parse(originalReader); - - TexParserResult parserResult = new DefaultTexParser().parse(texFile); - TexParserResult expectedParserResult = new TexParserResult(); - expectedParserResult.getFileList().add(texFile); - expectedParserResult.getNestedFiles().addAll(Arrays.asList(texFile2, texFile3)); - - CrossingKeysResult crossingResult = new CrossingKeys(parserResult, result.getDatabase()).resolveKeys(); - CrossingKeysResult expectedCrossingResult = new CrossingKeysResult(expectedParserResult, result.getDatabase()); - - addCite(expectedCrossingResult, EINSTEIN, texFile.getParent().resolve("paper.tex"), 4, 0, 19, "\\cite{Einstein1920}", true); - addCite(expectedCrossingResult, EINSTEIN, texFile.getParent().resolve("paper.tex"), 6, 14, 33, "Einstein said \\cite{Einstein1920} that lorem impsum, consectetur adipiscing elit.", true); - addCite(expectedCrossingResult, DARWIN, texFile.getParent().resolve("paper.tex"), 5, 0, 17, "\\cite{Darwin1888}.", true); - addCite(expectedCrossingResult, DARWIN, texFile.getParent().resolve("paper.tex"), 7, 67, 84, "Nunc ultricies leo nec libero rhoncus, eu vehicula enim efficitur. \\cite{Darwin1888}", true); - - assertEquals(expectedCrossingResult, crossingResult); - } - } - - @Test - void testCrossRef() throws URISyntaxException, IOException { - String EINSTEIN_A = "Einstein1920a"; - String EINSTEIN_B = "Einstein1920b"; - String EINSTEIN21 = "Einstein1921"; - String EINSTEIN_C = "Einstein1920c"; - - InputStream originalStream = TexParserTest.class.getResourceAsStream("origin.bib"); - Path texFile = Paths.get(TexParserTest.class.getResource("crossref.tex").toURI()); - - try (InputStreamReader originalReader = new InputStreamReader(originalStream, StandardCharsets.UTF_8)) { - ParserResult result = new BibtexParser(importFormatPreferences, new DummyFileUpdateMonitor()).parse(originalReader); - - TexParserResult parserResult = new DefaultTexParser().parse(texFile); - TexParserResult expectedParserResult = new TexParserResult(); - expectedParserResult.getFileList().add(texFile); + expectedParserResult.getFileList().addAll(Arrays.asList(texFile, texFile2)); + expectedParserResult.addKey(EINSTEIN, texFile, 4, 0, 19, "\\cite{Einstein1920}"); + expectedParserResult.addKey(DARWIN, texFile, 5, 0, 17, "\\cite{Darwin1888}."); + expectedParserResult.addKey(EINSTEIN, texFile, 6, 14, 33, "Einstein said \\cite{Einstein1920} that lorem impsum, consectetur adipiscing elit."); + expectedParserResult.addKey(DARWIN, texFile, 7, 67, 84, "Nunc ultricies leo nec libero rhoncus, eu vehicula enim efficitur. \\cite{Darwin1888}"); + expectedParserResult.addKey(DARWIN, texFile2, 4, 48, 65, "This is some content trying to cite a bib file: \\cite{Darwin1888}"); + expectedParserResult.addKey(EINSTEIN, texFile2, 5, 48, 67, "This is some content trying to cite a bib file: \\cite{Einstein1920}"); + expectedParserResult.addKey(NEWTON, texFile2, 6, 48, 65, "This is some content trying to cite a bib file: \\cite{Newton1999}"); - CrossingKeysResult crossingResult = new CrossingKeys(parserResult, result.getDatabase()).resolveKeys(); - CrossingKeysResult expectedCrossingResult = new CrossingKeysResult(expectedParserResult, result.getDatabase(), 0, 1); + CrossingKeysResult crossingResult = new CrossingKeys(parserResult, database).resolveKeys(); + CrossingKeysResult expectedCrossingResult = new CrossingKeysResult(expectedParserResult, database); - expectedParserResult.getCitations().put(EINSTEIN_A, new ArrayList<>()); - expectedParserResult.getCitations().get(EINSTEIN_A).add(new Citation(texFile, 4, 48, 68, "This is some content trying to cite a bib file: \\cite{Einstein1920a}")); - expectedCrossingResult.getNewDatabase().insertEntry((BibEntry) expectedCrossingResult.getMasterDatabase().getEntryByKey(EINSTEIN_A).get().clone()); - expectedCrossingResult.getNewDatabase().insertEntry((BibEntry) expectedCrossingResult.getMasterDatabase().getEntryByKey(EINSTEIN).get().clone()); + expectedCrossingResult.insertEntry(DARWIN); + expectedCrossingResult.insertEntry(EINSTEIN); + expectedCrossingResult.insertEntry(NEWTON); - expectedParserResult.getCitations().put(EINSTEIN_B, new ArrayList<>()); - expectedParserResult.getCitations().get(EINSTEIN_B).add(new Citation(texFile, 5, 48, 68, "This is some content trying to cite a bib file: \\cite{Einstein1920b}")); - expectedCrossingResult.getNewDatabase().insertEntry((BibEntry) expectedCrossingResult.getMasterDatabase().getEntryByKey(EINSTEIN_B).get().clone()); - expectedCrossingResult.getUnresolvedKeys().add(EINSTEIN21); + assertEquals(expectedCrossingResult, crossingResult); - expectedParserResult.getCitations().put(EINSTEIN_C, new ArrayList<>()); - expectedParserResult.getCitations().get(EINSTEIN_C).add(new Citation(texFile, 6, 48, 68, "This is some content trying to cite a bib file: \\cite{Einstein1920c}")); - expectedCrossingResult.getNewDatabase().insertEntry((BibEntry) expectedCrossingResult.getMasterDatabase().getEntryByKey(EINSTEIN_C).get().clone()); + CrossingKeysResult crossingResult2 = new CrossingKeys(parserResult, database2).resolveKeys(); + CrossingKeysResult expectedCrossingResult2 = new CrossingKeysResult(expectedParserResult, database2); - expectedParserResult.getCitations().put(UNRESOLVED, new ArrayList<>()); - expectedParserResult.getCitations().get(UNRESOLVED).add(new Citation(texFile, 7, 48, 68, "This is some content trying to cite a bib file: \\cite{UnresolvedKey}")); - expectedCrossingResult.getUnresolvedKeys().add(UNRESOLVED); + expectedCrossingResult2.insertEntry(EINSTEIN); + expectedCrossingResult2.addUnresolvedKey(DARWIN); + expectedCrossingResult2.addUnresolvedKey(NEWTON); - assertEquals(expectedCrossingResult, crossingResult); - } + assertEquals(expectedCrossingResult2, crossingResult2); } } diff --git a/src/test/resources/org/jabref/logic/texparser/config.bib b/src/test/resources/org/jabref/logic/texparser/config.bib deleted file mode 100644 index 3e3cac593a9..00000000000 --- a/src/test/resources/org/jabref/logic/texparser/config.bib +++ /dev/null @@ -1,26 +0,0 @@ -@String {maintainer = "Stefan Kolb"} -@preamble {"Maintained by " # maintainer} -@String {einstein = "Einstein, Albert"} - -@Book{Newton1999, - title = {The Principia: mathematical principles of natural philosophy}, - publisher = {Univ of California Press}, - year = {1999}, - author = {Newton, Isaac} -} - -@Book{Darwin1888, - title = {The descent of man, and selection in relation to sex}, - publisher = {J. Murray}, - year = {1888}, - author = {Darwin, Charles} -} - -@Book{Einstein1920, - title = {Relativity: The special and general theory}, - publisher = {Penguin}, - year = {1920}, - author = einstein -} - -@Comment{jabref-meta: databaseType:bibtex;} diff --git a/src/test/resources/org/jabref/logic/texparser/origin.bib b/src/test/resources/org/jabref/logic/texparser/origin.bib deleted file mode 100644 index fbea84e2bc4..00000000000 --- a/src/test/resources/org/jabref/logic/texparser/origin.bib +++ /dev/null @@ -1,39 +0,0 @@ -% Encoding: UTF-8 - -@Book{Newton1999, - title = {The Principia: mathematical principles of natural philosophy}, - publisher = {Univ of California Press}, - year = {1999}, - author = {Newton, Isaac} -} - -@Book{Darwin1888, - title = {The descent of man, and selection in relation to sex}, - publisher = {J. Murray}, - year = {1888}, - author = {Darwin, Charles} -} - -@Book{Einstein1920, - title = {Relativity: The special and general theory}, - publisher = {Penguin}, - year = {1920}, - author = {Einstein, Albert} -} - -@InBook{Einstein1920a, - crossref = {Einstein1920}, - pages = {22--23} -} - -@InBook{Einstein1920b, - crossref = {Einstein1921}, - pages = {22--23} -} - -@InBook{Einstein1920c, - crossref = {Einstein1920}, - pages = {25--33} -} - -@Comment{jabref-meta: databaseType:bibtex;} From b46aeb4310193006e513755ce95c2da432ded2e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20M=C3=A9ndez?= Date: Mon, 10 Jun 2019 08:01:02 +0200 Subject: [PATCH 07/57] Add test for two citations in the same line --- .../logic/texparser/DefaultTexParserTest.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/test/java/org/jabref/logic/texparser/DefaultTexParserTest.java b/src/test/java/org/jabref/logic/texparser/DefaultTexParserTest.java index 23f5c1ee7de..961fd9339b0 100644 --- a/src/test/java/org/jabref/logic/texparser/DefaultTexParserTest.java +++ b/src/test/java/org/jabref/logic/texparser/DefaultTexParserTest.java @@ -52,6 +52,19 @@ public void testCiteCommands() { testNonMatchCite("\\cit2et{UnresolvedKey}"); } + @Test + public void testTwoCitationsSameLine() { + String citeString = "\\citep{Einstein1920c} and \\citep{Einstein1920a}"; + + TexParserResult texParserResult = new DefaultTexParser().parse(citeString); + TexParserResult expectedParserResult = new TexParserResult(); + + expectedParserResult.addKey(EINSTEIN_C, Paths.get("foo/bar"), 1, 0, 21, citeString); + expectedParserResult.addKey(EINSTEIN_A, Paths.get("foo/bar"), 1, 26, 47, citeString); + + assertEquals(expectedParserResult, texParserResult); + } + @Test public void testSingleFile() throws URISyntaxException { Path texFile = Paths.get(DefaultTexParserTest.class.getResource("paper.tex").toURI()); From 20372950a1459c5817a6c39aa612de0db29108cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20M=C3=A9ndez?= Date: Tue, 18 Jun 2019 10:01:47 +0200 Subject: [PATCH 08/57] Rename CrossingKeys to TexBibEntriesResolver and fix minor issues --- ...ngKeys.java => TexBibEntriesResolver.java} | 30 ++-- .../model/texparser/CrossingKeysResult.java | 157 ------------------ .../TexBibEntriesResolverResult.java | 125 ++++++++++++++ .../org/jabref/model/texparser/TexParser.java | 6 +- .../model/texparser/TexParserResult.java | 24 ++- ...st.java => TexBibEntriesResolverTest.java} | 72 ++++---- .../jabref/logic/texparser/TexParserTest.java | 32 ++-- 7 files changed, 208 insertions(+), 238 deletions(-) rename src/main/java/org/jabref/logic/texparser/{CrossingKeys.java => TexBibEntriesResolver.java} (53%) delete mode 100644 src/main/java/org/jabref/model/texparser/CrossingKeysResult.java create mode 100644 src/main/java/org/jabref/model/texparser/TexBibEntriesResolverResult.java rename src/test/java/org/jabref/logic/texparser/{CrossingKeysTest.java => TexBibEntriesResolverTest.java} (60%) diff --git a/src/main/java/org/jabref/logic/texparser/CrossingKeys.java b/src/main/java/org/jabref/logic/texparser/TexBibEntriesResolver.java similarity index 53% rename from src/main/java/org/jabref/logic/texparser/CrossingKeys.java rename to src/main/java/org/jabref/logic/texparser/TexBibEntriesResolver.java index a08f15c0908..d32bc9c7289 100644 --- a/src/main/java/org/jabref/logic/texparser/CrossingKeys.java +++ b/src/main/java/org/jabref/logic/texparser/TexBibEntriesResolver.java @@ -6,34 +6,32 @@ import org.jabref.model.database.BibDatabase; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.FieldName; -import org.jabref.model.texparser.CrossingKeysResult; +import org.jabref.model.texparser.TexBibEntriesResolverResult; import org.jabref.model.texparser.TexParserResult; -public class CrossingKeys { +public class TexBibEntriesResolver { - private final CrossingKeysResult result; + private final BibDatabase masterDatabase; - public CrossingKeys(TexParserResult texParserResult, BibDatabase masterDatabase) { - this.result = new CrossingKeysResult(texParserResult, masterDatabase); - } - - public CrossingKeysResult getResult() { - return result; + public TexBibEntriesResolver(BibDatabase masterDatabase) { + this.masterDatabase = masterDatabase; } /** - * Look for an equivalent BibTeX entry within the reference database for all keys inside of the TEX files. + * Look for BibTeX entries within the reference database for all keys inside of the TEX files. + * Insert these data in the list of new entries. */ - public CrossingKeysResult resolveKeys() { + public TexBibEntriesResolverResult resolveKeys(TexParserResult texParserResult) { + TexBibEntriesResolverResult result = new TexBibEntriesResolverResult(texParserResult); Set keySet = result.getCitationsKeySet(); for (String key : keySet) { if (!result.checkEntryNewDatabase(key)) { - Optional entry = result.getEntryMasterDatabase(key); + Optional entry = masterDatabase.getEntryByKey(key); if (entry.isPresent()) { result.insertEntry(entry.get()); - resolveCrossReferences(entry.get()); + resolveCrossReferences(result, entry.get()); } else { result.addUnresolvedKey(key); } @@ -44,12 +42,12 @@ public CrossingKeysResult resolveKeys() { } /** - * Find cross references for inserting into the new database. + * Find cross references for inserting into the list of new entries. */ - private void resolveCrossReferences(BibEntry entry) { + private void resolveCrossReferences(TexBibEntriesResolverResult result, BibEntry entry) { entry.getField(FieldName.CROSSREF).ifPresent(crossRef -> { if (!result.checkEntryNewDatabase(crossRef)) { - Optional refEntry = result.getEntryMasterDatabase(crossRef); + Optional refEntry = masterDatabase.getEntryByKey(crossRef); if (refEntry.isPresent()) { result.insertEntry(refEntry.get()); diff --git a/src/main/java/org/jabref/model/texparser/CrossingKeysResult.java b/src/main/java/org/jabref/model/texparser/CrossingKeysResult.java deleted file mode 100644 index 1472cdad494..00000000000 --- a/src/main/java/org/jabref/model/texparser/CrossingKeysResult.java +++ /dev/null @@ -1,157 +0,0 @@ -package org.jabref.model.texparser; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; -import java.util.StringJoiner; - -import org.jabref.model.database.BibDatabase; -import org.jabref.model.entry.BibEntry; - -public class CrossingKeysResult { - - private final TexParserResult texParserResult; - private final BibDatabase masterDatabase; - private final List unresolvedKeys; - private final BibDatabase newDatabase; - private int crossRefsCount; - - public CrossingKeysResult(TexParserResult texParserResult, BibDatabase masterDatabase) { - this.texParserResult = texParserResult; - this.masterDatabase = masterDatabase; - this.unresolvedKeys = new ArrayList<>(); - this.newDatabase = new BibDatabase(); - this.crossRefsCount = 0; - } - - public TexParserResult getTexParserResult() { - return texParserResult; - } - - public BibDatabase getMasterDatabase() { - return masterDatabase; - } - - public List getUnresolvedKeys() { - return unresolvedKeys; - } - - public BibDatabase getNewDatabase() { - return newDatabase; - } - - public int getCrossRefsCount() { - return crossRefsCount; - } - - /** - * Return the citations map from the TexParserResult object. - */ - public Map> getCitations() { - return texParserResult.getCitations(); - } - - /** - * Return a set of strings with the keys of the citations map from the TexParserResult object. - */ - public Set getCitationsKeySet() { - return texParserResult.getCitations().keySet(); - } - - /** - * Return the master database in a set, for comparing two objects. - */ - public Set getMasterDatabaseSet() { - return new HashSet<>(masterDatabase.getEntries()); - } - - /** - * Get if an entry with the given key is present in the master database. - */ - public Optional getEntryMasterDatabase(String key) { - return masterDatabase.getEntryByKey(key); - } - - /** - * Add an unresolved key to the list. - */ - public void addUnresolvedKey(String key) { - unresolvedKeys.add(key); - } - - /** - * Return the new database in a set, for comparing two objects. - */ - public Set getNewDatabaseSet() { - return new HashSet<>(newDatabase.getEntries()); - } - - /** - * Check if an entry with the given key is present in the new database. - */ - public boolean checkEntryNewDatabase(String key) { - return newDatabase.getEntryByKey(key).isPresent(); - } - - /** - * Add 1 to cross references counter. - */ - public void increaseCrossRefsCount() { - crossRefsCount++; - } - - /** - * Insert into the database a clone of an entry with the given key. The cloned entry has a new unique ID. - */ - public void insertEntry(String key) { - insertEntry(masterDatabase.getEntryByKey(key).get()); - } - - /** - * Insert into the database a clone of the given entry. The cloned entry has a new unique ID. - */ - public void insertEntry(BibEntry entry) { - BibEntry clonedEntry = (BibEntry) entry.clone(); - newDatabase.insertEntry(clonedEntry); - } - - @Override - public String toString() { - return new StringJoiner(", ", this.getClass().getSimpleName() + "[", "]") - .add("texParserResult = " + texParserResult) - .add("masterDatabase = " + masterDatabase) - .add("unresolvedKeys = " + unresolvedKeys) - .add("newDatabase = " + newDatabase) - .add("crossRefsCount = " + crossRefsCount) - .toString(); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - - if (o == null || getClass() != o.getClass()) { - return false; - } - - CrossingKeysResult that = (CrossingKeysResult) o; - - return Objects.equals(texParserResult, that.texParserResult) - && Objects.equals(getMasterDatabaseSet(), that.getMasterDatabaseSet()) - && Objects.equals(masterDatabase, that.masterDatabase) - && Objects.equals(unresolvedKeys, that.unresolvedKeys) - && Objects.equals(getNewDatabaseSet(), that.getNewDatabaseSet()) - && Objects.equals(crossRefsCount, that.crossRefsCount); - } - - @Override - public int hashCode() { - return Objects.hash(texParserResult, masterDatabase, unresolvedKeys, newDatabase, crossRefsCount); - } -} diff --git a/src/main/java/org/jabref/model/texparser/TexBibEntriesResolverResult.java b/src/main/java/org/jabref/model/texparser/TexBibEntriesResolverResult.java new file mode 100644 index 00000000000..cac45ad21cd --- /dev/null +++ b/src/main/java/org/jabref/model/texparser/TexBibEntriesResolverResult.java @@ -0,0 +1,125 @@ +package org.jabref.model.texparser; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import java.util.StringJoiner; + +import org.jabref.model.database.BibDatabase; +import org.jabref.model.entry.BibEntry; + +import com.google.common.collect.Multimap; + +public class TexBibEntriesResolverResult { + + private final TexParserResult texParserResult; + private final List unresolvedKeys; + private final List newEntries; + private int crossRefsCount; + + public TexBibEntriesResolverResult(TexParserResult texParserResult) { + this.texParserResult = texParserResult; + this.unresolvedKeys = new ArrayList<>(); + this.newEntries = new ArrayList<>(); + this.crossRefsCount = 0; + } + + public TexParserResult getTexParserResult() { + return texParserResult; + } + + public List getUnresolvedKeys() { + return unresolvedKeys; + } + + public List getNewEntries() { + return newEntries; + } + + public int getCrossRefsCount() { + return crossRefsCount; + } + + /** + * Return the citations multimap from the TexParserResult object. + */ + public Multimap getCitations() { + return texParserResult.getCitations(); + } + + /** + * Return a set of strings with the keys of the citations multimap from the TexParserResult object. + */ + public Set getCitationsKeySet() { + return texParserResult.getCitationsKeySet(); + } + + /** + * Add an unresolved key to the list. + */ + public void addUnresolvedKey(String key) { + unresolvedKeys.add(key); + } + + /** + * Check if an entry with the given key is present in the list of new entries. + */ + public boolean checkEntryNewDatabase(String key) { + return newEntries.stream().anyMatch(e -> e.getCiteKeyOptional().get().equals(key)); + } + + /** + * Add 1 to the cross references counter. + */ + public void increaseCrossRefsCount() { + crossRefsCount++; + } + + /** + * Insert into the list of new entries an entry with the given key. + */ + public void insertEntry(BibDatabase masterDatabase, String key) { + insertEntry(masterDatabase.getEntryByKey(key).get()); + } + + /** + * Insert into the list of new entries the given entry. + */ + public void insertEntry(BibEntry entry) { + newEntries.add(entry); + } + + @Override + public String toString() { + return new StringJoiner(", ", this.getClass().getSimpleName() + "[", "]") + .add("texParserResult = " + texParserResult) + .add("unresolvedKeys = " + unresolvedKeys) + .add("newEntries = " + newEntries) + .add("crossRefsCount = " + crossRefsCount) + .toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + + if (o == null || getClass() != o.getClass()) { + return false; + } + + TexBibEntriesResolverResult that = (TexBibEntriesResolverResult) o; + + return Objects.equals(texParserResult, that.texParserResult) + && Objects.equals(unresolvedKeys, that.unresolvedKeys) + && Objects.equals(newEntries, that.newEntries) + && Objects.equals(crossRefsCount, that.crossRefsCount); + } + + @Override + public int hashCode() { + return Objects.hash(texParserResult, unresolvedKeys, newEntries, crossRefsCount); + } +} diff --git a/src/main/java/org/jabref/model/texparser/TexParser.java b/src/main/java/org/jabref/model/texparser/TexParser.java index 5b7b72be308..45ed93b8c1e 100644 --- a/src/main/java/org/jabref/model/texparser/TexParser.java +++ b/src/main/java/org/jabref/model/texparser/TexParser.java @@ -17,8 +17,7 @@ public interface TexParser { * Parse a single TEX file. * * @param texFile Path to a TEX file - * @return a TexParserResult, which contains the generated BibDatabase and all data related to the bibliographic - * entries + * @return a TexParserResult, which contains all data related to the bibliographic entries */ TexParserResult parse(Path texFile); @@ -26,8 +25,7 @@ public interface TexParser { * Parse a list of TEX files. * * @param texFiles List of Path objects linked to a TEX file - * @return a TexParserResult, which contains the generated BibDatabase and all data related to the bibliographic - * entries + * @return a TexParserResult, which contains all data related to the bibliographic entries */ TexParserResult parse(List texFiles); } diff --git a/src/main/java/org/jabref/model/texparser/TexParserResult.java b/src/main/java/org/jabref/model/texparser/TexParserResult.java index 923b24c4e3a..67dd5b0b445 100644 --- a/src/main/java/org/jabref/model/texparser/TexParserResult.java +++ b/src/main/java/org/jabref/model/texparser/TexParserResult.java @@ -2,24 +2,24 @@ import java.nio.file.Path; import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; import java.util.List; -import java.util.Map; import java.util.Objects; import java.util.Set; import java.util.StringJoiner; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.Multimap; + public class TexParserResult { private final List fileList; private final List nestedFiles; - private final Map> citations; + private final Multimap citations; public TexParserResult() { this.fileList = new ArrayList<>(); this.nestedFiles = new ArrayList<>(); - this.citations = new HashMap<>(); + this.citations = HashMultimap.create(); } public List getFileList() { @@ -30,10 +30,17 @@ public List getNestedFiles() { return nestedFiles; } - public Map> getCitations() { + public Multimap getCitations() { return citations; } + /** + * Return a set of strings with the keys of the citations multimap. + */ + public Set getCitationsKeySet() { + return citations.keySet(); + } + /** * Add a list of files to fileList or nestedFiles, depending on whether this is the first list. */ @@ -46,12 +53,11 @@ public void addFiles(List texFiles) { } /** - * Add a citation to the citations map. It puts a new key into the map if does not exist yet. + * Add a citation to the citations multimap. */ public void addKey(String key, Path file, int lineNumber, int start, int end, String line) { Citation citation = new Citation(file, lineNumber, start, end, line); - citations.computeIfAbsent(key, value -> new HashSet<>()) - .add(citation); + citations.put(key, citation); } @Override diff --git a/src/test/java/org/jabref/logic/texparser/CrossingKeysTest.java b/src/test/java/org/jabref/logic/texparser/TexBibEntriesResolverTest.java similarity index 60% rename from src/test/java/org/jabref/logic/texparser/CrossingKeysTest.java rename to src/test/java/org/jabref/logic/texparser/TexBibEntriesResolverTest.java index 56dbd6864fc..3d33d7639bb 100644 --- a/src/test/java/org/jabref/logic/texparser/CrossingKeysTest.java +++ b/src/test/java/org/jabref/logic/texparser/TexBibEntriesResolverTest.java @@ -8,7 +8,7 @@ import org.jabref.model.database.BibDatabase; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.BibtexEntryTypes; -import org.jabref.model.texparser.CrossingKeysResult; +import org.jabref.model.texparser.TexBibEntriesResolverResult; import org.jabref.model.texparser.TexParserResult; import org.junit.jupiter.api.BeforeEach; @@ -16,7 +16,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; -public class CrossingKeysTest { +public class TexBibEntriesResolverTest { private final static String DARWIN = "Darwin1888"; private final static String EINSTEIN = "Einstein1920"; private final static String NEWTON = "Newton1999"; @@ -78,58 +78,58 @@ private void setUp() { @Test public void testSingleFile() throws URISyntaxException { - Path texFile = Paths.get(CrossingKeysTest.class.getResource("paper.tex").toURI()); + Path texFile = Paths.get(TexBibEntriesResolverTest.class.getResource("paper.tex").toURI()); TexParserResult parserResult = new DefaultTexParser().parse(texFile); - CrossingKeysResult crossingResult = new CrossingKeys(parserResult, database).resolveKeys(); - CrossingKeysResult expectedCrossingResult = new CrossingKeysResult(parserResult, database); + TexBibEntriesResolverResult crossingResult = new TexBibEntriesResolver(database).resolveKeys(parserResult); + TexBibEntriesResolverResult expectedCrossingResult = new TexBibEntriesResolverResult(parserResult); - expectedCrossingResult.insertEntry(DARWIN); - expectedCrossingResult.insertEntry(EINSTEIN); + expectedCrossingResult.insertEntry(database, DARWIN); + expectedCrossingResult.insertEntry(database, EINSTEIN); assertEquals(expectedCrossingResult, crossingResult); } @Test public void testTwoFiles() throws URISyntaxException { - Path texFile = Paths.get(CrossingKeysTest.class.getResource("paper.tex").toURI()); - Path texFile2 = Paths.get(CrossingKeysTest.class.getResource("paper2.tex").toURI()); + Path texFile = Paths.get(TexBibEntriesResolverTest.class.getResource("paper.tex").toURI()); + Path texFile2 = Paths.get(TexBibEntriesResolverTest.class.getResource("paper2.tex").toURI()); TexParserResult parserResult = new DefaultTexParser().parse(Arrays.asList(texFile, texFile2)); - CrossingKeysResult crossingResult = new CrossingKeys(parserResult, database).resolveKeys(); - CrossingKeysResult expectedCrossingResult = new CrossingKeysResult(parserResult, database); + TexBibEntriesResolverResult crossingResult = new TexBibEntriesResolver(database).resolveKeys(parserResult); + TexBibEntriesResolverResult expectedCrossingResult = new TexBibEntriesResolverResult(parserResult); - expectedCrossingResult.insertEntry(DARWIN); - expectedCrossingResult.insertEntry(EINSTEIN); - expectedCrossingResult.insertEntry(NEWTON); + expectedCrossingResult.insertEntry(database, DARWIN); + expectedCrossingResult.insertEntry(database, EINSTEIN); + expectedCrossingResult.insertEntry(database, NEWTON); assertEquals(expectedCrossingResult, crossingResult); } @Test public void testDuplicateFiles() throws URISyntaxException { - Path texFile = Paths.get(CrossingKeysTest.class.getResource("paper.tex").toURI()); + Path texFile = Paths.get(TexBibEntriesResolverTest.class.getResource("paper.tex").toURI()); TexParserResult parserResult = new DefaultTexParser().parse(texFile); - CrossingKeysResult crossingResult = new CrossingKeys(parserResult, database).resolveKeys(); - CrossingKeysResult expectedCrossingResult = new CrossingKeysResult(parserResult, database); + TexBibEntriesResolverResult crossingResult = new TexBibEntriesResolver(database).resolveKeys(parserResult); + TexBibEntriesResolverResult expectedCrossingResult = new TexBibEntriesResolverResult(parserResult); - expectedCrossingResult.insertEntry(DARWIN); - expectedCrossingResult.insertEntry(EINSTEIN); + expectedCrossingResult.insertEntry(database, DARWIN); + expectedCrossingResult.insertEntry(database, EINSTEIN); assertEquals(expectedCrossingResult, crossingResult); } @Test public void testUnknownKey() throws URISyntaxException { - Path texFile = Paths.get(CrossingKeysTest.class.getResource("unknown_key.tex").toURI()); + Path texFile = Paths.get(TexBibEntriesResolverTest.class.getResource("unknown_key.tex").toURI()); TexParserResult parserResult = new DefaultTexParser().parse(texFile); - CrossingKeysResult crossingResult = new CrossingKeys(parserResult, database).resolveKeys(); - CrossingKeysResult expectedCrossingResult = new CrossingKeysResult(parserResult, database); + TexBibEntriesResolverResult crossingResult = new TexBibEntriesResolver(database).resolveKeys(parserResult); + TexBibEntriesResolverResult expectedCrossingResult = new TexBibEntriesResolverResult(parserResult); - expectedCrossingResult.insertEntry(DARWIN); - expectedCrossingResult.insertEntry(EINSTEIN); + expectedCrossingResult.insertEntry(database, DARWIN); + expectedCrossingResult.insertEntry(database, EINSTEIN); expectedCrossingResult.addUnresolvedKey(UNKNOWN); assertEquals(expectedCrossingResult, crossingResult); @@ -137,30 +137,30 @@ public void testUnknownKey() throws URISyntaxException { @Test public void testNestedFiles() throws URISyntaxException { - Path texFile = Paths.get(CrossingKeysTest.class.getResource("nested.tex").toURI()); + Path texFile = Paths.get(TexBibEntriesResolverTest.class.getResource("nested.tex").toURI()); TexParserResult parserResult = new DefaultTexParser().parse(texFile); - CrossingKeysResult crossingResult = new CrossingKeys(parserResult, database).resolveKeys(); - CrossingKeysResult expectedCrossingResult = new CrossingKeysResult(parserResult, database); + TexBibEntriesResolverResult crossingResult = new TexBibEntriesResolver(database).resolveKeys(parserResult); + TexBibEntriesResolverResult expectedCrossingResult = new TexBibEntriesResolverResult(parserResult); - expectedCrossingResult.insertEntry(DARWIN); - expectedCrossingResult.insertEntry(EINSTEIN); + expectedCrossingResult.insertEntry(database, DARWIN); + expectedCrossingResult.insertEntry(database, EINSTEIN); assertEquals(expectedCrossingResult, crossingResult); } @Test public void testCrossRef() throws URISyntaxException { - Path texFile = Paths.get(CrossingKeysTest.class.getResource("crossref.tex").toURI()); + Path texFile = Paths.get(TexBibEntriesResolverTest.class.getResource("crossref.tex").toURI()); TexParserResult parserResult = new DefaultTexParser().parse(texFile); - CrossingKeysResult crossingResult = new CrossingKeys(parserResult, database).resolveKeys(); - CrossingKeysResult expectedCrossingResult = new CrossingKeysResult(parserResult, database); + TexBibEntriesResolverResult crossingResult = new TexBibEntriesResolver(database).resolveKeys(parserResult); + TexBibEntriesResolverResult expectedCrossingResult = new TexBibEntriesResolverResult(parserResult); - expectedCrossingResult.insertEntry(EINSTEIN); - expectedCrossingResult.insertEntry(EINSTEIN_A); - expectedCrossingResult.insertEntry(EINSTEIN_B); - expectedCrossingResult.insertEntry(EINSTEIN_C); + expectedCrossingResult.insertEntry(database, EINSTEIN_B); + expectedCrossingResult.insertEntry(database, EINSTEIN_A); + expectedCrossingResult.insertEntry(database, EINSTEIN); + expectedCrossingResult.insertEntry(database, EINSTEIN_C); expectedCrossingResult.addUnresolvedKey(EINSTEIN_21); expectedCrossingResult.addUnresolvedKey(UNRESOLVED); expectedCrossingResult.increaseCrossRefsCount(); diff --git a/src/test/java/org/jabref/logic/texparser/TexParserTest.java b/src/test/java/org/jabref/logic/texparser/TexParserTest.java index 4437f6cc32b..92ffd08988c 100644 --- a/src/test/java/org/jabref/logic/texparser/TexParserTest.java +++ b/src/test/java/org/jabref/logic/texparser/TexParserTest.java @@ -8,7 +8,7 @@ import org.jabref.model.database.BibDatabase; import org.jabref.model.entry.BibEntry; import org.jabref.model.entry.BibtexEntryTypes; -import org.jabref.model.texparser.CrossingKeysResult; +import org.jabref.model.texparser.TexBibEntriesResolverResult; import org.jabref.model.texparser.TexParserResult; import org.junit.jupiter.api.BeforeEach; @@ -92,18 +92,18 @@ public void testSameFileDifferentDatabases() throws URISyntaxException { expectedParserResult.addKey(EINSTEIN, texFile, 6, 14, 33, "Einstein said \\cite{Einstein1920} that lorem impsum, consectetur adipiscing elit."); expectedParserResult.addKey(DARWIN, texFile, 7, 67, 84, "Nunc ultricies leo nec libero rhoncus, eu vehicula enim efficitur. \\cite{Darwin1888}"); - CrossingKeysResult crossingResult = new CrossingKeys(parserResult, database).resolveKeys(); - CrossingKeysResult expectedCrossingResult = new CrossingKeysResult(expectedParserResult, database); + TexBibEntriesResolverResult crossingResult = new TexBibEntriesResolver(database).resolveKeys(parserResult); + TexBibEntriesResolverResult expectedCrossingResult = new TexBibEntriesResolverResult(expectedParserResult); - expectedCrossingResult.insertEntry(DARWIN); - expectedCrossingResult.insertEntry(EINSTEIN); + expectedCrossingResult.insertEntry(database, DARWIN); + expectedCrossingResult.insertEntry(database, EINSTEIN); assertEquals(expectedCrossingResult, crossingResult); - CrossingKeysResult crossingResult2 = new CrossingKeys(parserResult, database2).resolveKeys(); - CrossingKeysResult expectedCrossingResult2 = new CrossingKeysResult(expectedParserResult, database2); + TexBibEntriesResolverResult crossingResult2 = new TexBibEntriesResolver(database2).resolveKeys(parserResult); + TexBibEntriesResolverResult expectedCrossingResult2 = new TexBibEntriesResolverResult(expectedParserResult); - expectedCrossingResult2.insertEntry(EINSTEIN); + expectedCrossingResult2.insertEntry(database2, EINSTEIN); expectedCrossingResult2.addUnresolvedKey(DARWIN); assertEquals(expectedCrossingResult2, crossingResult2); @@ -126,19 +126,19 @@ public void testTwoFilesDifferentDatabases() throws URISyntaxException { expectedParserResult.addKey(EINSTEIN, texFile2, 5, 48, 67, "This is some content trying to cite a bib file: \\cite{Einstein1920}"); expectedParserResult.addKey(NEWTON, texFile2, 6, 48, 65, "This is some content trying to cite a bib file: \\cite{Newton1999}"); - CrossingKeysResult crossingResult = new CrossingKeys(parserResult, database).resolveKeys(); - CrossingKeysResult expectedCrossingResult = new CrossingKeysResult(expectedParserResult, database); + TexBibEntriesResolverResult crossingResult = new TexBibEntriesResolver(database).resolveKeys(parserResult); + TexBibEntriesResolverResult expectedCrossingResult = new TexBibEntriesResolverResult(expectedParserResult); - expectedCrossingResult.insertEntry(DARWIN); - expectedCrossingResult.insertEntry(EINSTEIN); - expectedCrossingResult.insertEntry(NEWTON); + expectedCrossingResult.insertEntry(database, DARWIN); + expectedCrossingResult.insertEntry(database, EINSTEIN); + expectedCrossingResult.insertEntry(database, NEWTON); assertEquals(expectedCrossingResult, crossingResult); - CrossingKeysResult crossingResult2 = new CrossingKeys(parserResult, database2).resolveKeys(); - CrossingKeysResult expectedCrossingResult2 = new CrossingKeysResult(expectedParserResult, database2); + TexBibEntriesResolverResult crossingResult2 = new TexBibEntriesResolver(database2).resolveKeys(parserResult); + TexBibEntriesResolverResult expectedCrossingResult2 = new TexBibEntriesResolverResult(expectedParserResult); - expectedCrossingResult2.insertEntry(EINSTEIN); + expectedCrossingResult2.insertEntry(database2, EINSTEIN); expectedCrossingResult2.addUnresolvedKey(DARWIN); expectedCrossingResult2.addUnresolvedKey(NEWTON); From eda24dffeee1963bfc24f0aeb80453b59a28a6d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20M=C3=A9ndez?= Date: Fri, 12 Jul 2019 14:15:02 +0200 Subject: [PATCH 09/57] Add user interface for the TEX parser tool (#5103) --- CHANGELOG.md | 1 + src/main/java/org/jabref/gui/JabRefFrame.java | 2 + .../jabref/gui/actions/StandardActions.java | 1 + .../gui/texparser/FileNodeViewModel.java | 58 ++++++ .../jabref/gui/texparser/ParseTexAction.java | 25 +++ .../jabref/gui/texparser/ParseTexDialog.fxml | 42 ++++ .../gui/texparser/ParseTexDialogView.java | 113 +++++++++++ .../texparser/ParseTexDialogViewModel.java | 188 ++++++++++++++++++ .../jabref/gui/texparser/ParseTexResult.css | 16 ++ .../jabref/gui/texparser/ParseTexResult.fxml | 21 ++ .../gui/texparser/ParseTexResultView.java | 69 +++++++ .../texparser/ParseTexResultViewModel.java | 43 ++++ .../gui/texparser/ReferenceViewModel.java | 42 ++++ .../jabref/gui/util/RecursiveTreeItem.java | 8 +- .../jabref/logic/util/StandardFileType.java | 1 + .../org/jabref/model/texparser/Citation.java | 2 +- .../TexBibEntriesResolverResult.java | 4 +- .../model/texparser/TexParserResult.java | 8 + src/main/resources/l10n/JabRef_en.properties | 6 + 19 files changed, 645 insertions(+), 5 deletions(-) create mode 100644 src/main/java/org/jabref/gui/texparser/FileNodeViewModel.java create mode 100644 src/main/java/org/jabref/gui/texparser/ParseTexAction.java create mode 100644 src/main/java/org/jabref/gui/texparser/ParseTexDialog.fxml create mode 100644 src/main/java/org/jabref/gui/texparser/ParseTexDialogView.java create mode 100644 src/main/java/org/jabref/gui/texparser/ParseTexDialogViewModel.java create mode 100644 src/main/java/org/jabref/gui/texparser/ParseTexResult.css create mode 100644 src/main/java/org/jabref/gui/texparser/ParseTexResult.fxml create mode 100644 src/main/java/org/jabref/gui/texparser/ParseTexResultView.java create mode 100644 src/main/java/org/jabref/gui/texparser/ParseTexResultViewModel.java create mode 100644 src/main/java/org/jabref/gui/texparser/ReferenceViewModel.java diff --git a/CHANGELOG.md b/CHANGELOG.md index bca01e2d50f..ce6a2adcd32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -69,6 +69,7 @@ We refer to [GitHub issues](https://github.com/JabRef/jabref/issues) by using `# - We moved the dropdown menu for selecting the push-application from the toolbar into the external application preferences. [#674](https://github.com/JabRef/jabref/issues/674) - We removed the alphabetical ordering of the custom tabs and updated the error message when trying to create a general field with a name containing an illegal character. [#5019](https://github.com/JabRef/jabref/issues/5019) - We added a context menu to the bib(la)tex-source-editor to copy'n'paste. [#5007](https://github.com/JabRef/jabref/pull/5007) +- We added a bibliographic references search, for finding references in several LaTeX files. This tool scans directories and shows which entries are used, how many times and where. ### Fixed diff --git a/src/main/java/org/jabref/gui/JabRefFrame.java b/src/main/java/org/jabref/gui/JabRefFrame.java index 73f838d6315..9922adca817 100644 --- a/src/main/java/org/jabref/gui/JabRefFrame.java +++ b/src/main/java/org/jabref/gui/JabRefFrame.java @@ -101,6 +101,7 @@ import org.jabref.gui.search.GlobalSearchBar; import org.jabref.gui.shared.ConnectToSharedDatabaseCommand; import org.jabref.gui.specialfields.SpecialFieldMenuItemFactory; +import org.jabref.gui.texparser.ParseTexAction; import org.jabref.gui.undo.CountingUndoManager; import org.jabref.gui.util.BackgroundTask; import org.jabref.gui.util.DefaultTaskExecutor; @@ -768,6 +769,7 @@ private MenuBar createMenu() { pushToApplicationsManager.setMenuItem(pushToApplicationMenuItem); tools.getItems().addAll( + factory.createMenuItem(StandardActions.PARSE_TEX, new ParseTexAction(stateManager)), factory.createMenuItem(StandardActions.NEW_SUB_LIBRARY_FROM_AUX, new NewSubLibraryAction(this, stateManager)), factory.createMenuItem(StandardActions.FIND_UNLINKED_FILES, new FindUnlinkedFilesAction(this, stateManager)), factory.createMenuItem(StandardActions.WRITE_XMP, new OldDatabaseCommandWrapper(Actions.WRITE_XMP, this, stateManager)), diff --git a/src/main/java/org/jabref/gui/actions/StandardActions.java b/src/main/java/org/jabref/gui/actions/StandardActions.java index eb4c6508955..feb1e072275 100644 --- a/src/main/java/org/jabref/gui/actions/StandardActions.java +++ b/src/main/java/org/jabref/gui/actions/StandardActions.java @@ -85,6 +85,7 @@ public enum StandardActions implements Action { TOOGLE_OO(Localization.lang("OpenOffice/LibreOffice"), IconTheme.JabRefIcons.FILE_OPENOFFICE, KeyBinding.OPEN_OPEN_OFFICE_LIBRE_OFFICE_CONNECTION), TOGGLE_WEB_SEARCH(Localization.lang("Web search"), Localization.lang("Toggle web search interface"), IconTheme.JabRefIcons.WWW, KeyBinding.WEB_SEARCH), + PARSE_TEX(Localization.lang("LaTeX references search"), IconTheme.JabRefIcons.APPLICATION_TEXSTUDIO), NEW_SUB_LIBRARY_FROM_AUX(Localization.lang("New sublibrary based on AUX file") + "...", Localization.lang("New BibTeX sublibrary") + Localization.lang("This feature generates a new library based on which entries are needed in an existing LaTeX document."), IconTheme.JabRefIcons.NEW), WRITE_XMP(Localization.lang("Write XMP-metadata to PDFs"), Localization.lang("Will write XMP-metadata to the PDFs linked from selected entries."), KeyBinding.WRITE_XMP), OPEN_FOLDER(Localization.lang("Open folder"), Localization.lang("Open folder"), KeyBinding.OPEN_FOLDER), diff --git a/src/main/java/org/jabref/gui/texparser/FileNodeViewModel.java b/src/main/java/org/jabref/gui/texparser/FileNodeViewModel.java new file mode 100644 index 00000000000..63c743df3fc --- /dev/null +++ b/src/main/java/org/jabref/gui/texparser/FileNodeViewModel.java @@ -0,0 +1,58 @@ +package org.jabref.gui.texparser; + +import java.nio.file.Path; +import java.util.StringJoiner; + +import javafx.collections.FXCollections; +import javafx.collections.ObservableList; + +import org.jabref.logic.l10n.Localization; + +class FileNodeViewModel { + + private final Path path; + private final ObservableList children; + private int fileCount; + + public FileNodeViewModel(Path path) { + this.path = path; + this.fileCount = 0; + this.children = FXCollections.observableArrayList(); + } + + public Path getPath() { + return path; + } + + public int getFileCount() { + return fileCount; + } + + public void setFileCount(int fileCount) { + this.fileCount = fileCount; + } + + public ObservableList getChildren() { + return children; + } + + /** + * Return a string for displaying a node name (and its number of children if it is a directory). + */ + public String getDisplayText() { + if (path.toFile().isDirectory()) { + return String.format("%s (%s %s)", path.getFileName(), fileCount, + fileCount == 1 ? Localization.lang("file") : Localization.lang("files")); + } + return path.getFileName().toString(); + } + + @Override + public String toString() { + return new StringJoiner(", ", FileNodeViewModel.class.getSimpleName() + "[", "]") + .add("path=" + path) + .add("fileCount=" + fileCount) + .add("children=" + children) + .toString(); + } +} diff --git a/src/main/java/org/jabref/gui/texparser/ParseTexAction.java b/src/main/java/org/jabref/gui/texparser/ParseTexAction.java new file mode 100644 index 00000000000..b057f040ab2 --- /dev/null +++ b/src/main/java/org/jabref/gui/texparser/ParseTexAction.java @@ -0,0 +1,25 @@ +package org.jabref.gui.texparser; + +import org.jabref.gui.StateManager; +import org.jabref.gui.actions.SimpleCommand; +import org.jabref.model.database.BibDatabaseContext; + +import static org.jabref.gui.actions.ActionHelper.needsDatabase; + +public class ParseTexAction extends SimpleCommand { + + private final StateManager stateManager; + + public ParseTexAction(StateManager stateManager) { + this.stateManager = stateManager; + this.executable.bind(needsDatabase(stateManager)); + } + + @Override + public void execute() { + BibDatabaseContext database = stateManager.getActiveDatabase().orElseThrow(NullPointerException::new); + ParseTexDialogView dialog = new ParseTexDialogView(database); + + dialog.showAndWait(); + } +} diff --git a/src/main/java/org/jabref/gui/texparser/ParseTexDialog.fxml b/src/main/java/org/jabref/gui/texparser/ParseTexDialog.fxml new file mode 100644 index 00000000000..ee6fa85c697 --- /dev/null +++ b/src/main/java/org/jabref/gui/texparser/ParseTexDialog.fxml @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + +