Citavi Importer - Import all knowledge items

CHANGELOG.md

@@ -19,6 +19,8 @@ Note that this project **does not** adhere to [Semantic Versioning](http://semve
 
 ### Fixed
 
+- The [HtmlToLaTeXFormatter](https://docs.jabref.org/finding-sorting-and-cleaning-entries/saveactions#html-to-latex) keeps single `<` characters.
+
 ### Removed
 
 

src/main/java/org/jabref/logic/formatter/bibtexfields/HtmlToLatexFormatter.java

@@ -44,7 +44,12 @@ public String format(String text) {
             int c = result.charAt(i);
 
             if (c == '<') {
+                int oldI = i;
                 i = readTag(result, i);
+                if (oldI == i) {
+                    // just a single <, which needs to be kept
+                    sb.append('<');
+                }
             } else {
                 sb.append((char) c);
             }

src/main/java/org/jabref/logic/importer/fileformat/CitaviXmlImporter.java

@@ -20,6 +20,9 @@
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
+import java.util.StringJoiner;
+import java.util.function.Predicate;
+import java.util.regex.Matcher;
 import java.util.stream.Collectors;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipInputStream;
@@ -28,6 +31,7 @@
 import javax.xml.stream.XMLStreamException;
 import javax.xml.stream.XMLStreamReader;
 
+import org.jabref.logic.formatter.bibtexfields.HtmlToLatexFormatter;
 import org.jabref.logic.formatter.bibtexfields.NormalizePagesFormatter;
 import org.jabref.logic.importer.Importer;
 import org.jabref.logic.importer.Parser;
@@ -59,6 +63,7 @@ public class CitaviXmlImporter extends Importer implements Parser {
     private static final Logger LOGGER = LoggerFactory.getLogger(CitaviXmlImporter.class);
     private static final byte UUID_LENGTH = 36;
     private static final byte UUID_SEMICOLON_OFFSET_INDEX = 37;
+    private final HtmlToLatexFormatter htmlToLatexFormatter = new HtmlToLatexFormatter();
     private final NormalizePagesFormatter pagesFormatter = new NormalizePagesFormatter();
 
     private final Map<String, Author> knownPersons = new HashMap<>();
@@ -363,19 +368,33 @@ private String getPublisher(CitaviExchangeData.References.Reference data) {
     }
 
     private String getKnowledgeItem(CitaviExchangeData.References.Reference data) {
-        Optional<KnowledgeItem> knowledgeItem = knowledgeItems.getKnowledgeItem().stream().filter(p -> data.getId().equals(p.getReferenceID())).findFirst();
+        StringJoiner comment = new StringJoiner("\n\n");
+        List<KnowledgeItem> foundItems = knowledgeItems.getKnowledgeItem().stream().filter(p -> data.getId().equals(p.getReferenceID())).toList();
+        for (KnowledgeItem knowledgeItem : foundItems) {
+            Optional<String> title = Optional.ofNullable(knowledgeItem.getCoreStatement()).filter(Predicate.not(String::isEmpty));
+            title.ifPresent(t -> comment.add("# " + cleanUpText(t)));
 
-        StringBuilder comment = new StringBuilder();
-        Optional<String> title = knowledgeItem.map(item -> item.getCoreStatement());
-        title.ifPresent(t -> comment.append("# ").append(t).append("\n\n"));
-        Optional<String> text = knowledgeItem.map(item -> item.getText());
-        text.ifPresent(t -> comment.append(t).append("\n\n"));
-        Optional<Integer> pages = knowledgeItem.map(item -> item.getPageRangeNumber()).filter(range -> range != -1);
-        pages.ifPresent(p -> comment.append("page range: ").append(p));
+            Optional<String> text = Optional.ofNullable(knowledgeItem.getText()).filter(Predicate.not(String::isEmpty));
+            text.ifPresent(t -> comment.add(cleanUpText(t)));
 
+            Optional<Integer> pages = Optional.ofNullable(knowledgeItem.getPageRangeNumber()).filter(range -> range != -1);
+            pages.ifPresent(p -> comment.add("page range: " + p));
+        }
         return comment.toString();
     }
 
+    String cleanUpText(String text) {
+        String result = removeSpacesBeforeLineBreak(text);
+        result = result.replaceAll("(?<!\\\\)\\{", "\\\\{");
+        result = result.replaceAll("(?<!\\\\)}", "\\\\}");
+        return result;
+    }
+
+    private String removeSpacesBeforeLineBreak(String string) {
+        return string.replaceAll(" +\r\n", "\r\n")
+              .replaceAll(" +\n", "\n");
+    }
+
     private void initUnmarshaller() throws JAXBException {
         if (unmarshaller == null) {
             // Lazy init because this is expensive
@@ -453,8 +472,9 @@ private static InputStream checkForUtf8BOMAndDiscardIfAny(InputStream inputStrea
     }
 
     private String clean(String input) {
-        return StringUtil.unifyLineBreaks(input, " ")
+        String result = StringUtil.unifyLineBreaks(input, " ")
                          .trim()
                          .replaceAll(" +", " ");
+        return htmlToLatexFormatter.format(result);
     }
 }

src/main/java/org/jabref/logic/layout/format/RemoveWhitespace.java

@@ -3,7 +3,7 @@
 import org.jabref.logic.layout.LayoutFormatter;
 
 /**
- * Remove non printable character formatter.
+ * Remove non-printable character formatter.
  */
 public class RemoveWhitespace implements LayoutFormatter {
 

src/test/java/org/jabref/logic/formatter/bibtexfields/HtmlToLatexFormatterTest.java

@@ -58,6 +58,12 @@ public void testHTMLCombiningAccents() {
         assertEquals("{\\\"{a}}b", formatter.format("äb"));
     }
 
+    @Test
+    public void keepsSingleLessThan() {
+        String text = "(p < 0.01)";
+        assertEquals(text, formatter.format(text));
+    }
+
     @Test
     public void formatExample() {
         assertEquals("JabRef", formatter.format(formatter.getExampleInput()));

src/test/java/org/jabref/logic/importer/fileformat/CitaviXmlImporterTest.java

@@ -0,0 +1,28 @@
+package org.jabref.logic.importer.fileformat;
+
+import java.util.stream.Stream;
+
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+class CitaviXmlImporterTest {
+
+    CitaviXmlImporter citaviXmlImporter = new CitaviXmlImporter();
+
+    public static Stream<Arguments> cleanUpText() {
+        return Stream.of(
+                Arguments.of("no action", "no action"),
+                Arguments.of("\\{action\\}", "{action}"),
+                Arguments.of("\\}", "}"));
+    }
+
+    @ParameterizedTest
+    @MethodSource
+    void cleanUpText(String expected, String input) {
+        assertEquals(expected, citaviXmlImporter.cleanUpText(input));
+    }
+}