Revert "Switch from tika-parsers to tika-core (#5217)"

This reverts commit 29cf4f2
JabRef · Aug 26, 2019 · be558fe · be558fe
1 parent c34f1d3
commit be558fe
Show file tree

Hide file tree

Showing 5 changed files with 9 additions and 66 deletions.
diff --git a/build.gradle b/build.gradle
@@ -122,7 +122,7 @@ dependencies {
     compile 'org.apache.pdfbox:fontbox:2.0.16'
     compile 'org.apache.pdfbox:xmpbox:2.0.16'
 
-    compile group: 'org.apache.tika', name: 'tika-core', version: '1.22'
+    compile group: 'org.apache.tika', name: 'tika-parsers', version: '1.22'
 
     // required for reading write-protected PDFs - see https://github.com/JabRef/jabref/pull/942#issuecomment-209252635
     compile 'org.bouncycastle:bcprov-jdk15on:1.62'

diff --git a/docs/adr/0005-fully-support-utf8-only-for-latex-files.md b/docs/adr/0005-fully-support-utf8-only-for-latex-files.md
diff --git a/docs/adr/index.md b/docs/adr/index.md
@@ -9,7 +9,6 @@ This log lists the architectural decisions for JabRef.
 - [ADR-0002](0002-use-slf4j-for-logging.md) - Use slf4j together with log4j2 for logging
 - [ADR-0003](0003-use-gradle-as-build-tool.md) - Use Gradle as build tool
 - [ADR-0004](0004-use-mariadb-connector.md) - Use MariaDB Connector
-- [ADR-0005](0005-fully-support-utf8-only-for-latex-files.md) - Fully Support UTF-8 Only For LaTeX Files
 
 <!-- adrlogstop -->
 

diff --git a/src/main/java/org/jabref/logic/texparser/DefaultTexParser.java b/src/main/java/org/jabref/logic/texparser/DefaultTexParser.java
@@ -1,13 +1,10 @@
 package org.jabref.logic.texparser;
 
 import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
 import java.io.LineNumberReader;
 import java.io.Reader;
 import java.io.UncheckedIOException;
 import java.nio.channels.ClosedChannelException;
-import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
@@ -20,6 +17,7 @@
 import org.jabref.model.texparser.TexParser;
 import org.jabref.model.texparser.TexParserResult;
 
+import org.apache.tika.parser.txt.CharsetDetector;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -84,8 +82,7 @@ public TexParserResult parse(List<Path> texFiles) {
             }
 
             try (
-                    InputStream inputStream = Files.newInputStream(file);
-                    Reader reader = new InputStreamReader(inputStream, StandardCharsets.UTF_8);
+                    Reader reader = new CharsetDetector().setText(Files.readAllBytes(file)).detect().getReader();
                     LineNumberReader lineNumberReader = new LineNumberReader(reader)) {
                 for (String line = lineNumberReader.readLine(); line != null; line = lineNumberReader.readLine()) {
                     // Skip comments and blank lines.

diff --git a/src/test/java/org/jabref/logic/texparser/DefaultTexParserTest.java b/src/test/java/org/jabref/logic/texparser/DefaultTexParserTest.java
@@ -12,7 +12,6 @@
 import static org.junit.jupiter.api.Assertions.assertEquals;
 
 public class DefaultTexParserTest {
-
     private final static String DARWIN = "Darwin1888";
     private final static String EINSTEIN = "Einstein1920";
     private final static String NEWTON = "Newton1999";
@@ -89,9 +88,7 @@ public void testFileEncodingIso88591() throws URISyntaxException {
         TexParserResult expectedParserResult = new TexParserResult();
 
         expectedParserResult.getFileList().add(texFile);
-        // The character � is on purpose - we cannot use Apache Tika's CharsetDetector - see ADR-0005
-        expectedParserResult
-                .addKey("anykey", texFile, 1, 32, 45, "Danach wir anschlie�end mittels \\cite{anykey}.");
+        expectedParserResult.addKey("anykey", texFile, 1, 32, 45, "Danach wir anschließend mittels \\cite{anykey}.");
 
         assertEquals(expectedParserResult, parserResult);
     }
@@ -104,9 +101,7 @@ public void testFileEncodingIso885915() throws URISyntaxException {
         TexParserResult expectedParserResult = new TexParserResult();
 
         expectedParserResult.getFileList().add(texFile);
-        // The character � is on purpose - we cannot use Apache Tika's CharsetDetector - see ADR-0005
-        expectedParserResult
-                .addKey("anykey", texFile, 1, 32, 45, "Danach wir anschlie�end mittels \\cite{anykey}.");
+        expectedParserResult.addKey("anykey", texFile, 1, 32, 45, "Danach wir anschließend mittels \\cite{anykey}.");
 
         assertEquals(expectedParserResult, parserResult);
     }
@@ -117,17 +112,13 @@ public void testFileEncodingForThreeFiles() throws URISyntaxException {
         Path texFile2 = Paths.get(DefaultTexParserTest.class.getResource("iso-8859-1.tex").toURI());
         Path texFile3 = Paths.get(DefaultTexParserTest.class.getResource("iso-8859-15.tex").toURI());
 
-        TexParserResult parserResult = new DefaultTexParser()
-                .parse(Arrays.asList(texFile, texFile2, texFile3));
+        TexParserResult parserResult = new DefaultTexParser().parse(Arrays.asList(texFile, texFile2, texFile3));
         TexParserResult expectedParserResult = new TexParserResult();
 
         expectedParserResult.getFileList().addAll(Arrays.asList(texFile, texFile2, texFile3));
-        expectedParserResult
-                .addKey("anykey", texFile, 1, 32, 45, "Danach wir anschließend mittels \\cite{anykey}.");
-        expectedParserResult
-                .addKey("anykey", texFile2, 1, 32, 45, "Danach wir anschlie�end mittels \\cite{anykey}.");
-        expectedParserResult
-                .addKey("anykey", texFile3, 1, 32, 45, "Danach wir anschlie�end mittels \\cite{anykey}.");
+        expectedParserResult.addKey("anykey", texFile, 1, 32, 45, "Danach wir anschließend mittels \\cite{anykey}.");
+        expectedParserResult.addKey("anykey", texFile2, 1, 32, 45, "Danach wir anschließend mittels \\cite{anykey}.");
+        expectedParserResult.addKey("anykey", texFile3, 1, 32, 45, "Danach wir anschließend mittels \\cite{anykey}.");
 
         assertEquals(expectedParserResult, parserResult);
     }