Create fscrawler-tika and fscrawler-beans modules

Related to #502.
dadoonet · Feb 13, 2018 · 9996b73 · 9996b73
1 parent 635ddb8
commit 9996b73
Show file tree

Hide file tree

Showing 17 changed files with 222 additions and 6 deletions.
diff --git a/beans/pom.xml b/beans/pom.xml
@@ -0,0 +1,74 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>fscrawler-parent</artifactId>
+        <groupId>fr.pilato.elasticsearch.crawler</groupId>
+        <version>2.5-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>fscrawler-beans</artifactId>
+    <name>FSCrawler Beans</name>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-help-plugin</artifactId>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-resources-plugin</artifactId>
+            </plugin>
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>versions-maven-plugin</artifactId>
+            </plugin>
+            <plugin>
+                <groupId>com.carrotsearch.randomizedtesting</groupId>
+                <artifactId>junit4-maven-plugin</artifactId>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-enforcer-plugin</artifactId>
+            </plugin>
+        </plugins>
+    </build>
+
+    <dependencies>
+        <!-- Our framework -->
+        <dependency>
+            <groupId>fr.pilato.elasticsearch.crawler</groupId>
+            <artifactId>fscrawler-framework</artifactId>
+        </dependency>
+
+        <!-- Our Settings -->
+        <dependency>
+            <groupId>fr.pilato.elasticsearch.crawler</groupId>
+            <artifactId>fscrawler-settings</artifactId>
+        </dependency>
+
+        <!-- Test dependencies -->
+        <dependency>
+            <groupId>fr.pilato.elasticsearch.crawler</groupId>
+            <artifactId>fscrawler-test-framework</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>fr.pilato.elasticsearch.crawler</groupId>
+            <artifactId>fscrawler-test-documents</artifactId>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+</project>
diff --git a/...earch/crawler/fs/meta/doc/Attributes.java → ...earch/crawler/fs/meta/doc/Attributes.java b/...earch/crawler/fs/meta/doc/Attributes.java → ...earch/crawler/fs/meta/doc/Attributes.java
diff --git a/...lasticsearch/crawler/fs/meta/doc/Doc.java → ...lasticsearch/crawler/fs/meta/doc/Doc.java b/...lasticsearch/crawler/fs/meta/doc/Doc.java → ...lasticsearch/crawler/fs/meta/doc/Doc.java
diff --git a/...search/crawler/fs/meta/doc/DocParser.java → ...search/crawler/fs/meta/doc/DocParser.java b/...search/crawler/fs/meta/doc/DocParser.java → ...search/crawler/fs/meta/doc/DocParser.java
diff --git a/...asticsearch/crawler/fs/meta/doc/File.java → ...asticsearch/crawler/fs/meta/doc/File.java b/...asticsearch/crawler/fs/meta/doc/File.java → ...asticsearch/crawler/fs/meta/doc/File.java
diff --git a/...asticsearch/crawler/fs/meta/doc/Meta.java → ...asticsearch/crawler/fs/meta/doc/Meta.java b/...asticsearch/crawler/fs/meta/doc/Meta.java → ...asticsearch/crawler/fs/meta/doc/Meta.java
diff --git a/...asticsearch/crawler/fs/meta/doc/Path.java → ...asticsearch/crawler/fs/meta/doc/Path.java b/...asticsearch/crawler/fs/meta/doc/Path.java → ...asticsearch/crawler/fs/meta/doc/Path.java
diff --git a/...earch/crawler/fs/meta/doc/PathParser.java → ...earch/crawler/fs/meta/doc/PathParser.java b/...earch/crawler/fs/meta/doc/PathParser.java → ...earch/crawler/fs/meta/doc/PathParser.java
diff --git a/core/pom.xml b/core/pom.xml
@@ -64,6 +64,12 @@
             <artifactId>fscrawler-settings</artifactId>
         </dependency>
 
+        <!-- Tika Extraction -->
+        <dependency>
+            <groupId>fr.pilato.elasticsearch.crawler</groupId>
+            <artifactId>fscrawler-tika</artifactId>
+        </dependency>
+
         <!-- Our Elasticsearch Client -->
         <dependency>
             <groupId>fr.pilato.elasticsearch.crawler</groupId>

diff --git a/pom.xml b/pom.xml
@@ -16,6 +16,8 @@
         <module>integration-tests</module>
         <module>distribution</module>
         <module>cli</module>
+        <module>tika</module>
+        <module>beans</module>
     </modules>
     <name>FSCrawler</name>
     <url>https://github.com/dadoonet/fscrawler/</url>
@@ -302,6 +304,21 @@
                 <artifactId>fscrawler-elasticsearch-client</artifactId>
                 <version>2.5-SNAPSHOT</version>
             </dependency>
+            <dependency>
+                <groupId>fr.pilato.elasticsearch.crawler</groupId>
+                <artifactId>fscrawler-cli</artifactId>
+                <version>2.5-SNAPSHOT</version>
+            </dependency>
+            <dependency>
+                <groupId>fr.pilato.elasticsearch.crawler</groupId>
+                <artifactId>fscrawler-beans</artifactId>
+                <version>2.5-SNAPSHOT</version>
+            </dependency>
+            <dependency>
+                <groupId>fr.pilato.elasticsearch.crawler</groupId>
+                <artifactId>fscrawler-tika</artifactId>
+                <version>2.5-SNAPSHOT</version>
+            </dependency>
 
             <dependency>
                 <groupId>org.elasticsearch.client</groupId>

diff --git a/tika/pom.xml b/tika/pom.xml
@@ -0,0 +1,122 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>fscrawler-parent</artifactId>
+        <groupId>fr.pilato.elasticsearch.crawler</groupId>
+        <version>2.5-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>fscrawler-tika</artifactId>
+    <name>FSCrawler Tika</name>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-help-plugin</artifactId>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-resources-plugin</artifactId>
+            </plugin>
+            <plugin>
+                <groupId>org.codehaus.mojo</groupId>
+                <artifactId>versions-maven-plugin</artifactId>
+            </plugin>
+            <plugin>
+                <groupId>com.carrotsearch.randomizedtesting</groupId>
+                <artifactId>junit4-maven-plugin</artifactId>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-enforcer-plugin</artifactId>
+            </plugin>
+        </plugins>
+    </build>
+
+    <dependencies>
+        <!-- Our framework -->
+        <dependency>
+            <groupId>fr.pilato.elasticsearch.crawler</groupId>
+            <artifactId>fscrawler-framework</artifactId>
+        </dependency>
+
+        <!-- Our Beans -->
+        <dependency>
+            <groupId>fr.pilato.elasticsearch.crawler</groupId>
+            <artifactId>fscrawler-beans</artifactId>
+        </dependency>
+
+        <!-- Our Settings -->
+        <dependency>
+            <groupId>fr.pilato.elasticsearch.crawler</groupId>
+            <artifactId>fscrawler-settings</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.tika</groupId>
+            <artifactId>tika-parsers</artifactId>
+        </dependency>
+        <!-- Add some optional dependencies -->
+        <dependency>
+            <groupId>org.xerial</groupId>
+            <artifactId>sqlite-jdbc</artifactId>
+        </dependency>
+
+        <!--
+        For legal reasons (incompatible license), these dependencies are to marked as optional so users who might want
+        to use them have to provided them manually in the lib dir.
+        -->
+        <dependency>
+            <groupId>com.levigo.jbig2</groupId>
+            <artifactId>levigo-jbig2-imageio</artifactId>
+            <optional>true</optional>
+        </dependency>
+        <dependency>
+            <groupId>com.github.jai-imageio</groupId>
+            <artifactId>jai-imageio-core</artifactId>
+            <optional>true</optional>
+        </dependency>
+        <dependency>
+            <groupId>com.github.jai-imageio</groupId>
+            <artifactId>jai-imageio-jpeg2000</artifactId>
+            <optional>true</optional>
+        </dependency>
+
+        <!-- For Language detection -->
+        <dependency>
+            <groupId>org.apache.tika</groupId>
+            <artifactId>tika-langdetect</artifactId>
+        </dependency>
+        <!--Dependency for parsing remote ssh directory [http://www.jcraft.com/jsch/]-->
+        <dependency>
+            <groupId>com.jcraft</groupId>
+            <artifactId>jsch</artifactId>
+        </dependency>
+
+        <!-- Test dependencies -->
+        <dependency>
+            <groupId>fr.pilato.elasticsearch.crawler</groupId>
+            <artifactId>fscrawler-test-framework</artifactId>
+            <scope>test</scope>
+        </dependency>
+        <dependency>
+            <groupId>fr.pilato.elasticsearch.crawler</groupId>
+            <artifactId>fscrawler-test-documents</artifactId>
+            <scope>test</scope>
+        </dependency>
+
+    </dependencies>
+
+</project>
diff --git a/...search/crawler/fs/tika/TikaDocParser.java → ...search/crawler/fs/tika/TikaDocParser.java b/...search/crawler/fs/tika/TikaDocParser.java → ...search/crawler/fs/tika/TikaDocParser.java
diff --git a/...csearch/crawler/fs/tika/TikaInstance.java → ...csearch/crawler/fs/tika/TikaInstance.java b/...csearch/crawler/fs/tika/TikaInstance.java → ...csearch/crawler/fs/tika/TikaInstance.java
diff --git a/...csearch/crawler/fs/tika/XmlDocParser.java → ...csearch/crawler/fs/tika/XmlDocParser.java b/...csearch/crawler/fs/tika/XmlDocParser.java → ...csearch/crawler/fs/tika/XmlDocParser.java
diff --git a/...s/test/unit/parser/DocParserTestCase.java → ...ch/crawler/fs/tika/DocParserTestCase.java b/...s/test/unit/parser/DocParserTestCase.java → ...ch/crawler/fs/tika/DocParserTestCase.java
@@ -17,7 +17,7 @@
  * under the License.
  */
 
-package fr.pilato.elasticsearch.crawler.fs.test.unit.parser;
+package fr.pilato.elasticsearch.crawler.fs.tika;
 
 import fr.pilato.elasticsearch.crawler.fs.test.framework.AbstractFSCrawlerTestCase;
 

diff --git a/...s/test/unit/parser/TikaDocParserTest.java → ...ch/crawler/fs/tika/TikaDocParserTest.java b/...s/test/unit/parser/TikaDocParserTest.java → ...ch/crawler/fs/tika/TikaDocParserTest.java
@@ -17,13 +17,11 @@
  * under the License.
  */
 
-package fr.pilato.elasticsearch.crawler.fs.test.unit.parser;
+package fr.pilato.elasticsearch.crawler.fs.tika;
 
 import fr.pilato.elasticsearch.crawler.fs.meta.doc.Doc;
 import fr.pilato.elasticsearch.crawler.fs.meta.settings.Fs;
 import fr.pilato.elasticsearch.crawler.fs.meta.settings.FsSettings;
-import fr.pilato.elasticsearch.crawler.fs.tika.TikaDocParser;
-import fr.pilato.elasticsearch.crawler.fs.tika.TikaInstance;
 import org.apache.tika.parser.external.ExternalParser;
 import org.junit.Test;
 

diff --git a/...fs/test/unit/parser/XmlDocParserTest.java → ...rch/crawler/fs/tika/XmlDocParserTest.java b/...fs/test/unit/parser/XmlDocParserTest.java → ...rch/crawler/fs/tika/XmlDocParserTest.java
@@ -17,9 +17,8 @@
  * under the License.
  */
 
-package fr.pilato.elasticsearch.crawler.fs.test.unit.parser;
+package fr.pilato.elasticsearch.crawler.fs.tika;
 
-import fr.pilato.elasticsearch.crawler.fs.tika.XmlDocParser;
 import org.junit.Test;
 
 import java.io.IOException;