Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Create fscrawler-tika and fscrawler-beans modules #503

Merged
merged 2 commits into from
Feb 14, 2018
Merged

Create fscrawler-tika and fscrawler-beans modules #503

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9996b73
Create fscrawler-tika and fscrawler-beans modules
dadoonet Feb 13, 2018
12e22cc
Fix missing LOG4J2 file
dadoonet Feb 13, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 74 additions & 0 deletions beans/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>fscrawler-parent</artifactId>
<groupId>fr.pilato.elasticsearch.crawler</groupId>
<version>2.5-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

<artifactId>fscrawler-beans</artifactId>
<name>FSCrawler Beans</name>

<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-help-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>versions-maven-plugin</artifactId>
</plugin>
<plugin>
<groupId>com.carrotsearch.randomizedtesting</groupId>
<artifactId>junit4-maven-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-enforcer-plugin</artifactId>
</plugin>
</plugins>
</build>

<dependencies>
<!-- Our framework -->
<dependency>
<groupId>fr.pilato.elasticsearch.crawler</groupId>
<artifactId>fscrawler-framework</artifactId>
</dependency>

<!-- Our Settings -->
<dependency>
<groupId>fr.pilato.elasticsearch.crawler</groupId>
<artifactId>fscrawler-settings</artifactId>
</dependency>

<!-- Test dependencies -->
<dependency>
<groupId>fr.pilato.elasticsearch.crawler</groupId>
<artifactId>fscrawler-test-framework</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>fr.pilato.elasticsearch.crawler</groupId>
<artifactId>fscrawler-test-documents</artifactId>
<scope>test</scope>
</dependency>
</dependencies>

</project>
0 ...earch/crawler/fs/meta/doc/Attributes.java → ...earch/crawler/fs/meta/doc/Attributes.java
View file
Open in desktop
File renamed without changes.
0 ...lasticsearch/crawler/fs/meta/doc/Doc.java → ...lasticsearch/crawler/fs/meta/doc/Doc.java
View file
Open in desktop
File renamed without changes.
0 ...search/crawler/fs/meta/doc/DocParser.java → ...search/crawler/fs/meta/doc/DocParser.java
View file
Open in desktop
File renamed without changes.
0 ...asticsearch/crawler/fs/meta/doc/File.java → ...asticsearch/crawler/fs/meta/doc/File.java
View file
Open in desktop
File renamed without changes.
0 ...asticsearch/crawler/fs/meta/doc/Meta.java → ...asticsearch/crawler/fs/meta/doc/Meta.java
View file
Open in desktop
File renamed without changes.
0 ...asticsearch/crawler/fs/meta/doc/Path.java → ...asticsearch/crawler/fs/meta/doc/Path.java
View file
Open in desktop
File renamed without changes.
0 ...earch/crawler/fs/meta/doc/PathParser.java → ...earch/crawler/fs/meta/doc/PathParser.java
View file
Open in desktop
File renamed without changes.
6 changes: 6 additions & 0 deletions core/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,12 @@
<artifactId>fscrawler-settings</artifactId>
</dependency>

<!-- Tika Extraction -->
<dependency>
<groupId>fr.pilato.elasticsearch.crawler</groupId>
<artifactId>fscrawler-tika</artifactId>
</dependency>

<!-- Our Elasticsearch Client -->
<dependency>
<groupId>fr.pilato.elasticsearch.crawler</groupId>
Expand Down
17 changes: 17 additions & 0 deletions pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@
<module>integration-tests</module>
<module>distribution</module>
<module>cli</module>
<module>tika</module>
<module>beans</module>
</modules>
<name>FSCrawler</name>
<url>https://github.com/dadoonet/fscrawler/</url>
Expand Down Expand Up @@ -302,6 +304,21 @@
<artifactId>fscrawler-elasticsearch-client</artifactId>
<version>2.5-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>fr.pilato.elasticsearch.crawler</groupId>
<artifactId>fscrawler-cli</artifactId>
<version>2.5-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>fr.pilato.elasticsearch.crawler</groupId>
<artifactId>fscrawler-beans</artifactId>
<version>2.5-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>fr.pilato.elasticsearch.crawler</groupId>
<artifactId>fscrawler-tika</artifactId>
<version>2.5-SNAPSHOT</version>
</dependency>

<dependency>
<groupId>org.elasticsearch.client</groupId>
Expand Down
122 changes: 122 additions & 0 deletions tika/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,122 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>fscrawler-parent</artifactId>
<groupId>fr.pilato.elasticsearch.crawler</groupId>
<version>2.5-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>

<artifactId>fscrawler-tika</artifactId>
<name>FSCrawler Tika</name>

<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-help-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-resources-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>versions-maven-plugin</artifactId>
</plugin>
<plugin>
<groupId>com.carrotsearch.randomizedtesting</groupId>
<artifactId>junit4-maven-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-enforcer-plugin</artifactId>
</plugin>
</plugins>
</build>

<dependencies>
<!-- Our framework -->
<dependency>
<groupId>fr.pilato.elasticsearch.crawler</groupId>
<artifactId>fscrawler-framework</artifactId>
</dependency>

<!-- Our Beans -->
<dependency>
<groupId>fr.pilato.elasticsearch.crawler</groupId>
<artifactId>fscrawler-beans</artifactId>
</dependency>

<!-- Our Settings -->
<dependency>
<groupId>fr.pilato.elasticsearch.crawler</groupId>
<artifactId>fscrawler-settings</artifactId>
</dependency>

<dependency>
<groupId>org.apache.tika</groupId>
<artifactId>tika-parsers</artifactId>
</dependency>
<!-- Add some optional dependencies -->
<dependency>
<groupId>org.xerial</groupId>
<artifactId>sqlite-jdbc</artifactId>
</dependency>

<!--
For legal reasons (incompatible license), these dependencies are to marked as optional so users who might want
to use them have to provided them manually in the lib dir.
-->
<dependency>
<groupId>com.levigo.jbig2</groupId>
<artifactId>levigo-jbig2-imageio</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>com.github.jai-imageio</groupId>
<artifactId>jai-imageio-core</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>com.github.jai-imageio</groupId>
<artifactId>jai-imageio-jpeg2000</artifactId>
<optional>true</optional>
</dependency>

<!-- For Language detection -->
<dependency>
<groupId>org.apache.tika</groupId>
<artifactId>tika-langdetect</artifactId>
</dependency>
<!--Dependency for parsing remote ssh directory [http://www.jcraft.com/jsch/]-->
<dependency>
<groupId>com.jcraft</groupId>
<artifactId>jsch</artifactId>
</dependency>

<!-- Test dependencies -->
<dependency>
<groupId>fr.pilato.elasticsearch.crawler</groupId>
<artifactId>fscrawler-test-framework</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>fr.pilato.elasticsearch.crawler</groupId>
<artifactId>fscrawler-test-documents</artifactId>
<scope>test</scope>
</dependency>

</dependencies>

</project>
0 ...search/crawler/fs/tika/TikaDocParser.java → ...search/crawler/fs/tika/TikaDocParser.java
View file
Open in desktop
File renamed without changes.
0 ...csearch/crawler/fs/tika/TikaInstance.java → ...csearch/crawler/fs/tika/TikaInstance.java
View file
Open in desktop
File renamed without changes.
0 ...csearch/crawler/fs/tika/XmlDocParser.java → ...csearch/crawler/fs/tika/XmlDocParser.java
View file
Open in desktop
File renamed without changes.
2 changes: 1 addition & 1 deletion ...s/test/unit/parser/DocParserTestCase.java → ...ch/crawler/fs/tika/DocParserTestCase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
* under the License.
*/

package fr.pilato.elasticsearch.crawler.fs.test.unit.parser;
package fr.pilato.elasticsearch.crawler.fs.tika;

import fr.pilato.elasticsearch.crawler.fs.test.framework.AbstractFSCrawlerTestCase;

Expand Down
4 changes: 1 addition & 3 deletions ...s/test/unit/parser/TikaDocParserTest.java → ...ch/crawler/fs/tika/TikaDocParserTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,11 @@
* under the License.
*/

package fr.pilato.elasticsearch.crawler.fs.test.unit.parser;
package fr.pilato.elasticsearch.crawler.fs.tika;

import fr.pilato.elasticsearch.crawler.fs.meta.doc.Doc;
import fr.pilato.elasticsearch.crawler.fs.meta.settings.Fs;
import fr.pilato.elasticsearch.crawler.fs.meta.settings.FsSettings;
import fr.pilato.elasticsearch.crawler.fs.tika.TikaDocParser;
import fr.pilato.elasticsearch.crawler.fs.tika.TikaInstance;
import org.apache.tika.parser.external.ExternalParser;
import org.junit.Test;

Expand Down
3 changes: 1 addition & 2 deletions ...fs/test/unit/parser/XmlDocParserTest.java → ...rch/crawler/fs/tika/XmlDocParserTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,8 @@
* under the License.
*/

package fr.pilato.elasticsearch.crawler.fs.test.unit.parser;
package fr.pilato.elasticsearch.crawler.fs.tika;

import fr.pilato.elasticsearch.crawler.fs.tika.XmlDocParser;
import org.junit.Test;

import java.io.IOException;
Expand Down
55 changes: 55 additions & 0 deletions tika/src/test/resources/log4j2.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
<?xml version="1.0" encoding="UTF-8"?>
<Configuration status="fatal">
<Appenders>
<Console name="CONSOLE" target="SYSTEM_OUT">
<PatternLayout pattern="%d{ABSOLUTE} %highlight{%-5p} [%c{1.}] %m%n"/>
</Console>
<Console name="CONSOLE_NO_DECORATOR" target="SYSTEM_OUT">
<PatternLayout pattern="%m%n"/>
</Console>
</Appenders>
<Loggers>
<Logger name="fr.pilato.elasticsearch.crawler.fs" level="info" additivity="false">
<AppenderRef ref="CONSOLE"/>
</Logger>
<Logger name="fr.pilato.elasticsearch.crawler.fs.FsCrawler" level="info" additivity="false">
<AppenderRef ref="CONSOLE"/>
</Logger>
<Logger name="fr.pilato.elasticsearch.crawler.fs.FsCrawlerImpl" level="info" additivity="false">
<AppenderRef ref="CONSOLE"/>
</Logger>
<Logger name="fr.pilato.elasticsearch.crawler.fs.client" level="info" additivity="false">
<AppenderRef ref="CONSOLE"/>
</Logger>
<Logger name="fr.pilato.elasticsearch.crawler.fs.rest" level="info" additivity="false">
<AppenderRef ref="CONSOLE"/>
</Logger>
<Logger name="fr.pilato.elasticsearch.crawler.fs.tika" level="info" additivity="false">
<AppenderRef ref="CONSOLE_NO_DECORATOR"/>
</Logger>
<Logger name="fr.pilato.elasticsearch.crawler.fs.util" level="info" additivity="false">
<AppenderRef ref="CONSOLE"/>
</Logger>
<Logger name="fr.pilato.elasticsearch.crawler.fs.test" level="info" additivity="false">
<AppenderRef ref="CONSOLE"/>
</Logger>
<Logger name="fr.pilato.elasticsearch.crawler.fs.test.integration" level="info" additivity="false">
<AppenderRef ref="CONSOLE"/>
</Logger>
<Logger name="fr.pilato.elasticsearch.crawler.fs.test.unit" level="info" additivity="false">
<AppenderRef ref="CONSOLE"/>
</Logger>
<Logger name="org.elasticsearch" level="error" additivity="false">
<AppenderRef ref="CONSOLE"/>
</Logger>
<Logger name="org.glassfish" level="error" additivity="false">
<AppenderRef ref="CONSOLE"/>
</Logger>
<Logger name="org.apache.tika.parser.ocr.TesseractOCRParser" level="error" additivity="false">
<AppenderRef ref="CONSOLE"/>
</Logger>
<Root level="info">
<AppenderRef ref="CONSOLE"/>
</Root>
</Loggers>
</Configuration>