Skip to content

Commit

Permalink
[LINKIS #1632] Add linkis-engineplugin-elasticsearch module
Browse files Browse the repository at this point in the history
  • Loading branch information
@wForget
wForget committed Jun 19, 2022
1 parent 210c932 commit 2b6a149
Show file tree
Hide file tree
Showing 24 changed files with 1,566 additions and 0 deletions.
2 changes: 2 additions & 0 deletions ...label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/EngineType.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,8 @@ object EngineType extends Enumeration with Logging {

val OPENLOOKENG = Value("openlookeng")

val ELASTICSEARCH = Value("elasticsearch")

def mapFsTypeToEngineType(fsType: String): String = {
fsType match {
case "file" =>
Expand Down
2 changes: 2 additions & 0 deletions ...is-label-common/src/main/java/org/apache/linkis/manager/label/entity/engine/RunType.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,5 +38,7 @@ object RunType extends Enumeration {
val JAR = Value("jar")
val APPCONN = Value("appconn")
val FUNCTION_MDQ_TYPE = Value("function.mdq")
val ES_SQL = Value("essql")
val ES_JSON = Value("esjson")

}
167 changes: 167 additions & 0 deletions linkis-engineconn-plugins/engineconn-plugins/elasticsearch/pom.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,167 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->

<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<artifactId>linkis</artifactId>
<groupId>org.apache.linkis</groupId>
<version>1.1.2</version>
<relativePath>../../../pom.xml</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>

<artifactId>linkis-engineplugin-elasticsearch</artifactId>

<properties>
<elasticsearch.version>7.6.2</elasticsearch.version>
<fasterxml.jackson.version>2.11.3</fasterxml.jackson.version>
</properties>

<dependencies>
<dependency>
<groupId>org.apache.linkis</groupId>
<artifactId>linkis-engineconn-plugin-core</artifactId>
<version>${linkis.version}</version>
</dependency>

<dependency>
<groupId>org.apache.linkis</groupId>
<artifactId>linkis-computation-engineconn</artifactId>
<version>${linkis.version}</version>
</dependency>

<dependency>
<groupId>org.apache.linkis</groupId>
<artifactId>linkis-storage</artifactId>
<version>${linkis.version}</version>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>org.apache.linkis</groupId>
<artifactId>linkis-rpc</artifactId>
<version>${linkis.version}</version>
<scope>provided</scope>
</dependency>

<dependency>
<groupId>org.apache.linkis</groupId>
<artifactId>linkis-common</artifactId>
<version>${linkis.version}</version>
<scope>provided</scope>
</dependency>

<!-- elasticsearch -->
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-client</artifactId>
<version>${elasticsearch.version}</version>
</dependency>
<dependency>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-client-sniffer</artifactId>
<version>${elasticsearch.version}</version>
<exclusions>
<exclusion>
<artifactId>jackson-core</artifactId>
<groupId>com.fasterxml.jackson.core</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.dataformat</groupId>
<artifactId>jackson-dataformat-yaml</artifactId>
<version>${fasterxml.jackson.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.dataformat</groupId>
<artifactId>jackson-dataformat-cbor</artifactId>
<version>${fasterxml.jackson.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.dataformat</groupId>
<artifactId>jackson-dataformat-smile</artifactId>
<version>${fasterxml.jackson.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.dataformat</groupId>
<artifactId>jackson-dataformat-csv</artifactId>
<version>${fasterxml.jackson.version}</version>
</dependency>

</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
</plugin>

<plugin>
<groupId>net.alchim31.maven</groupId>
<artifactId>scala-maven-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
<version>2.3</version>
<inherited>false</inherited>
<executions>
<execution>
<id>make-assembly</id>
<phase>package</phase>
<goals>
<goal>single</goal>
</goals>
<configuration>
<descriptors>
<descriptor>src/main/assembly/distribution.xml</descriptor>
</descriptors>
</configuration>
</execution>
</executions>
<configuration>
<skipAssembly>false</skipAssembly>
<finalName>out</finalName>
<appendAssemblyId>false</appendAssemblyId>
<attach>false</attach>
<descriptors>
<descriptor>src/main/assembly/distribution.xml</descriptor>
</descriptors>
</configuration>
</plugin>
</plugins>
<resources>
<resource>
<directory>${basedir}/src/main/resources</directory>
<includes>
<include>**/*.properties</include>
<include>**/*.xml</include>
<include>**/*.yml</include>
</includes>
</resource>
</resources>
</build>

</project>
73 changes: 73 additions & 0 deletions ...is-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/assembly/distribution.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->

<assembly
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/2.3"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/2.3 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
<id>linkis-engineplugin-elasticsearch</id>
<formats>
<format>dir</format>
<format>zip</format>
</formats>
<includeBaseDirectory>true</includeBaseDirectory>
<baseDirectory>elasticsearch</baseDirectory>

<dependencySets>
<dependencySet>
<!-- Enable access to all projects in the current multimodule build! <useAllReactorProjects>true</useAllReactorProjects> -->
<!-- Now, select which projects to include in this module-set. -->
<outputDirectory>/dist/v${elasticsearch.version}/lib</outputDirectory>
<useProjectArtifact>true</useProjectArtifact>
<useTransitiveDependencies>true</useTransitiveDependencies>
<unpack>false</unpack>
<useStrictFiltering>false</useStrictFiltering>
<useTransitiveFiltering>true</useTransitiveFiltering>

</dependencySet>
</dependencySets>

<fileSets>

<fileSet>
<directory>${basedir}/src/main/resources</directory>
<includes>
<include>linkis-engineconn.properties</include>
<include>log4j2.xml</include>
</includes>
<fileMode>0777</fileMode>
<outputDirectory>/dist/v${elasticsearch.version}/conf</outputDirectory>
<lineEnding>unix</lineEnding>
</fileSet>

<fileSet>
<directory>${basedir}/target</directory>
<includes>
<include>*.jar</include>
</includes>
<excludes>
<exclude>*doc.jar</exclude>
</excludes>
<fileMode>0777</fileMode>
<outputDirectory>/plugin/${elasticsearch.version}</outputDirectory>
</fileSet>

</fileSets>

</assembly>

23 changes: 23 additions & 0 deletions ...-plugins/engineconn-plugins/elasticsearch/src/main/resources/linkis-engineconn.properties
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#

wds.linkis.server.version=v1
#wds.linkis.engineconn.debug.enable=true
wds.linkis.engineconn.plugin.default.class=org.apache.linkis.engineplugin.elasticsearch.ElasticSearchEngineConnPlugin
wds.linkis.engineconn.support.parallelism=true

# ElasticSearch
wds.linkis.es.cluster=127.0.0.1:9200

91 changes: 91 additions & 0 deletions linkis-engineconn-plugins/engineconn-plugins/elasticsearch/src/main/resources/log4j2.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one or more
~ contributor license agreements. See the NOTICE file distributed with
~ this work for additional information regarding copyright ownership.
~ The ASF licenses this file to You under the Apache License, Version 2.0
~ (the "License"); you may not use this file except in compliance with
~ the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing, software
~ distributed under the License is distributed on an "AS IS" BASIS,
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
~ See the License for the specific language governing permissions and
~ limitations under the License.
-->

<configuration status="error" monitorInterval="30">
<appenders>
<Console name="Console" target="SYSTEM_OUT">
<ThresholdFilter level="INFO" onMatch="ACCEPT" onMismatch="DENY"/>
<PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} %-5level [%t] %logger{36} %L %M - %msg%xEx%n"/>
</Console>

<Send name="Send" >
<Filters>
<ThresholdFilter level="WARN" onMatch="ACCEPT" onMismatch="DENY" />
</Filters>
<PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} %-5level [%t] %logger{36} %L %M - %msg%xEx%n"/>
</Send>

<Send name="SendPackage" >
<PatternLayout pattern="%d{yyyy-MM-dd HH:mm:ss.SSS} %-5level [%t] %logger{36} %L %M - %msg%xEx%n"/>
</Send>

<Console name="stderr" target="SYSTEM_ERR">
<ThresholdFilter level="ERROR" onMatch="ACCEPT" onMismatch="DENY" />
<PatternLayout pattern="%d{HH:mm:ss.SSS} %-5level %class{36} %L %M - %msg%xEx%n"/>
</Console>
</appenders>

<loggers>
<root level="INFO">
<appender-ref ref="stderr"/>
<appender-ref ref="Console"/>
<appender-ref ref="Send"/>
</root>
<logger name="org.apache.hadoop.hive.ql.exec.StatsTask" level="info" additivity="true">
<appender-ref ref="SendPackage"/>
</logger>
<logger name="org.springframework.boot.diagnostics.LoggingFailureAnalysisReporter" level="error" additivity="true">
<appender-ref ref="stderr"/>
</logger>
<logger name="com.netflix.discovery" level="warn" additivity="true">
<appender-ref ref="Send"/>
</logger>
<logger name="org.apache.hadoop.yarn" level="warn" additivity="true">
<appender-ref ref="Send"/>
</logger>
<logger name="org.springframework" level="warn" additivity="true">
<appender-ref ref="Send"/>
</logger>
<logger name="org.apache.linkis.server.security" level="warn" additivity="true">
<appender-ref ref="Send"/>
</logger>
<logger name="org.apache.hadoop.hive.ql.exec.mr.ExecDriver" level="fatal" additivity="true">
<appender-ref ref="Send"/>
</logger>
<logger name="org.apache.hadoop.hdfs.KeyProviderCache" level="fatal" additivity="true">
<appender-ref ref="Send"/>
</logger>
<logger name="org.spark_project.jetty" level="ERROR" additivity="true">
<appender-ref ref="Send"/>
</logger>
<logger name="org.eclipse.jetty" level="ERROR" additivity="true">
<appender-ref ref="Send"/>
</logger>
<logger name="org.springframework" level="ERROR" additivity="true">
<appender-ref ref="Send"/>
</logger>
<logger name="org.reflections.Reflections" level="ERROR" additivity="true">
<appender-ref ref="Send"/>
</logger>

<logger name="org.apache.hadoop.ipc.Client" level="ERROR" additivity="true">
<appender-ref ref="Send"/>
</logger>

</loggers>
</configuration>
Loading

0 comments on commit 2b6a149

Please sign in to comment.