release assembly-all (#9399)

intel-analytics · Nov 9, 2023 · fb93c0f · fb93c0f
1 parent fc3e1a7
commit fb93c0f
Show file tree

Hide file tree

Showing 10 changed files with 195 additions and 11 deletions.
diff --git a/.github/workflows/bigdl-release-scala.yml b/.github/workflows/bigdl-release-scala.yml
@@ -103,7 +103,7 @@ jobs:
         sed -i 's/<artifactId>bigdl-parent-spark_${spark.version}<\/artifactId>/<artifactId>bigdl-parent-spark_3.1.3<\/artifactId>/' scala/serving/pom.xml
         sed -i 's/<artifactId>bigdl-parent-spark_${spark.version}<\/artifactId>/<artifactId>bigdl-parent-spark_3.1.3<\/artifactId>/' scala/ppml/pom.xml
         sed -i 's/<artifactId>bigdl-parent-spark_${spark.version}<\/artifactId>/<artifactId>bigdl-parent-spark_3.1.3<\/artifactId>/' scala/assembly/pom.xml
-        mvn clean deploy -DskipTests -Dspark.version=3.1.3 -DSPARK_PLATFORM=SPARK_3.1 -P spark_3.x -P sign -X -DstagingProgressTimeoutMinutes=20 -Dgpg.passphrase=BigDL2017 --file scala/pom.xml 
+        mvn clean deploy -DskipTests -Dspark.version=3.1.3 -DSPARK_PLATFORM=SPARK_3.1 -P spark_3.x -P sign -P release-assembly -X -DstagingProgressTimeoutMinutes=20 -Dgpg.passphrase=BigDL2017 --file scala/pom.xml 
         mv scala/pom.xml.origin scala/pom.xml
         mv scala/common/spark-version/pom.xml.origin scala/common/spark-version/pom.xml
         mv scala/common/spark-version/3.0/pom.xml.origin scala/common/spark-version/3.0/pom.xml
@@ -138,7 +138,7 @@ jobs:
         sed -i 's/<artifactId>bigdl-parent-spark_${spark.version}<\/artifactId>/<artifactId>bigdl-parent-spark_2.4.6<\/artifactId>/' scala/serving/pom.xml
         sed -i 's/<artifactId>bigdl-parent-spark_${spark.version}<\/artifactId>/<artifactId>bigdl-parent-spark_2.4.6<\/artifactId>/' scala/ppml/pom.xml
         sed -i 's/<artifactId>bigdl-parent-spark_${spark.version}<\/artifactId>/<artifactId>bigdl-parent-spark_2.4.6<\/artifactId>/' scala/assembly/pom.xml
-        mvn clean deploy -DskipTests -Dspark.version=2.4.6 -DSPARK_PLATFORM=SPARK_2.4 -P spark_2.x -P sign -X -DstagingProgressTimeoutMinutes=20 -Dgpg.passphrase=BigDL2017 --file scala/pom.xml
+        mvn clean deploy -DskipTests -Dspark.version=2.4.6 -DSPARK_PLATFORM=SPARK_2.4 -P spark_2.x -P sign -P release-assembly -X -DstagingProgressTimeoutMinutes=20 -Dgpg.passphrase=BigDL2017 --file scala/pom.xml
         mv scala/pom.xml.origin scala/pom.xml
         mv scala/common/spark-version/pom.xml.origin scala/common/spark-version/pom.xml
         mv scala/common/spark-version/2.0/pom.xml.origin scala/common/spark-version/2.0/pom.xml

diff --git a/scala/assembly/pom.xml b/scala/assembly/pom.xml
@@ -172,7 +172,7 @@
                         </goals>
                         <configuration>
                             <descriptors>
-                                <descriptor>src/main/assembly/assembly.xml</descriptor>
+                                <descriptor>${assembly.xml.path}</descriptor>
                             </descriptors>
                         </configuration>
                     </execution>
@@ -239,7 +239,7 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>
-                <version>3.2.1</version>
+                <version>3.5.1</version>
                 <executions>
                     <execution>
                         <id>without-spark</id>

diff --git a/scala/assembly/src/main/assembly/assembly-all.xml b/scala/assembly/src/main/assembly/assembly-all.xml
@@ -0,0 +1,177 @@
+<assembly xmlns="http://maven.apache.org/ASSEMBLY/2.0.0"
+          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+          xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.0.0 http://maven.apache.org/xsd/assembly-2.0.0.xsd">
+  <id>dist-all</id>
+  <formats>
+    <format>dir</format>
+    <format>zip</format>
+  </formats>
+  <includeBaseDirectory>false</includeBaseDirectory>
+  <fileSets>
+    <fileSet>
+        <outputDirectory>/conf</outputDirectory>
+        <directory>${project.parent.basedir}/dllib/src/main/resources</directory>
+        <includes>
+            <include>spark-bigdl.conf</include>
+        </includes>
+    </fileSet>
+    <fileSet>
+      <outputDirectory>/bin</outputDirectory>
+      <directory>${project.parent.basedir}/../scripts</directory>
+      <includes>
+        <include>*with-dllib.sh</include>
+      </includes>
+    </fileSet>
+    <fileSet>
+      <outputDirectory>/apps</outputDirectory>
+      <directory>${project.parent.basedir}/../apps</directory>
+      <includes>
+        <include>**/*</include>
+      </includes>
+    </fileSet>
+    <fileSet>
+      <outputDirectory>/python</outputDirectory>
+      <directory>${project.parent.basedir}/dllib/target</directory>
+      <includes>
+          <include>bigdl-dllib*-python-api.zip</include>
+      </includes>
+    </fileSet>
+    <fileSet>
+      <outputDirectory>/python</outputDirectory>
+      <directory>${project.parent.basedir}/orca/target</directory>
+      <includes>
+          <include>bigdl-orca*-python-api.zip</include>
+      </includes>
+    </fileSet>
+    <fileSet>
+      <outputDirectory>/python</outputDirectory>
+      <directory>${project.parent.basedir}/friesian/target</directory>
+      <includes>
+          <include>bigdl-friesian*-python-api.zip</include>
+      </includes>
+    </fileSet>
+    <fileSet>
+      <outputDirectory>/python</outputDirectory>
+      <directory>${project.parent.basedir}/serving/target</directory>
+      <includes>
+          <include>bigdl-serving*-python-api.zip</include>
+      </includes>
+    </fileSet>
+    <fileSet>
+      <outputDirectory>/python</outputDirectory>
+      <directory>${project.parent.basedir}/ppml/target</directory>
+      <includes>
+        <include>bigdl-ppml*-python-api.zip</include>
+      </includes>
+    </fileSet>
+    <fileSet>
+      <outputDirectory>/python</outputDirectory>
+      <directory>${project.parent.basedir}/target</directory>
+      <includes>
+          <include>bigdl*-python-api.zip</include>
+      </includes>
+    </fileSet>
+    <fileSet>
+      <outputDirectory>/examples/dllib</outputDirectory>
+      <directory>${project.parent.basedir}/../python/dllib/examples</directory>
+      <includes>
+          <include>/**</include>
+      </includes>
+    </fileSet>
+    <fileSet>
+      <outputDirectory>/examples/orca</outputDirectory>
+      <directory>${project.parent.basedir}/../python/orca/example</directory>
+      <includes>
+          <include>/**</include>
+      </includes>
+    </fileSet>
+    <fileSet>
+      <outputDirectory>/examples/chronos</outputDirectory>
+      <directory>${project.parent.basedir}/../python/chronos/example</directory>
+      <includes>
+          <include>/**</include>
+      </includes>
+    </fileSet>
+    <fileSet>
+      <outputDirectory>/examples/friesian</outputDirectory>
+      <directory>${project.parent.basedir}/../python/friesian/example</directory>
+      <includes>
+          <include>/**</include>
+      </includes>
+    </fileSet>
+    <fileSet>
+      <outputDirectory>/examples/nano</outputDirectory>
+      <directory>${project.parent.basedir}/../python/nano/example</directory>
+      <includes>
+          <include>/**</include>
+      </includes>
+    </fileSet>
+    <fileSet>
+      <outputDirectory>/examples/serving</outputDirectory>
+      <directory>${project.parent.basedir}/../python/serving/example</directory>
+      <includes>
+          <include>/**</include>
+      </includes>
+    </fileSet>
+    <fileSet>
+      <outputDirectory>/</outputDirectory>
+      <directory>${project.parent.basedir}/tpp</directory>
+      <includes>
+          <include>/**</include>
+      </includes>
+    </fileSet>
+    <fileSet>
+      <outputDirectory>/python</outputDirectory>
+      <directory>${project.parent.basedir}/../python/ppml/scripts</directory>
+      <includes>
+          <include>start-fl-server.py</include>
+      </includes>
+    </fileSet>
+    <fileSet>
+      <outputDirectory>/python</outputDirectory>
+      <directory>${project.parent.basedir}/../python/ppml/scripts</directory>
+      <includes>
+          <include>start-fgboost-server.py</include>
+      </includes>
+    </fileSet>
+    <fileSet>
+      <outputDirectory>/python/requirements</outputDirectory>
+      <directory>${project.parent.basedir}/../python/requirements</directory>
+      <includes>
+          <include>/**</include>
+      </includes>
+    </fileSet>
+  </fileSets>
+  <dependencySets>
+    <dependencySet>
+      <outputDirectory>jars</outputDirectory>
+      <includes>
+        <include>com.intel.analytics.bigdl:*:jar</include>
+      </includes>
+      <excludes>
+        <exclude>com.intel.analytics.bigdl:bigdl-assembly*:jar</exclude>
+      </excludes>
+    </dependencySet>
+    <dependencySet>
+      <outputDirectory>jars</outputDirectory>
+      <useTransitiveDependencies>true</useTransitiveDependencies>
+      <unpack>false</unpack>
+      <scope>runtime</scope>
+      <useProjectArtifact>false</useProjectArtifact>
+      <excludes>
+	<exclude>com.intel.analytics.bigdl:*:jar</exclude>
+        <exclude>com.intel.analytics.zoo:*:jar</exclude>
+	<exclude>com.intel.analytics.bigdl.core.dist:all:jar</exclude>
+	<exclude>io.grpc:grpc-netty-shaded:jar</exclude>
+	<exclude>io.netty:netty-tcnative-boringssl-static:jar</exclude>
+	<exclude>com.intel.analytics.bigdl.core.native.opencv:opencv-java-x86_64-linux:jar</exclude>
+	<exclude>org.lz4:lz4-java:jar</exclude>
+        <exclude>org.apache.hadoop:*:jar</exclude>
+        <exclude>org.apache.flink:*:jar</exclude>
+        <exclude>org.apache.spark:*:jar</exclude>
+        <exclude>*:commons*:jar</exclude>
+        <exclude>*:slf4j*:jar</exclude>
+      </excludes>
+    </dependencySet>
+  </dependencySets>
+</assembly>
diff --git a/scala/common/spark-version/pom.xml b/scala/common/spark-version/pom.xml
@@ -27,7 +27,7 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>
-                <version>3.2.1</version>
+                <version>3.5.1</version>
                 <configuration>
                     <artifactSet>
                         <includes>

diff --git a/scala/dllib/pom.xml b/scala/dllib/pom.xml
@@ -472,7 +472,7 @@
                  or shade plugin will be executed after assembly plugin. -->
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>
-                <version>3.2.1</version>
+                <version>3.5.1</version>
                 <configuration>
                     <filters>
                         <filter>

diff --git a/scala/friesian/pom.xml b/scala/friesian/pom.xml
@@ -447,7 +447,7 @@
                  or shade plugin will be executed after assembly plugin. -->
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>
-                <version>3.2.1</version>
+                <version>3.5.1</version>
                 <configuration>
                     <filters>
                         <filter>

diff --git a/scala/orca/pom.xml b/scala/orca/pom.xml
@@ -425,7 +425,7 @@
                  or shade plugin will be executed after assembly plugin. -->
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>
-                <version>3.2.1</version>
+                <version>3.5.1</version>
                 <configuration>
                     <filters>
                         <filter>

diff --git a/scala/pom.xml b/scala/pom.xml
@@ -82,6 +82,7 @@
     </licenses>
 
     <properties>
+        <assembly.xml.path>src/main/assembly/assembly.xml</assembly.xml.path>
         <failIfNoTests>false</failIfNoTests>
         <maven.test.redirectTestOutputToFile>true</maven.test.redirectTestOutputToFile>
 
@@ -167,7 +168,7 @@
         <thrift.version>0.9.2</thrift.version>
         <slf4j.version>1.7.7</slf4j.version>
         <log4j.version>2.17.1</log4j.version>
-        <jackson.version>2.14.1</jackson.version>
+        <jackson.version>2.15.3</jackson.version>
         <jetty.version>6.1.26</jetty.version>
         <jetty.jspapi.version>6.1.14</jetty.jspapi.version>
         <commons-cli.version>1.2</commons-cli.version>
@@ -974,5 +975,11 @@
                 <dist-os-name>${os-flag}</dist-os-name>
             </properties>
         </profile>
+        <profile>
+            <id>release-assembly</id>
+            <properties>
+                <assembly.xml.path>src/main/assembly/assembly-all.xml</assembly.xml.path>
+            </properties>
+        </profile>
     </profiles>
 </project>
diff --git a/scala/ppml/pom.xml b/scala/ppml/pom.xml
@@ -475,7 +475,7 @@
                  or shade plugin will be executed after assembly plugin. -->
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>
-                <version>3.2.1</version>
+                <version>3.5.1</version>
                 <configuration>
                     <filters>
                         <filter>

diff --git a/scala/serving/pom.xml b/scala/serving/pom.xml
@@ -403,7 +403,7 @@
                  or shade plugin will be executed after assembly plugin. -->
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>
-                <version>3.2.1</version>
+                <version>3.5.1</version>
                 <configuration>
                     <filters>
                         <filter>