ThalesGroup · harsh-takkar-guavus · Mar 30, 2022 · Mar 30, 2022 · Mar 30, 2022 · Mar 30, 2022
diff --git a/.VERSION b/.VERSION
@@ -0,0 +1 @@
+3.2.1,1.0
diff --git a/Dockerfile b/Dockerfile
@@ -0,0 +1,29 @@
+ARG spark_image_tag=3.2.1-hadoop3.3-1.0
+
+FROM artifacts.ggn.in.guavus.com:4244/spark:${spark_image_tag}
+
+ARG spark_uid=185
+
+USER root
+
+RUN apt-get update -y && \
+    apt-get -y install curl && \
+    curl -fSL https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-s3/1.11.901/aws-java-sdk-s3-1.11.901.jar -o /opt/spark/jars/aws-java-sdk-s3-1.11.901.jar && \
+    curl -fSL https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk/1.11.901/aws-java-sdk-1.11.901.jar -o /opt/spark/jars/aws-java-sdk-1.11.901.jar && \
+    curl -fSL https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-core/1.11.901/aws-java-sdk-core-1.11.901.jar -o /opt/spark/jars/aws-java-sdk-core-1.11.901.jar && \
+    curl -fSL https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-dynamodb/1.11.901/aws-java-sdk-dynamodb-1.11.901.jar -o /opt/spark/jars/aws-java-sdk-dynamodb-1.11.901.jar && \
+    curl -fSL https://repo1.maven.org/maven2/org/apache/commons/commons-pool2/2.11.1/commons-pool2-2.11.1.jar -o /opt/spark/jars/commons-pool2-2.11.1.jar && \
+    curl -fSL https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.3.1/hadoop-aws-3.3.1.jar -o /opt/spark/jars/hadoop-aws-3.3.1.jar && \
+    curl -fSL https://repo1.maven.org/maven2/org/apache/spark/spark-avro_2.12/3.2.1/spark-avro_2.12-3.2.1.jar -o /opt/spark/jars/spark-avro_2.12-3.2.1.jar && \
+    curl -fSL https://repo1.maven.org/maven2/org/apache/spark/spark-token-provider-kafka-0-10_2.12/3.2.1/spark-token-provider-kafka-0-10_2.12-3.2.1.jar -o /opt/spark/jars/spark-token-provider-kafka-0-10_2.12-3.2.1.jar && \
+    curl -fSL https://repo1.maven.org/maven2/net/java/dev/jets3t/jets3t/0.9.0/jets3t-0.9.0.jar -o /opt/spark/jars/jets3t-0.9.0.jar && \
+    curl -fSL https://repo1.maven.org/maven2/org/apache/spark/spark-sql-kafka-0-10_2.12/3.2.1/spark-sql-kafka-0-10_2.12-3.2.1.jar -o /opt/spark/jars/spark-sql-kafka-0-10_2.12-3.2.1.jar && \
+    curl -fSL https://repo1.maven.org/maven2/org/apache/kafka/kafka-clients/3.0.0/kafka-clients-3.0.0.jar -o /opt/spark/jars/kafka-clients-3.0.0.jar && \
+    curl -fSL https://repo1.maven.org/maven2/org/apache/spark/spark-streaming-kafka-0-10_2.12/3.2.1/spark-streaming-kafka-0-10_2.12-3.2.1.jar -o /opt/spark/jars/spark-streaming-kafka-0-10_2.12-3.2.1.jar && \
+    curl -fSL https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.13.0/jmx_prometheus_javaagent-0.13.0.jar -o /opt/spark/jars/jmx_prometheus_javaagent-0.13.0.jar
+
+
+ENTRYPOINT [ "/opt/entrypoint.sh" ]
+
+# Specify the User that the actual main process will run as
+USER ${spark_uid}
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -0,0 +1,110 @@
+@Library('jenkins_lib')_
+pipeline
+ {
+    agent {label 'slave'}
+
+    environment {
+
+        project = "apache-spark";
+        buildNum = currentBuild.getNumber() ;
+        //ex. like feat, release, fix
+        buildType = BRANCH_NAME.split("/").first();
+        //ex. like OP-<User-Story ID>
+        branchVersion = BRANCH_NAME.split("/").last().toUpperCase();
+        // Define global environment variables in this section
+    }
+
+    stages {
+        stage("Define Release version") {
+            steps {
+                script {
+                    //Global Lib for Environment Versions Definition
+                    versionDefine()
+                    env.GUAVUS_SPARK_VERSION = "${VERSION}".split(",").first();
+                    env.GUAVUS_DOCKER_VERSION = "${VERSION}".split(",").last();
+                    env.dockerTag = "${GUAVUS_SPARK_VERSION}-hadoop3.3-${GUAVUS_DOCKER_VERSION}-${RELEASE}" 
+                    echo "GUAVUS_SPARK_VERSION : ${GUAVUS_SPARK_VERSION}"
+                    echo "GUAVUS_DOCKER_VERSION : ${GUAVUS_DOCKER_VERSION}"
+                    echo "DOCKER TAG : ${dockerTag}"
+                }
+            }
+        }
+
+        stage("Versioning") {
+            steps {
+                echo "GUAVUS_SPARK_VERSION : ${GUAVUS_SPARK_VERSION}"
+                echo "GUAVUS_DOCKER_VERSION : ${GUAVUS_DOCKER_VERSION}"
+                sh 'mvn versions:set -DnewVersion=${GUAVUS_SPARK_VERSION}'
+            }
+        }
+
+        stage("Initialize Variable") {
+            steps {
+                script {
+                    PUSH_JAR = false;
+                    PUSH_DOCKER = false;
+                    DOCKER_IMAGE_NAME = "spark-opsiq";
+                    PYSPARK_DOCKER_IMAGE_NAME = "spark-py-opsiq";
+                    longCommit = sh(returnStdout: true, script: "git rev-parse HEAD").trim()
+
+                    if( env.buildType in ['release'] )
+                    {
+                        PUSH_JAR = false; // enable it if we have changes in code
+                        PUSH_DOCKER = true;
+                    }
+                    else if ( env.buildType ==~ /PR-.*/ ) {
+                        PUSH_DOCKER = true;
+                    }
+
+                }
+
+            }
+        }
+
+        stage("Push JAR to Maven Artifactory") {
+            when {
+                expression { PUSH_JAR == true }
+            }
+            steps {
+                script {
+                    echo "Pushing JAR to Maven Artifactory"
+                    sh "mvn deploy -U -Dcheckstyle.skip=true -Denforcer.skip=true -DskipTests=true;"
+                }
+            }
+        }
+
+        stage("Build and Push Docker") {
+                    when {
+                        expression { PUSH_DOCKER == true }
+                        }
+                    stages {
+                        stage("Create Docker Image") {
+                            steps {
+                                script {
+                                    echo "Creating docker build..."
+                                    sh "./dev/make-distribution.sh --name guavus_spark-${GUAVUS_SPARK_VERSION}-3.3 -Phive -Phive-thriftserver -Pkubernetes -Phadoop-3.3 -Dhadoop.version=3.3.1"
+                                    sh "./dist/bin/docker-image-tool.sh -r artifacts.ggn.in.guavus.com:4244 -t ${GUAVUS_SPARK_VERSION}-hadoop3.3-${GUAVUS_DOCKER_VERSION} build"
+                                    sh "./dist/bin/docker-image-tool.sh -r artifacts.ggn.in.guavus.com:4244 -t ${GUAVUS_SPARK_VERSION}-hadoop3.3-${GUAVUS_DOCKER_VERSION} push"
+                                    sh "./dist/bin/docker-image-tool.sh -r artifacts.ggn.in.guavus.com:4244 -t ${GUAVUS_SPARK_VERSION}-hadoop3.3-${GUAVUS_DOCKER_VERSION} -p ./resource-managers/kubernetes/docker/src/main/dockerfiles/spark/bindings/python/Dockerfile build"
+                                    sh "./dist/bin/docker-image-tool.sh -r artifacts.ggn.in.guavus.com:4244 -t ${GUAVUS_SPARK_VERSION}-hadoop3.3-${GUAVUS_DOCKER_VERSION} -p ./resource-managers/kubernetes/docker/src/main/dockerfiles/spark/bindings/python/Dockerfile push"
+                                    sh "docker build -t ${DOCKER_IMAGE_NAME} --build-arg GIT_HEAD=${longCommit} --build-arg GIT_BRANCH=${env.BRANCH_NAME} --build-arg VERSION=${dockerTag} --build-arg BUILD_NUMBER=${env.BUILD_NUMBER} ."
+                                    sh "docker build -f PysparkDockerfile -t ${PYSPARK_DOCKER_IMAGE_NAME} --build-arg GIT_HEAD=${longCommit} --build-arg GIT_BRANCH=${env.BRANCH_NAME} --build-arg VERSION=${dockerTag} --build-arg BUILD_NUMBER=${env.BUILD_NUMBER} ."
+                                }
+                            }
+                        }
+
+                        stage("PUSH Docker") {
+                            steps {
+                                script {
+                                    echo "Docker PUSH..."
+                                    docker_push( buildType, DOCKER_IMAGE_NAME )
+                                    docker_push( buildType, PYSPARK_DOCKER_IMAGE_NAME )
+                                }
+                            }
+                        }
+                    }
+                }
+
+    }
+
+ }
diff --git a/PysparkDockerfile b/PysparkDockerfile
@@ -0,0 +1,29 @@
+ARG spark_image_tag=3.2.1-hadoop3.3-1.0
+
+FROM artifacts.ggn.in.guavus.com:4244/spark-py:${spark_image_tag}
+
+ARG spark_uid=185
+
+USER root
+
+RUN apt-get update -y && \
+    apt-get -y install curl && \
+    curl -fSL https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-s3/1.11.901/aws-java-sdk-s3-1.11.901.jar -o /opt/spark/jars/aws-java-sdk-s3-1.11.901.jar && \
+    curl -fSL https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk/1.11.901/aws-java-sdk-1.11.901.jar -o /opt/spark/jars/aws-java-sdk-1.11.901.jar && \
+    curl -fSL https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-core/1.11.901/aws-java-sdk-core-1.11.901.jar -o /opt/spark/jars/aws-java-sdk-core-1.11.901.jar && \
+    curl -fSL https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk-dynamodb/1.11.901/aws-java-sdk-dynamodb-1.11.901.jar -o /opt/spark/jars/aws-java-sdk-dynamodb-1.11.901.jar && \
+    curl -fSL https://repo1.maven.org/maven2/org/apache/commons/commons-pool2/2.11.1/commons-pool2-2.11.1.jar -o /opt/spark/jars/commons-pool2-2.11.1.jar && \
+    curl -fSL https://repo1.maven.org/maven2/org/apache/hadoop/hadoop-aws/3.3.1/hadoop-aws-3.3.1.jar -o /opt/spark/jars/hadoop-aws-3.3.1.jar && \
+    curl -fSL https://repo1.maven.org/maven2/org/apache/spark/spark-avro_2.12/3.2.1/spark-avro_2.12-3.2.1.jar -o /opt/spark/jars/spark-avro_2.12-3.2.1.jar && \
+    curl -fSL https://repo1.maven.org/maven2/org/apache/spark/spark-token-provider-kafka-0-10_2.12/3.2.1/spark-token-provider-kafka-0-10_2.12-3.2.1.jar -o /opt/spark/jars/spark-token-provider-kafka-0-10_2.12-3.2.1.jar && \
+    curl -fSL https://repo1.maven.org/maven2/net/java/dev/jets3t/jets3t/0.9.0/jets3t-0.9.0.jar -o /opt/spark/jars/jets3t-0.9.0.jar && \
+    curl -fSL https://repo1.maven.org/maven2/org/apache/spark/spark-sql-kafka-0-10_2.12/3.2.1/spark-sql-kafka-0-10_2.12-3.2.1.jar -o /opt/spark/jars/spark-sql-kafka-0-10_2.12-3.2.1.jar && \
+    curl -fSL https://repo1.maven.org/maven2/org/apache/kafka/kafka-clients/3.0.0/kafka-clients-3.0.0.jar -o /opt/spark/jars/kafka-clients-3.0.0.jar && \
+    curl -fSL https://repo1.maven.org/maven2/org/apache/spark/spark-streaming-kafka-0-10_2.12/3.2.1/spark-streaming-kafka-0-10_2.12-3.2.1.jar -o /opt/spark/jars/spark-streaming-kafka-0-10_2.12-3.2.1.jar && \
+    curl -fSL https://repo1.maven.org/maven2/io/prometheus/jmx/jmx_prometheus_javaagent/0.13.0/jmx_prometheus_javaagent-0.13.0.jar -o /opt/spark/jars/jmx_prometheus_javaagent-0.13.0.jar
+
+
+ENTRYPOINT [ "/opt/entrypoint.sh" ]
+
+# Specify the User that the actual main process will run as
+USER ${spark_uid}
diff --git a/pom.xml b/pom.xml
@@ -37,50 +37,16 @@
       <distribution>repo</distribution>
     </license>
   </licenses>
-  <scm>
-    <connection>scm:git:[email protected]:apache/spark.git</connection>
-    <developerConnection>scm:git:https://gitbox.apache.org/repos/asf/spark.git</developerConnection>
-    <url>scm:git:[email protected]:apache/spark.git</url>
-    <tag>HEAD</tag>
-  </scm>
-  <developers>
-    <developer>
-      <id>matei</id>
-      <name>Matei Zaharia</name>
-      <email>[email protected]</email>
-      <url>https://cs.stanford.edu/people/matei</url>
-      <organization>Apache Software Foundation</organization>
-      <organizationUrl>http://spark.apache.org</organizationUrl>
-    </developer>
-  </developers>
-  <issueManagement>
-    <system>JIRA</system>
-    <url>https://issues.apache.org/jira/browse/SPARK</url>
-  </issueManagement>
-
-  <mailingLists>
-    <mailingList>
-      <name>Dev Mailing List</name>
-      <post>[email protected]</post>
-      <subscribe>[email protected]</subscribe>
-      <unsubscribe>[email protected]</unsubscribe>
-    </mailingList>
-
-    <mailingList>
-      <name>User Mailing List</name>
-      <post>[email protected]</post>
-      <subscribe>[email protected]</subscribe>
-      <unsubscribe>[email protected]</unsubscribe>
-    </mailingList>
-
-    <mailingList>
-      <name>Commits Mailing List</name>
-      <post>[email protected]</post>
-      <subscribe>[email protected]</subscribe>
-      <unsubscribe>[email protected]</unsubscribe>
-    </mailingList>
-  </mailingLists>
-
+  <distributionManagement>
+    <repository>
+      <id>central</id>
+      <url>http://artifacts.ggn.in.guavus.com/libs-release-local</url>
+    </repository>
+    <snapshotRepository>
+      <id>snapshots</id>
+      <url>http://artifacts.ggn.in.guavus.com/libs-snapshot-local</url>
+    </snapshotRepository>
+  </distributionManagement>
   <modules>
     <module>common/sketch</module>
     <module>common/kvstore</module>

diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml
@@ -162,7 +162,7 @@
         <groupId>org.scalatest</groupId>
         <artifactId>scalatest-maven-plugin</artifactId>
         <configuration>
-          <argLine>-ea -Xmx4g -Xss4m -XX:ReservedCodeCacheSize=${CodeCacheSize} -Dio.netty.tryReflectionSetAccessible=true</argLine>
+          <argLine>-ea -Xmx4g -Xss16m -XX:ReservedCodeCacheSize=${CodeCacheSize} -Dio.netty.tryReflectionSetAccessible=true</argLine>
         </configuration>
       </plugin>
       <plugin>