From c99e75cdc1a1e141bb83caedd1bd4d817ee07840 Mon Sep 17 00:00:00 2001 From: Jeff Zhang Date: Fri, 18 Feb 2022 13:38:15 +0800 Subject: [PATCH] [HOTFIX] Fix CI failure --- .github/workflows/core.yml | 65 +++++++++++++------ .../zeppelin/spark/SparkInterpreter.java | 10 +-- .../launcher/SparkInterpreterLauncher.java | 1 + 3 files changed, 50 insertions(+), 26 deletions(-) diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index 675717906e1..21a355d22af 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -58,15 +58,14 @@ jobs: activate-environment: python_3_with_R environment-file: testing/env_python_${{ matrix.python }}_with_R.yml python-version: ${{ matrix.python }} + mamba-version: "*" + channels: conda-forge,defaults + channel-priority: true auto-activate-base: false - channel-priority: strict + use-mamba: true - name: Make IRkernel available to Jupyter run: | - R -e "install.packages('devtools', repos = 'http://cran.us.r-project.org')" - R -e "devtools::install_github('IRkernel/IRkernel')" R -e "IRkernel::installspec()" - conda list - conda info - name: install application with some interpreter run: mvn install -Pbuild-distr -DskipRat -DskipTests -pl zeppelin-server,zeppelin-web,spark-submit,spark/spark-dependencies,markdown,angular,shell -am -Phelium-dev -Pexamples -P${{ matrix.hadoop }} -B - name: install and test plugins @@ -102,11 +101,13 @@ jobs: activate-environment: python_3_with_R_and_tensorflow environment-file: testing/env_python_3_with_R_and_tensorflow.yml python-version: 3.7 + mamba-version: "*" + channels: conda-forge,defaults + channel-priority: true auto-activate-base: false + use-mamba: true - name: Make IRkernel available to Jupyter run: | - R -e "install.packages('devtools', repos = 'http://cran.us.r-project.org')" - R -e "devtools::install_github('IRkernel/IRkernel')" R -e "IRkernel::installspec()" - name: verify interpreter run: mvn verify -DskipRat -am -pl .,zeppelin-interpreter,zeppelin-interpreter-shaded,${INTERPRETERS} -Pscala-2.10 -B @@ -137,11 +138,13 @@ jobs: activate-environment: python_3_with_R environment-file: testing/env_python_3_with_R.yml python-version: 3.7 + mamba-version: "*" + channels: conda-forge,defaults + channel-priority: true auto-activate-base: false + use-mamba: true - name: Make IRkernel available to Jupyter run: | - R -e "install.packages('devtools', repos = 'http://cran.us.r-project.org')" - R -e "devtools::install_github('IRkernel/IRkernel')" R -e "IRkernel::installspec()" - name: install environment run: | @@ -180,7 +183,11 @@ jobs: activate-environment: python_3_with_flink environment-file: testing/env_python_3_with_flink_${{ matrix.flink }}.yml python-version: 3.7 + mamba-version: "*" + channels: conda-forge,defaults + channel-priority: true auto-activate-base: false + use-mamba: true - name: install environment run: | mvn install -DskipTests -DskipRat -am -pl flink/flink-scala-2.11,flink/flink-scala-2.12,flink-cmd,zeppelin-interpreter-integration -Pflink-${{ matrix.flink }} -Pintegration -B @@ -214,11 +221,13 @@ jobs: activate-environment: python_3_with_R environment-file: testing/env_python_3_with_R.yml python-version: 3.7 + mamba-version: "*" + channels: conda-forge,defaults + channel-priority: true auto-activate-base: false + use-mamba: true - name: Make IRkernel available to Jupyter run: | - R -e "install.packages('devtools', repos = 'http://cran.us.r-project.org')" - R -e "devtools::install_github('IRkernel/IRkernel')" R -e "IRkernel::installspec()" - name: install environment run: | @@ -257,11 +266,13 @@ jobs: activate-environment: python_3_with_R environment-file: testing/env_python_3_with_R.yml python-version: 3.7 + mamba-version: "*" + channels: conda-forge,defaults + channel-priority: true auto-activate-base: false + use-mamba: true - name: Make IRkernel available to Jupyter run: | - R -e "install.packages('devtools', repos = 'http://cran.us.r-project.org')" - R -e "devtools::install_github('IRkernel/IRkernel')" R -e "IRkernel::installspec()" - name: install environment run: | @@ -297,11 +308,13 @@ jobs: activate-environment: python_3_with_R environment-file: testing/env_python_3_with_R.yml python-version: 3.7 + mamba-version: "*" + channels: conda-forge,defaults + channel-priority: true auto-activate-base: false + use-mamba: true - name: Make IRkernel available to Jupyter run: | - R -e "install.packages('devtools', repos = 'http://cran.us.r-project.org')" - R -e "devtools::install_github('IRkernel/IRkernel')" R -e "IRkernel::installspec()" - name: install environment run: | @@ -340,11 +353,13 @@ jobs: activate-environment: python_3_with_R environment-file: testing/env_python_${{ matrix.python }}_with_R.yml python-version: ${{ matrix.python }} + mamba-version: "*" + channels: conda-forge,defaults + channel-priority: true auto-activate-base: false + use-mamba: true - name: Make IRkernel available to Jupyter run: | - R -e "install.packages('devtools', repos = 'http://cran.us.r-project.org')" - R -e "devtools::install_github('IRkernel/IRkernel')" R -e "IRkernel::installspec()" - name: install environment run: | @@ -383,11 +398,13 @@ jobs: activate-environment: python_3_with_R environment-file: testing/env_python_${{ matrix.python }}_with_R.yml python-version: ${{ matrix.python }} + mamba-version: "*" + channels: conda-forge,defaults + channel-priority: true auto-activate-base: false + use-mamba: true - name: Make IRkernel available to Jupyter run: | - R -e "install.packages('devtools', repos = 'http://cran.us.r-project.org')" - R -e "devtools::install_github('IRkernel/IRkernel')" R -e "IRkernel::installspec()" - name: install environment run: mvn install -DskipTests -DskipRat -pl spark-submit,spark/spark-dependencies -am -Pspark-3.1 -Pspark-scala-2.12 -Phadoop2 -B @@ -425,11 +442,13 @@ jobs: activate-environment: python_3_with_R environment-file: testing/env_python_${{ matrix.python }}_with_R.yml python-version: ${{ matrix.python }} + mamba-version: "*" + channels: conda-forge,defaults + channel-priority: true auto-activate-base: false + use-mamba: true - name: Make IRkernel available to Jupyter run: | - R -e "install.packages('devtools', repos = 'http://cran.us.r-project.org')" - R -e "devtools::install_github('IRkernel/IRkernel')" R -e "IRkernel::installspec()" - name: install environment run: mvn install -DskipTests -DskipRat -pl spark-submit,spark/spark-dependencies -am -Pspark-3.2 -Pspark-scala-2.12 -Phadoop2 -B @@ -463,7 +482,11 @@ jobs: activate-environment: python_3_with_R environment-file: testing/env_python_3_with_R.yml python-version: 3.7 + mamba-version: "*" + channels: conda-forge,defaults + channel-priority: true auto-activate-base: false + use-mamba: true - name: install environment run: | mvn install -DskipTests -DskipRat -pl livy -am -B diff --git a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java index a4c9b76e90c..500fd2842af 100644 --- a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java +++ b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/SparkInterpreter.java @@ -264,17 +264,17 @@ private String extractScalaVersion(SparkConf conf) throws InterpreterException { if (conf.contains("zeppelin.spark.scala.version")) { scalaVersionString = conf.get("zeppelin.spark.scala.version"); } else { - scalaVersionString = scala.util.Properties.versionString(); + scalaVersionString = scala.util.Properties.versionNumberString(); } LOGGER.info("Using Scala: " + scalaVersionString); if (StringUtils.isEmpty(scalaVersionString)) { throw new InterpreterException("Scala Version is empty"); - } else if (scalaVersionString.contains("2.10")) { + } else if (scalaVersionString.startsWith("2.10")) { return "2.10"; - } else if (scalaVersionString.contains("2.11")) { + } else if (scalaVersionString.startsWith("2.11")) { return "2.11"; - } else if (scalaVersionString.contains("2.12")) { + } else if (scalaVersionString.startsWith("2.12")) { return "2.12"; } else { throw new InterpreterException("Unsupported scala version: " + scalaVersionString); @@ -286,7 +286,7 @@ public boolean isScala212() throws InterpreterException { } public boolean isScala210() throws InterpreterException { - return extractScalaVersion().equals("2.10"); + return scalaVersion.equals("2.10"); } private List getDependencyFiles() throws InterpreterException { diff --git a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java index 2aa7ccbe791..4277cee3dd9 100644 --- a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java +++ b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java @@ -121,6 +121,7 @@ public Map buildEnvFromProperties(InterpreterLaunchContext conte String scalaVersion = null; try { scalaVersion = detectSparkScalaVersion(getEnv("SPARK_HOME"), env); + LOGGER.info("Scala version: {}", scalaVersion); context.getProperties().put("zeppelin.spark.scala.version", scalaVersion); } catch (Exception e) { throw new IOException("Fail to detect scala version, the reason is:"+ e.getMessage());