diff --git a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest.java b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest.java index cdc040feea2..94c9188c8ad 100644 --- a/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest.java +++ b/zeppelin-interpreter-integration/src/test/java/org/apache/zeppelin/integration/SparkIntegrationTest.java @@ -17,6 +17,7 @@ package org.apache.zeppelin.integration; +import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest; import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse; @@ -46,6 +47,7 @@ import java.io.File; import java.io.FileReader; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.EnumSet; import java.util.List; import java.util.stream.Collectors; @@ -101,6 +103,18 @@ protected void setUpSparkInterpreterSetting(InterpreterSetting interpreterSettin // sub class can customize spark interpreter setting. } + /** + * Configures ivy to download jar libraries only from remote. + * + * @param interpreterSetting + * @throws IOException + */ + private void setupIvySettings(InterpreterSetting interpreterSetting) throws IOException { + File ivysettings = new File(zeppelin.getZeppelinConfDir(), "ivysettings.xml"); + FileUtils.copyToFile(SparkIntegrationTest.class.getResourceAsStream("/ivysettings.xml"), ivysettings); + interpreterSetting.setProperty("spark.jars.ivySettings", ivysettings.getAbsolutePath()); + } + private boolean isHadoopVersionMatch() { String version = org.apache.hadoop.util.VersionInfo.getVersion(); String majorVersion = version.split("\\.")[0]; @@ -184,6 +198,7 @@ public void testLocalMode() throws IOException, YarnException, InterpreterExcept try { setUpSparkInterpreterSetting(sparkInterpreterSetting); + setupIvySettings(sparkInterpreterSetting); testInterpreterBasics(); // no yarn application launched @@ -215,6 +230,7 @@ public void testYarnClientMode() throws IOException, YarnException, InterruptedE try { setUpSparkInterpreterSetting(sparkInterpreterSetting); + setupIvySettings(sparkInterpreterSetting); testInterpreterBasics(); // 1 yarn application launched @@ -270,6 +286,7 @@ public void testYarnClusterMode() throws IOException, YarnException, Interrupted String yarnAppId = null; try { setUpSparkInterpreterSetting(sparkInterpreterSetting); + setupIvySettings(sparkInterpreterSetting); testInterpreterBasics(); // 1 yarn application launched @@ -366,6 +383,6 @@ private String getPythonExec() throws IOException, InterruptedException { if (process.waitFor() != 0) { throw new RuntimeException("Fail to run command: which python."); } - return IOUtils.toString(process.getInputStream()).trim(); + return IOUtils.toString(process.getInputStream(), StandardCharsets.UTF_8).trim(); } } diff --git a/zeppelin-interpreter-integration/src/test/resources/ivysettings.xml b/zeppelin-interpreter-integration/src/test/resources/ivysettings.xml new file mode 100644 index 00000000000..3b872e1aa12 --- /dev/null +++ b/zeppelin-interpreter-integration/src/test/resources/ivysettings.xml @@ -0,0 +1,6 @@ + + + + + +