Merge remote-tracking branch 'origin/master' into deprecate_reindex_sort

elastic · Nov 22, 2019 · 459b66c · 459b66c
2 parents 5c49c63 + e9941c2
commit 459b66c
Show file tree

Hide file tree

Showing 108 changed files with 2,685 additions and 1,120 deletions.
diff --git a/buildSrc/build.gradle b/buildSrc/build.gradle
@@ -17,6 +17,7 @@
  * under the License.
  */
 
+import org.gradle.internal.jvm.Jvm
 import org.gradle.util.GradleVersion
 
 plugins {
@@ -168,6 +169,9 @@ if (project != rootProject) {
   forbiddenApisTest.enabled = false
   jarHell.enabled = false
   thirdPartyAudit.enabled = false
+  if (Boolean.parseBoolean(System.getProperty("tests.fips.enabled"))){
+    test.enabled = false
+  }
 
   configurations {
     distribution
@@ -223,8 +227,13 @@ if (project != rootProject) {
   }
   check.dependsOn(integTest)
 
+  // for now we hardcode the tests for our build to use the gradle jvm.
+  tasks.withType(Test).configureEach {
+    it.executable = Jvm.current().getJavaExecutable()
+  }
+
   /*
-   * We alread configure publication and we don't need or want this one that
+   * We already configure publication and we don't need or want this one that
    * comes from the java-gradle-plugin.
    */
   afterEvaluate {

diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GenerateGlobalBuildInfoTask.java
@@ -41,14 +41,12 @@ public class GenerateGlobalBuildInfoTask extends DefaultTask {
     private final RegularFileProperty outputFile;
     private final RegularFileProperty compilerVersionFile;
     private final RegularFileProperty runtimeVersionFile;
-    private final RegularFileProperty fipsJvmFile;
 
     @Inject
     public GenerateGlobalBuildInfoTask(ObjectFactory objectFactory) {
         this.outputFile = objectFactory.fileProperty();
         this.compilerVersionFile = objectFactory.fileProperty();
         this.runtimeVersionFile = objectFactory.fileProperty();
-        this.fipsJvmFile = objectFactory.fileProperty();
     }
 
     @Input
@@ -113,11 +111,6 @@ public RegularFileProperty getRuntimeVersionFile() {
         return runtimeVersionFile;
     }
 
-    @OutputFile
-    public RegularFileProperty getFipsJvmFile() {
-        return fipsJvmFile;
-    }
-
     @TaskAction
     public void generate() {
         String javaVendorVersion = System.getProperty("java.vendor.version", System.getProperty("java.vendor"));
@@ -130,7 +123,6 @@ public void generate() {
         String runtimeJavaVersionDetails = gradleJavaVersionDetails;
         JavaVersion runtimeJavaVersionEnum = JavaVersion.current();
         File gradleJavaHome = Jvm.current().getJavaHome();
-        boolean inFipsJvm = false;
 
         try {
             if (Files.isSameFile(compilerJavaHome.toPath(), gradleJavaHome.toPath()) == false) {
@@ -146,8 +138,6 @@ public void generate() {
                 if (runtimeJavaHome.exists()) {
                     runtimeJavaVersionDetails = findJavaVersionDetails(runtimeJavaHome);
                     runtimeJavaVersionEnum = JavaVersion.toVersion(findJavaSpecificationVersion(runtimeJavaHome));
-
-                    inFipsJvm = Boolean.parseBoolean(System.getProperty("tests.fips.enabled"));
                 } else {
                     throw new RuntimeException("Runtime Java home path of '" + compilerJavaHome + "' does not exist");
                 }
@@ -213,7 +203,6 @@ public void generate() {
 
         writeToFile(compilerVersionFile.getAsFile().get(), compilerJavaVersionEnum.name());
         writeToFile(runtimeVersionFile.getAsFile().get(), runtimeJavaVersionEnum.name());
-        writeToFile(fipsJvmFile.getAsFile().get(), Boolean.toString(inFipsJvm));
     }
 
     private void writeToFile(File file, String content) {

diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/GlobalBuildInfoPlugin.java
@@ -76,14 +76,12 @@ public void apply(Project project) {
                 task.getOutputFile().set(new File(project.getBuildDir(), "global-build-info"));
                 task.getCompilerVersionFile().set(new File(project.getBuildDir(), "java-compiler-version"));
                 task.getRuntimeVersionFile().set(new File(project.getBuildDir(), "java-runtime-version"));
-                task.getFipsJvmFile().set(new File(project.getBuildDir(), "in-fips-jvm"));
             });
 
         PrintGlobalBuildInfoTask printTask = project.getTasks().create("printGlobalBuildInfo", PrintGlobalBuildInfoTask.class, task -> {
             task.getBuildInfoFile().set(generateTask.getOutputFile());
             task.getCompilerVersionFile().set(generateTask.getCompilerVersionFile());
             task.getRuntimeVersionFile().set(generateTask.getRuntimeVersionFile());
-            task.getFipsJvmFile().set(generateTask.getFipsJvmFile());
             task.setGlobalInfoListeners(extension.listeners);
         });
 
@@ -103,6 +101,7 @@ public void apply(Project project) {
             params.setIsCi(System.getenv("JENKINS_URL") != null);
             params.setIsInternal(GlobalBuildInfoPlugin.class.getResource("/buildSrc.marker") != null);
             params.setDefaultParallel(findDefaultParallel(project));
+            params.setInFipsJvm(isInFipsJvm());
         });
 
         project.allprojects(p -> {
@@ -153,6 +152,10 @@ private static String getJavaHomeEnvVarName(String version) {
         return "JAVA" + version + "_HOME";
     }
 
+    private static boolean isInFipsJvm() {
+        return Boolean.parseBoolean(System.getProperty("tests.fips.enabled"));
+    }
+
     private static String getResourceContents(String resourcePath) {
         try (BufferedReader reader = new BufferedReader(
             new InputStreamReader(GlobalBuildInfoPlugin.class.getResourceAsStream(resourcePath))

diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java b/buildSrc/src/main/java/org/elasticsearch/gradle/info/PrintGlobalBuildInfoTask.java
@@ -16,15 +16,13 @@ public class PrintGlobalBuildInfoTask extends DefaultTask {
     private final RegularFileProperty buildInfoFile;
     private final RegularFileProperty compilerVersionFile;
     private final RegularFileProperty runtimeVersionFile;
-    private final RegularFileProperty fipsJvmFile;
     private List<Runnable> globalInfoListeners = new ArrayList<>();
 
     @Inject
     public PrintGlobalBuildInfoTask(ObjectFactory objectFactory) {
         this.buildInfoFile = objectFactory.fileProperty();
         this.compilerVersionFile = objectFactory.fileProperty();
         this.runtimeVersionFile = objectFactory.fileProperty();
-        this.fipsJvmFile = objectFactory.fileProperty();
     }
 
     @InputFile
@@ -42,11 +40,6 @@ public RegularFileProperty getRuntimeVersionFile() {
         return runtimeVersionFile;
     }
 
-    @InputFile
-    public RegularFileProperty getFipsJvmFile() {
-        return fipsJvmFile;
-    }
-
     public void setGlobalInfoListeners(List<Runnable> globalInfoListeners) {
         this.globalInfoListeners = globalInfoListeners;
     }
@@ -57,6 +50,7 @@ public void print() {
         getLogger().quiet("Elasticsearch Build Hamster says Hello!");
         getLogger().quiet(getFileText(getBuildInfoFile()).asString());
         getLogger().quiet("  Random Testing Seed   : " + BuildParams.getTestSeed());
+        getLogger().quiet("  In FIPS 140 mode      : " + BuildParams.isInFipsJvm());
         getLogger().quiet("=======================================");
 
         setGlobalProperties();
@@ -76,7 +70,6 @@ private void setGlobalProperties() {
         BuildParams.init(params -> {
             params.setCompilerJavaVersion(JavaVersion.valueOf(getFileText(getCompilerVersionFile()).asString()));
             params.setRuntimeJavaVersion(JavaVersion.valueOf(getFileText(getRuntimeVersionFile()).asString()));
-            params.setInFipsJvm(Boolean.parseBoolean(getFileText(getFipsJvmFile()).asString()));
         });
     }
 }
diff --git a/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java b/buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java
@@ -415,6 +415,11 @@ public synchronized void start() {
         } catch (IOException e) {
             throw new UncheckedIOException("Failed to create working directory for " + this, e);
         }
+
+        copyExtraJars();
+
+        copyExtraConfigFiles();
+
         createConfiguration();
 
         if (plugins.isEmpty() == false) {
@@ -438,7 +443,7 @@ public synchronized void start() {
             runElaticsearchBinScript("elasticsearch-keystore", "create");
 
             keystoreSettings.forEach((key, value) ->
-                runElaticsearchBinScriptWithInput(value.toString(), "elasticsearch-keystore", "add", "-x", key)
+                runElasticsearchBinScriptWithInput(value.toString(), "elasticsearch-keystore", "add", "-x", key)
             );
 
             for (Map.Entry<String, File> entry : keystoreFiles.entrySet()) {
@@ -453,10 +458,6 @@ public synchronized void start() {
 
         installModules();
 
-        copyExtraConfigFiles();
-
-        copyExtraJars();
-
         if (isSettingTrue("xpack.security.enabled")) {
             if (credentials.isEmpty()) {
                 user(Collections.emptyMap());
@@ -622,7 +623,7 @@ public void user(Map<String, String> userSpec) {
         credentials.add(cred);
     }
 
-    private void runElaticsearchBinScriptWithInput(String input, String tool, String... args) {
+    private void runElasticsearchBinScriptWithInput(String input, String tool, String... args) {
         if (
             Files.exists(getDistroDir().resolve("bin").resolve(tool)) == false &&
                 Files.exists(getDistroDir().resolve("bin").resolve(tool + ".bat")) == false
@@ -663,7 +664,7 @@ private void runElaticsearchBinScriptWithInput(String input, String tool, String
     }
 
     private void runElaticsearchBinScript(String tool, String... args) {
-        runElaticsearchBinScriptWithInput("", tool, args);
+        runElasticsearchBinScriptWithInput("", tool, args);
     }
 
     private Map<String, String> getESEnvironment() {
@@ -676,6 +677,10 @@ private Map<String, String> getESEnvironment() {
         if (systemProperties.isEmpty() == false) {
             systemPropertiesString = " " + systemProperties.entrySet().stream()
                 .map(entry -> "-D" + entry.getKey() + "=" + entry.getValue())
+                // ES_PATH_CONF is also set as an environment variable and for a reference to ${ES_PATH_CONF}
+                // to work ES_JAVA_OPTS, we need to make sure that ES_PATH_CONF before ES_JAVA_OPTS. Instead,
+                // we replace the reference with the actual value in other environment variables
+                .map(p -> p.replace("${ES_PATH_CONF}", configFile.getParent().toString()))
                 .collect(Collectors.joining(" "));
         }
         String jvmArgsString = "";

diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/MLRequestConverters.java
@@ -58,6 +58,7 @@
 import org.elasticsearch.client.ml.GetModelSnapshotsRequest;
 import org.elasticsearch.client.ml.GetOverallBucketsRequest;
 import org.elasticsearch.client.ml.GetRecordsRequest;
+import org.elasticsearch.client.ml.GetTrainedModelsRequest;
 import org.elasticsearch.client.ml.MlInfoRequest;
 import org.elasticsearch.client.ml.OpenJobRequest;
 import org.elasticsearch.client.ml.PostCalendarEventRequest;
@@ -709,6 +710,38 @@ static Request estimateMemoryUsage(PutDataFrameAnalyticsRequest estimateRequest)
         return request;
     }
 
+    static Request getTrainedModels(GetTrainedModelsRequest getTrainedModelsRequest) {
+        String endpoint = new EndpointBuilder()
+            .addPathPartAsIs("_ml", "inference")
+            .addPathPart(Strings.collectionToCommaDelimitedString(getTrainedModelsRequest.getIds()))
+            .build();
+        RequestConverters.Params params = new RequestConverters.Params();
+        if (getTrainedModelsRequest.getPageParams() != null) {
+            PageParams pageParams = getTrainedModelsRequest.getPageParams();
+            if (pageParams.getFrom() != null) {
+                params.putParam(PageParams.FROM.getPreferredName(), pageParams.getFrom().toString());
+            }
+            if (pageParams.getSize() != null) {
+                params.putParam(PageParams.SIZE.getPreferredName(), pageParams.getSize().toString());
+            }
+        }
+        if (getTrainedModelsRequest.getAllowNoMatch() != null) {
+            params.putParam(GetTrainedModelsRequest.ALLOW_NO_MATCH,
+                Boolean.toString(getTrainedModelsRequest.getAllowNoMatch()));
+        }
+        if (getTrainedModelsRequest.getDecompressDefinition() != null) {
+            params.putParam(GetTrainedModelsRequest.DECOMPRESS_DEFINITION,
+                Boolean.toString(getTrainedModelsRequest.getDecompressDefinition()));
+        }
+        if (getTrainedModelsRequest.getIncludeDefinition() != null) {
+            params.putParam(GetTrainedModelsRequest.INCLUDE_MODEL_DEFINITION,
+                Boolean.toString(getTrainedModelsRequest.getIncludeDefinition()));
+        }
+        Request request = new Request(HttpGet.METHOD_NAME, endpoint);
+        request.addParameters(params.asMap());
+        return request;
+    }
+
     static Request putFilter(PutFilterRequest putFilterRequest) throws IOException {
         String endpoint = new EndpointBuilder()
             .addPathPartAsIs("_ml")

diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/MachineLearningClient.java
@@ -73,6 +73,8 @@
 import org.elasticsearch.client.ml.GetOverallBucketsResponse;
 import org.elasticsearch.client.ml.GetRecordsRequest;
 import org.elasticsearch.client.ml.GetRecordsResponse;
+import org.elasticsearch.client.ml.GetTrainedModelsRequest;
+import org.elasticsearch.client.ml.GetTrainedModelsResponse;
 import org.elasticsearch.client.ml.MlInfoRequest;
 import org.elasticsearch.client.ml.MlInfoResponse;
 import org.elasticsearch.client.ml.OpenJobRequest;
@@ -2290,4 +2292,48 @@ public Cancellable estimateMemoryUsageAsync(PutDataFrameAnalyticsRequest request
             listener,
             Collections.emptySet());
     }
+
+    /**
+     * Gets trained model configs
+     * <p>
+     * For additional info
+     * see <a href="TODO">
+     *     GET Trained Model Configs documentation</a>
+     *
+     * @param request The {@link GetTrainedModelsRequest}
+     * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return {@link GetTrainedModelsResponse} response object
+     */
+    public GetTrainedModelsResponse getTrainedModels(GetTrainedModelsRequest request,
+                                                     RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(request,
+            MLRequestConverters::getTrainedModels,
+            options,
+            GetTrainedModelsResponse::fromXContent,
+            Collections.emptySet());
+    }
+
+    /**
+     * Gets trained model configs asynchronously and notifies listener upon completion
+     * <p>
+     * For additional info
+     * see <a href="TODO">
+     *     GET Trained Model Configs documentation</a>
+     *
+     * @param request The {@link GetTrainedModelsRequest}
+     * @param options Additional request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener Listener to be notified upon request completion
+     * @return cancellable that may be used to cancel the request
+     */
+    public Cancellable getTrainedModelsAsync(GetTrainedModelsRequest request,
+                                             RequestOptions options,
+                                             ActionListener<GetTrainedModelsResponse> listener) {
+        return restHighLevelClient.performRequestAsyncAndParseEntity(request,
+            MLRequestConverters::getTrainedModels,
+            options,
+            GetTrainedModelsResponse::fromXContent,
+            listener,
+            Collections.emptySet());
+    }
+
 }