Skip to content

Commit

Permalink
Merge branch 'main' of github.com:elastic/elasticsearch into ml-embed…
Browse files Browse the repository at this point in the history
…ding-type
  • Loading branch information
jonathan-buttner committed Mar 26, 2024
2 parents 6b8f052 + daf46b5 commit 463833c
Show file tree
Hide file tree
Showing 110 changed files with 5,188 additions and 1,677 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,142 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0 and the Server Side Public License, v 1; you may not use this file except
* in compliance with, at your election, the Elastic License 2.0 or the Server
* Side Public License, v 1.
*/

package org.elasticsearch.gradle.internal.shadow;

import com.github.jengelman.gradle.plugins.shadow.ShadowStats;
import com.github.jengelman.gradle.plugins.shadow.relocation.RelocateClassContext;
import com.github.jengelman.gradle.plugins.shadow.relocation.Relocator;
import com.github.jengelman.gradle.plugins.shadow.transformers.Transformer;
import com.github.jengelman.gradle.plugins.shadow.transformers.TransformerContext;

import org.apache.commons.io.IOUtils;
import org.apache.tools.zip.ZipEntry;
import org.apache.tools.zip.ZipOutputStream;
import org.gradle.api.file.FileTreeElement;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

import java.io.BufferedInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.List;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.transform.TransformerException;
import javax.xml.transform.TransformerFactory;
import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;

public class XmlClassRelocationTransformer implements Transformer {

boolean hasTransformedResource = false;

private Document doc;

private String resource;

@Override
public boolean canTransformResource(FileTreeElement element) {
String path = element.getRelativePath().getPathString();
if (resource != null && resource.equals(path)) {
return true;
}
return false;
}

@Override
public void transform(TransformerContext context) {
try {
BufferedInputStream bis = new BufferedInputStream(context.getIs());
DocumentBuilderFactory dbFactory = DocumentBuilderFactory.newInstance();
DocumentBuilder dBuilder = dbFactory.newDocumentBuilder();
doc = dBuilder.parse(bis);
doc.getDocumentElement().normalize();
Node root = doc.getDocumentElement();
walkThroughNodes(root, context);
if (hasTransformedResource == false) {
this.doc = null;
}
} catch (Exception e) {
throw new RuntimeException("Error parsing xml file in " + context.getIs(), e);
}
}

private static String getRelocatedClass(String className, TransformerContext context) {
List<Relocator> relocators = context.getRelocators();
ShadowStats stats = context.getStats();
if (className != null && className.length() > 0 && relocators != null) {
for (Relocator relocator : relocators) {
if (relocator.canRelocateClass(className)) {
RelocateClassContext relocateClassContext = new RelocateClassContext(className, stats);
return relocator.relocateClass(relocateClassContext);
}
}
}

return className;
}

private void walkThroughNodes(Node node, TransformerContext context) {
if (node.getNodeType() == Node.TEXT_NODE) {
String nodeValue = node.getNodeValue();
if (nodeValue.isBlank() == false) {
String relocatedClass = getRelocatedClass(nodeValue, context);
if (relocatedClass.equals(nodeValue) == false) {
node.setNodeValue(relocatedClass);
hasTransformedResource = true;
}
}
}
NodeList nodeList = node.getChildNodes();
for (int i = 0; i < nodeList.getLength(); i++) {
Node currentNode = nodeList.item(i);
walkThroughNodes(currentNode, context);
}
}

@Override
public boolean hasTransformedResource() {
return hasTransformedResource;
}

@Override
public void modifyOutputStream(ZipOutputStream os, boolean preserveFileTimestamps) {
ZipEntry entry = new ZipEntry(resource);
entry.setTime(TransformerContext.getEntryTimestamp(preserveFileTimestamps, entry.getTime()));

try {
// Write the content back to the XML file
TransformerFactory transformerFactory = TransformerFactory.newInstance();
DOMSource source = new DOMSource(doc);

// Result stream will be a ByteArrayOutputStream
ByteArrayOutputStream baos = new ByteArrayOutputStream();
StreamResult result = new StreamResult(baos);
// Do the transformation and serialization
transformerFactory.newTransformer().transform(source, result);
os.putNextEntry(entry);
IOUtils.write(baos.toByteArray(), os);
os.closeEntry();
} catch (IOException e) {
throw new RuntimeException(e);
} catch (TransformerException e) {
throw new RuntimeException(e);
} finally {
hasTransformedResource = false;
doc = null;
}
}

@Override
public String getName() {
return getClass().getSimpleName();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

package org.elasticsearch.gradle.internal.test.rest;

import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask;
import org.elasticsearch.gradle.internal.test.RestIntegTestTask;
import org.elasticsearch.gradle.util.GradleUtils;
import org.gradle.api.Plugin;
import org.gradle.api.Project;
Expand Down Expand Up @@ -40,13 +40,7 @@ public void apply(Project project) {
}

// setup the javaRestTest task
// we use a StandloneRestIntegTestTask here so that the conventions of RestTestBasePlugin don't create a test cluster
TaskProvider<StandaloneRestIntegTestTask> testTask = registerTestTask(
project,
javaTestSourceSet,
SOURCE_SET_NAME,
StandaloneRestIntegTestTask.class
);
TaskProvider<RestIntegTestTask> testTask = registerTestTask(project, javaTestSourceSet, SOURCE_SET_NAME, RestIntegTestTask.class);

project.getTasks().named(JavaBasePlugin.CHECK_TASK_NAME).configure(check -> check.dependsOn(testTask));

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

package org.elasticsearch.gradle.internal.test.rest;

import org.elasticsearch.gradle.testclusters.StandaloneRestIntegTestTask;
import org.elasticsearch.gradle.internal.test.RestIntegTestTask;
import org.elasticsearch.gradle.util.GradleUtils;
import org.gradle.api.Plugin;
import org.gradle.api.Project;
Expand Down Expand Up @@ -36,12 +36,7 @@ public void apply(Project project) {
SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
SourceSet yamlTestSourceSet = sourceSets.create(SOURCE_SET_NAME);

TaskProvider<StandaloneRestIntegTestTask> testTask = registerTestTask(
project,
yamlTestSourceSet,
SOURCE_SET_NAME,
StandaloneRestIntegTestTask.class
);
TaskProvider<RestIntegTestTask> testTask = registerTestTask(project, yamlTestSourceSet, SOURCE_SET_NAME, RestIntegTestTask.class);

project.getTasks().named(JavaBasePlugin.CHECK_TASK_NAME).configure(check -> check.dependsOn(testTask));

Expand Down
5 changes: 5 additions & 0 deletions docs/changelog/105454.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
pr: 105454
summary: "ESQL: Sum of constants"
area: ES|QL
type: enhancement
issues: []
6 changes: 6 additions & 0 deletions docs/changelog/106708.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
pr: 106708
summary: Improve error message when rolling over DS alias
area: Data streams
type: bug
issues:
- 106137
8 changes: 4 additions & 4 deletions docs/reference/inference/delete-inference.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,9 @@ own model, use the <<ml-df-trained-models-apis>>.
[[delete-inference-api-request]]
==== {api-request-title}

`DELETE /_inference/<model_id>`
`DELETE /_inference/<inference_id>`

`DELETE /_inference/<task_type>/<model_id>`
`DELETE /_inference/<task_type>/<inference_id>`

[discrete]
[[delete-inference-api-prereqs]]
Expand All @@ -32,9 +32,9 @@ own model, use the <<ml-df-trained-models-apis>>.
[[delete-inference-api-path-params]]
==== {api-path-parms-title}

<model_id>::
<inference_id>::
(Required, string)
The unique identifier of the {infer} model to delete.
The unique identifier of the {infer} endpoint to delete.

<task_type>::
(Optional, string)
Expand Down
10 changes: 5 additions & 5 deletions docs/reference/inference/get-inference.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,11 @@ own model, use the <<ml-df-trained-models-apis>>.

`GET /_inference/_all`

`GET /_inference/<model_id>`
`GET /_inference/<inference_id>`

`GET /_inference/<task_type>/_all`

`GET /_inference/<task_type>/<model_id>`
`GET /_inference/<task_type>/<inference_id>`

[discrete]
[[get-inference-api-prereqs]]
Expand All @@ -47,9 +47,9 @@ and a wildcard expression,
[[get-inference-api-path-params]]
==== {api-path-parms-title}

`<model_id>`::
`<inference_id>`::
(Optional, string)
The unique identifier of the {infer} model.
The unique identifier of the {infer} endpoint.


`<task_type>`::
Expand Down Expand Up @@ -77,7 +77,7 @@ The API returns the following response:
[source,console-result]
------------------------------------------------------------
{
"model_id": "my-elser-model",
"inference_id": "my-elser-model",
"task_type": "sparse_embedding",
"service": "elser",
"service_settings": {
Expand Down
12 changes: 6 additions & 6 deletions docs/reference/inference/post-inference.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,9 @@ own model, use the <<ml-df-trained-models-apis>>.
[[post-inference-api-request]]
==== {api-request-title}

`POST /_inference/<model_id>`
`POST /_inference/<inference_id>`

`POST /_inference/<task_type>/<model_id>`
`POST /_inference/<task_type>/<inference_id>`


[discrete]
Expand All @@ -32,8 +32,8 @@ own model, use the <<ml-df-trained-models-apis>>.
[[post-inference-api-desc]]
==== {api-description-title}

The perform {infer} API enables you to use {infer} models to perform specific
tasks on data that you provide as an input. The API returns a response with the
The perform {infer} API enables you to use {ml} models to perform specific tasks
on data that you provide as an input. The API returns a response with the
resutls of the tasks. The {infer} model you use can perform one specific task
that has been defined when the model was created with the <<put-inference-api>>.

Expand All @@ -42,9 +42,9 @@ that has been defined when the model was created with the <<put-inference-api>>.
[[post-inference-api-path-params]]
==== {api-path-parms-title}

`<model_id>`::
`<inference_id>`::
(Required, string)
The unique identifier of the {infer} model.
The unique identifier of the {infer} endpoint.


`<task_type>`::
Expand Down
20 changes: 10 additions & 10 deletions docs/reference/inference/put-inference.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ or if you want to use non-NLP models, use the <<ml-df-trained-models-apis>>.
[[put-inference-api-desc]]
==== {api-description-title}

The create {infer} API enables you to create and configure an {infer} model to
The create {infer} API enables you to create and configure a {ml} model to
perform a specific {infer} task.

The following services are available through the {infer} API:
Expand All @@ -50,9 +50,9 @@ The following services are available through the {infer} API:
==== {api-path-parms-title}


`<model_id>`::
`<inference_id>`::
(Required, string)
The unique identifier of the model.
The unique identifier of the {infer} endpoint.

`<task_type>`::
(Required, string)
Expand Down Expand Up @@ -246,7 +246,7 @@ This section contains example API calls for every service type.
[[inference-example-cohere]]
===== Cohere service

The following example shows how to create an {infer} model called
The following example shows how to create an {infer} endpoint called
`cohere_embeddings` to perform a `text_embedding` task type.

[source,console]
Expand All @@ -268,7 +268,7 @@ PUT _inference/text_embedding/cohere-embeddings
[[inference-example-e5]]
===== E5 via the elasticsearch service

The following example shows how to create an {infer} model called
The following example shows how to create an {infer} endpoint called
`my-e5-model` to perform a `text_embedding` task type.

[source,console]
Expand All @@ -293,7 +293,7 @@ further details, refer to the {ml-docs}/ml-nlp-e5.html[E5 model documentation].
[[inference-example-elser]]
===== ELSER service

The following example shows how to create an {infer} model called
The following example shows how to create an {infer} endpoint called
`my-elser-model` to perform a `sparse_embedding` task type.

[source,console]
Expand All @@ -315,7 +315,7 @@ Example response:
[source,console-result]
------------------------------------------------------------
{
"model_id": "my-elser-model",
"inference_id": "my-elser-model",
"task_type": "sparse_embedding",
"service": "elser",
"service_settings": {
Expand All @@ -332,7 +332,7 @@ Example response:
[[inference-example-hugging-face]]
===== Hugging Face service

The following example shows how to create an {infer} model called
The following example shows how to create an {infer} endpoint called
`hugging-face_embeddings` to perform a `text_embedding` task type.

[source,console]
Expand Down Expand Up @@ -362,7 +362,7 @@ after the endpoint initialization has been finished.
[[inference-example-eland]]
===== Models uploaded by Eland via the elasticsearch service

The following example shows how to create an {infer} model called
The following example shows how to create an {infer} endpoint called
`my-msmarco-minilm-model` to perform a `text_embedding` task type.

[source,console]
Expand All @@ -387,7 +387,7 @@ been
[[inference-example-openai]]
===== OpenAI service

The following example shows how to create an {infer} model called
The following example shows how to create an {infer} endpoint called
`openai_embeddings` to perform a `text_embedding` task type.

[source,console]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,9 @@ include::{es-repo-dir}/tab-widgets/inference-api/infer-api-requirements-widget.a

[discrete]
[[infer-text-embedding-task]]
==== Create the inference task
==== Create an inference endpoint

Create the {infer} task by using the <<put-inference-api>>:
Create an {infer} endpoint by using the <<put-inference-api>>:

include::{es-repo-dir}/tab-widgets/inference-api/infer-api-task-widget.asciidoc[]

Expand Down
Loading

0 comments on commit 463833c

Please sign in to comment.