Skip to content

Commit

Permalink
[ML] [Transforms] prefer secondary auth headers for transforms (#86757)
Browse files Browse the repository at this point in the history
When creating and updating transforms, it is possible for clients to provide secondary headers.

When PUT, _preview, _update is called with secondary authorization headers, those are then used or stored with the transform.

closes: #86731
  • Loading branch information
benwtrent authored May 16, 2022
1 parent 132633e commit b90b345
Show file tree
Hide file tree
Showing 13 changed files with 379 additions and 194 deletions.
5 changes: 5 additions & 0 deletions docs/changelog/86757.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
pr: 86757
summary: "Prefer secondary auth headers for transforms"
area: Transform
type: enhancement
issues: []
9 changes: 7 additions & 2 deletions docs/reference/transform/apis/preview-transform.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,15 @@ Previews a {transform}.

Requires the following privileges:

* cluster: `manage_transform` (the `transform_admin` built-in role grants this
* cluster: `manage_transform` (the `transform_admin` built-in role grants this
privilege)
* source indices: `read`, `view_index_metadata`.

+
--
NOTE: If you provide
<<http-clients-secondary-authorization,secondary authorization headers>>, those
credentials are used.
--
[[preview-transform-desc]]
== {api-description-title}

Expand Down
6 changes: 6 additions & 0 deletions docs/reference/transform/apis/put-transform.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,12 @@ Requires the following privileges:
* source indices: `read`, `view_index_metadata`
* destination index: `read`, `create_index`, `index`. If a `retention_policy` is configured, the `delete` privilege is
also required.
+
--
NOTE: If you provide
<<http-clients-secondary-authorization,secondary authorization headers>>, those
credentials are used.
--

[[put-transform-desc]]
== {api-description-title}
Expand Down
4 changes: 3 additions & 1 deletion docs/reference/transform/apis/update-transform.asciidoc
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,9 @@ each checkpoint.
* When {es} {security-features} are enabled, your {transform} remembers which
roles the user who updated it had at the time of update and runs with those
privileges.
privileges. If you provide
<<http-clients-secondary-authorization,secondary authorization headers>>, those
credentials are used instead.
* You must use {kib} or this API to update a {transform}. Do not update a
{transform} directly via `.transform-internal*` indices using the {es} index API.
If {es} {security-features} are enabled, do not give users any privileges on
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,12 +42,14 @@

public class TransformPivotRestIT extends TransformRestTestCase {

private static final String TEST_USER_NAME_NO_ACCESS = "no_authorization";
private static final String TEST_USER_NAME = "transform_admin_plus_data";
private static final String DATA_ACCESS_ROLE = "test_data_access";
private static final String BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS = basicAuthHeaderValue(
TEST_USER_NAME,
TEST_PASSWORD_SECURE_STRING
);
private static final String BASIC_AUTH_VALUE_NO_ACCESS = basicAuthHeaderValue(TEST_USER_NAME_NO_ACCESS, TEST_PASSWORD_SECURE_STRING);

private static boolean indicesCreated = false;

Expand Down Expand Up @@ -96,6 +98,34 @@ public void testSimplePivot() throws Exception {
assertOneCount(transformIndex + "/_search?q=reviewer:user_26", "hits.hits._source.affiliate_missing", 0);
}

public void testSimplePivotWithSecondaryHeaders() throws Exception {
setupUser(TEST_USER_NAME_NO_ACCESS, List.of("transform_admin"));
String transformId = "simple-pivot";
String transformIndex = "pivot_reviews";
setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, transformIndex);
createPivotReviewsTransform(
transformId,
transformIndex,
null,
null,
BASIC_AUTH_VALUE_NO_ACCESS,
BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS,
REVIEWS_INDEX_NAME
);
startAndWaitForTransform(
transformId,
transformIndex,
BASIC_AUTH_VALUE_NO_ACCESS,
BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS,
new String[0]
);

// we expect 27 documents as there shall be 27 user_id's
// Just need to validate that things ran with secondary headers
Map<String, Object> indexStats = getAsMap(transformIndex + "/_stats");
assertEquals(27, XContentMapValues.extractValue("_all.total.docs.count", indexStats));
}

public void testSimpleDataStreamPivot() throws Exception {
String indexName = "reviews_data_stream";
createReviewsIndex(indexName, 1000, 27, "date", true, -1, null);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
public abstract class TransformRestTestCase extends ESRestTestCase {

protected static final String TEST_PASSWORD = "x-pack-test-password";
private static final String SECONDARY_AUTH_KEY = "es-secondary-authorization";
protected static final SecureString TEST_PASSWORD_SECURE_STRING = new SecureString(TEST_PASSWORD.toCharArray());
private static final String BASIC_AUTH_VALUE_SUPER_USER = basicAuthHeaderValue("x_pack_rest_user", TEST_PASSWORD_SECURE_STRING);

Expand Down Expand Up @@ -267,7 +268,7 @@ protected void createContinuousPivotReviewsTransform(String transformId, String
}
}""".formatted(transformIndex, REVIEWS_INDEX_NAME);

createReviewsTransform(transformId, authHeader, config);
createReviewsTransform(transformId, authHeader, null, config);
}

protected void createPivotReviewsTransform(
Expand All @@ -277,6 +278,18 @@ protected void createPivotReviewsTransform(
String pipeline,
String authHeader,
String sourceIndex
) throws IOException {
createPivotReviewsTransform(transformId, transformIndex, query, pipeline, authHeader, null, sourceIndex);
}

protected void createPivotReviewsTransform(
String transformId,
String transformIndex,
String query,
String pipeline,
String authHeader,
String secondaryAuthHeader,
String sourceIndex
) throws IOException {
String config = "{";

Expand Down Expand Up @@ -326,7 +339,7 @@ protected void createPivotReviewsTransform(
"frequency": "1s"
}""";

createReviewsTransform(transformId, authHeader, config);
createReviewsTransform(transformId, authHeader, secondaryAuthHeader, config);
}

protected void createLatestReviewsTransform(String transformId, String transformIndex) throws IOException {
Expand All @@ -347,11 +360,17 @@ protected void createLatestReviewsTransform(String transformId, String transform
"frequency": "1s"
}""".formatted(transformIndex, REVIEWS_INDEX_NAME);

createReviewsTransform(transformId, null, config);
createReviewsTransform(transformId, null, null, config);
}

private void createReviewsTransform(String transformId, String authHeader, String config) throws IOException {
final Request createTransformRequest = createRequestWithAuth("PUT", getTransformEndpoint() + transformId, authHeader);
private void createReviewsTransform(String transformId, String authHeader, String secondaryAuthHeader, String config)
throws IOException {
final Request createTransformRequest = createRequestWithSecondaryAuth(
"PUT",
getTransformEndpoint() + transformId,
authHeader,
secondaryAuthHeader
);
createTransformRequest.setJsonEntity(config);

Map<String, Object> createTransformResponse = entityAsMap(client().performRequest(createTransformRequest));
Expand All @@ -360,7 +379,7 @@ private void createReviewsTransform(String transformId, String authHeader, Strin

protected void createPivotReviewsTransform(String transformId, String transformIndex, String query, String pipeline, String authHeader)
throws IOException {
createPivotReviewsTransform(transformId, transformIndex, query, pipeline, authHeader, REVIEWS_INDEX_NAME);
createPivotReviewsTransform(transformId, transformIndex, query, pipeline, authHeader, null, REVIEWS_INDEX_NAME);
}

protected void startTransform(String transformId) throws IOException {
Expand All @@ -369,7 +388,18 @@ protected void startTransform(String transformId) throws IOException {

protected void startTransform(String transformId, String authHeader, String... warnings) throws IOException {
// start the transform
final Request startTransformRequest = createRequestWithAuth("POST", getTransformEndpoint() + transformId + "/_start", authHeader);
startTransform(transformId, authHeader, null, warnings);
}

protected void startTransform(String transformId, String authHeader, String secondaryAuthHeader, String... warnings)
throws IOException {
// start the transform
final Request startTransformRequest = createRequestWithSecondaryAuth(
"POST",
getTransformEndpoint() + transformId + "/_start",
authHeader,
secondaryAuthHeader
);
if (warnings.length > 0) {
startTransformRequest.setOptions(expectWarnings(warnings));
}
Expand Down Expand Up @@ -404,8 +434,18 @@ protected void startAndWaitForTransform(String transformId, String transformInde

protected void startAndWaitForTransform(String transformId, String transformIndex, String authHeader, String... warnings)
throws Exception {
startAndWaitForTransform(transformId, transformIndex, authHeader, null, warnings);
}

protected void startAndWaitForTransform(
String transformId,
String transformIndex,
String authHeader,
String secondaryAuthHeader,
String... warnings
) throws Exception {
// start the transform
startTransform(transformId, authHeader, warnings);
startTransform(transformId, authHeader, secondaryAuthHeader, warnings);
assertTrue(indexExists(transformIndex));
// wait until the transform has been created and all data is available
waitForTransformCheckpoint(transformId);
Expand Down Expand Up @@ -435,18 +475,29 @@ protected void resetTransform(String transformId, boolean force) throws IOExcept
assertThat(resetTransformResponse.get("acknowledged"), equalTo(Boolean.TRUE));
}

protected Request createRequestWithAuth(final String method, final String endpoint, final String authHeader) {
protected Request createRequestWithSecondaryAuth(
final String method,
final String endpoint,
final String authHeader,
final String secondaryAuthHeader
) {
final Request request = new Request(method, endpoint);

RequestOptions.Builder options = request.getOptions().toBuilder();
if (authHeader != null) {
RequestOptions.Builder options = request.getOptions().toBuilder();
options.addHeader("Authorization", authHeader);
request.setOptions(options);
}

if (secondaryAuthHeader != null) {
options.addHeader(SECONDARY_AUTH_KEY, secondaryAuthHeader);
}
request.setOptions(options);
return request;
}

protected Request createRequestWithAuth(final String method, final String endpoint, final String authHeader) {
return createRequestWithSecondaryAuth(method, endpoint, authHeader, null);
}

void waitForTransformStopped(String transformId) throws Exception {
assertBusy(() -> { assertEquals("stopped", getTransformState(transformId)); }, 15, TimeUnit.SECONDS);
}
Expand Down
Loading

0 comments on commit b90b345

Please sign in to comment.