[ML] [Transforms] prefer secondary auth headers for transforms

docs/changelog/86757.yaml

@@ -0,0 +1,5 @@
+pr: 86757
+summary: "Prefer secondary auth headers for transforms"
+area: Transform
+type: enhancement
+issues: []

docs/reference/transform/apis/preview-transform.asciidoc

@@ -27,10 +27,15 @@ Previews a {transform}.
 
 Requires the following privileges:
 
-* cluster: `manage_transform` (the `transform_admin` built-in role grants this 
+* cluster: `manage_transform` (the `transform_admin` built-in role grants this
   privilege)
 * source indices: `read`, `view_index_metadata`.
-
++
+--
+NOTE:  If you provide
+<<http-clients-secondary-authorization,secondary authorization headers>>, those
+credentials are used.
+--
 [[preview-transform-desc]]
 == {api-description-title}
 

docs/reference/transform/apis/put-transform.asciidoc

@@ -24,6 +24,12 @@ Requires the following privileges:
 * source indices: `read`, `view_index_metadata`
 * destination index: `read`, `create_index`, `index`. If a `retention_policy` is configured, the `delete` privilege is
   also required.
++
+--
+NOTE:  If you provide
+<<http-clients-secondary-authorization,secondary authorization headers>>, those
+credentials are used.
+--
 
 [[put-transform-desc]]
 == {api-description-title}

docs/reference/transform/apis/update-transform.asciidoc

@@ -44,7 +44,9 @@ each checkpoint.
 
 * When {es} {security-features} are enabled, your {transform} remembers which
 roles the user who updated it had at the time of update and runs with those
-privileges.
+privileges. If you provide
+<<http-clients-secondary-authorization,secondary authorization headers>>, those
+credentials are used instead.
 * You must use {kib} or this API to update a {transform}. Do not update a
 {transform} directly via `.transform-internal*` indices using the {es} index API.
 If {es} {security-features} are enabled, do not give users any privileges on

x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformPivotRestIT.java

@@ -42,12 +42,14 @@
 
 public class TransformPivotRestIT extends TransformRestTestCase {
 
+    private static final String TEST_USER_NAME_NO_ACCESS = "no_authorization";
     private static final String TEST_USER_NAME = "transform_admin_plus_data";
     private static final String DATA_ACCESS_ROLE = "test_data_access";
     private static final String BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS = basicAuthHeaderValue(
         TEST_USER_NAME,
         TEST_PASSWORD_SECURE_STRING
     );
+    private static final String BASIC_AUTH_VALUE_NO_ACCESS = basicAuthHeaderValue(TEST_USER_NAME_NO_ACCESS, TEST_PASSWORD_SECURE_STRING);
 
     private static boolean indicesCreated = false;
 
@@ -96,6 +98,34 @@ public void testSimplePivot() throws Exception {
         assertOneCount(transformIndex + "/_search?q=reviewer:user_26", "hits.hits._source.affiliate_missing", 0);
     }
 
+    public void testSimplePivotWithSecondaryHeaders() throws Exception {
+        setupUser(TEST_USER_NAME_NO_ACCESS, List.of("transform_admin"));
+        String transformId = "simple-pivot";
+        String transformIndex = "pivot_reviews";
+        setupDataAccessRole(DATA_ACCESS_ROLE, REVIEWS_INDEX_NAME, transformIndex);
+        createPivotReviewsTransform(
+            transformId,
+            transformIndex,
+            null,
+            null,
+            BASIC_AUTH_VALUE_NO_ACCESS,
+            BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS,
+            REVIEWS_INDEX_NAME
+        );
+        startAndWaitForTransform(
+            transformId,
+            transformIndex,
+            BASIC_AUTH_VALUE_NO_ACCESS,
+            BASIC_AUTH_VALUE_TRANSFORM_ADMIN_WITH_SOME_DATA_ACCESS,
+            new String[0]
+        );
+
+        // we expect 27 documents as there shall be 27 user_id's
+        // Just need to validate that things ran with secondary headers
+        Map<String, Object> indexStats = getAsMap(transformIndex + "/_stats");
+        assertEquals(27, XContentMapValues.extractValue("_all.total.docs.count", indexStats));
+    }
+
     public void testSimpleDataStreamPivot() throws Exception {
         String indexName = "reviews_data_stream";
         createReviewsIndex(indexName, 1000, 27, "date", true, -1, null);

x-pack/plugin/transform/qa/single-node-tests/src/javaRestTest/java/org/elasticsearch/xpack/transform/integration/TransformRestTestCase.java

@@ -43,6 +43,7 @@
 public abstract class TransformRestTestCase extends ESRestTestCase {
 
     protected static final String TEST_PASSWORD = "x-pack-test-password";
+    private static final String SECONDARY_AUTH_KEY = "es-secondary-authorization";
     protected static final SecureString TEST_PASSWORD_SECURE_STRING = new SecureString(TEST_PASSWORD.toCharArray());
     private static final String BASIC_AUTH_VALUE_SUPER_USER = basicAuthHeaderValue("x_pack_rest_user", TEST_PASSWORD_SECURE_STRING);
 
@@ -267,7 +268,7 @@ protected void createContinuousPivotReviewsTransform(String transformId, String
               }
             }""".formatted(transformIndex, REVIEWS_INDEX_NAME);
 
-        createReviewsTransform(transformId, authHeader, config);
+        createReviewsTransform(transformId, authHeader, null, config);
     }
 
     protected void createPivotReviewsTransform(
@@ -277,6 +278,18 @@ protected void createPivotReviewsTransform(
         String pipeline,
         String authHeader,
         String sourceIndex
+    ) throws IOException {
+        createPivotReviewsTransform(transformId, transformIndex, query, pipeline, authHeader, null, sourceIndex);
+    }
+
+    protected void createPivotReviewsTransform(
+        String transformId,
+        String transformIndex,
+        String query,
+        String pipeline,
+        String authHeader,
+        String secondaryAuthHeader,
+        String sourceIndex
     ) throws IOException {
         String config = "{";
 
@@ -326,7 +339,7 @@ protected void createPivotReviewsTransform(
               "frequency": "1s"
             }""";
 
-        createReviewsTransform(transformId, authHeader, config);
+        createReviewsTransform(transformId, authHeader, secondaryAuthHeader, config);
     }
 
     protected void createLatestReviewsTransform(String transformId, String transformIndex) throws IOException {
@@ -347,11 +360,17 @@ protected void createLatestReviewsTransform(String transformId, String transform
               "frequency": "1s"
             }""".formatted(transformIndex, REVIEWS_INDEX_NAME);
 
-        createReviewsTransform(transformId, null, config);
+        createReviewsTransform(transformId, null, null, config);
     }
 
-    private void createReviewsTransform(String transformId, String authHeader, String config) throws IOException {
-        final Request createTransformRequest = createRequestWithAuth("PUT", getTransformEndpoint() + transformId, authHeader);
+    private void createReviewsTransform(String transformId, String authHeader, String secondaryAuthHeader, String config)
+        throws IOException {
+        final Request createTransformRequest = createRequestWithSecondaryAuth(
+            "PUT",
+            getTransformEndpoint() + transformId,
+            authHeader,
+            secondaryAuthHeader
+        );
         createTransformRequest.setJsonEntity(config);
 
         Map<String, Object> createTransformResponse = entityAsMap(client().performRequest(createTransformRequest));
@@ -360,7 +379,7 @@ private void createReviewsTransform(String transformId, String authHeader, Strin
 
     protected void createPivotReviewsTransform(String transformId, String transformIndex, String query, String pipeline, String authHeader)
         throws IOException {
-        createPivotReviewsTransform(transformId, transformIndex, query, pipeline, authHeader, REVIEWS_INDEX_NAME);
+        createPivotReviewsTransform(transformId, transformIndex, query, pipeline, authHeader, null, REVIEWS_INDEX_NAME);
     }
 
     protected void startTransform(String transformId) throws IOException {
@@ -369,7 +388,18 @@ protected void startTransform(String transformId) throws IOException {
 
     protected void startTransform(String transformId, String authHeader, String... warnings) throws IOException {
         // start the transform
-        final Request startTransformRequest = createRequestWithAuth("POST", getTransformEndpoint() + transformId + "/_start", authHeader);
+        startTransform(transformId, authHeader, null, warnings);
+    }
+
+    protected void startTransform(String transformId, String authHeader, String secondaryAuthHeader, String... warnings)
+        throws IOException {
+        // start the transform
+        final Request startTransformRequest = createRequestWithSecondaryAuth(
+            "POST",
+            getTransformEndpoint() + transformId + "/_start",
+            authHeader,
+            secondaryAuthHeader
+        );
         if (warnings.length > 0) {
             startTransformRequest.setOptions(expectWarnings(warnings));
         }
@@ -404,8 +434,18 @@ protected void startAndWaitForTransform(String transformId, String transformInde
 
     protected void startAndWaitForTransform(String transformId, String transformIndex, String authHeader, String... warnings)
         throws Exception {
+        startAndWaitForTransform(transformId, transformIndex, authHeader, null, warnings);
+    }
+
+    protected void startAndWaitForTransform(
+        String transformId,
+        String transformIndex,
+        String authHeader,
+        String secondaryAuthHeader,
+        String... warnings
+    ) throws Exception {
         // start the transform
-        startTransform(transformId, authHeader, warnings);
+        startTransform(transformId, authHeader, secondaryAuthHeader, warnings);
         assertTrue(indexExists(transformIndex));
         // wait until the transform has been created and all data is available
         waitForTransformCheckpoint(transformId);
@@ -435,18 +475,29 @@ protected void resetTransform(String transformId, boolean force) throws IOExcept
         assertThat(resetTransformResponse.get("acknowledged"), equalTo(Boolean.TRUE));
     }
 
-    protected Request createRequestWithAuth(final String method, final String endpoint, final String authHeader) {
+    protected Request createRequestWithSecondaryAuth(
+        final String method,
+        final String endpoint,
+        final String authHeader,
+        final String secondaryAuthHeader
+    ) {
         final Request request = new Request(method, endpoint);
 
+        RequestOptions.Builder options = request.getOptions().toBuilder();
         if (authHeader != null) {
-            RequestOptions.Builder options = request.getOptions().toBuilder();
             options.addHeader("Authorization", authHeader);
-            request.setOptions(options);
         }
-
+        if (secondaryAuthHeader != null) {
+            options.addHeader(SECONDARY_AUTH_KEY, secondaryAuthHeader);
+        }
+        request.setOptions(options);
         return request;
     }
 
+    protected Request createRequestWithAuth(final String method, final String endpoint, final String authHeader) {
+        return createRequestWithSecondaryAuth(method, endpoint, authHeader, null);
+    }
+
     void waitForTransformStopped(String transformId) throws Exception {
         assertBusy(() -> { assertEquals("stopped", getTransformState(transformId)); }, 15, TimeUnit.SECONDS);
     }